snapshot of https://github.com/mattermost/mattermost.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
Server CI / Postgres (shard 0) (push) Has been skipped
Server CI / Postgres (shard 3) (push) Has been skipped
BuildEnv Docker Image / build-image (push) Failing after 4s
BuildEnv Docker Image / build-image-fips (push) Failing after 4s
Migration Review & Release Notes / Generate migration review & release notes (push) Failing after 3s
Server CI / Build Environment (push) Failing after 3s
Server CI / Postgres (shard 1) (push) Has been skipped
Tools CI / check-style (mattermost-govet) (push) Failing after 4s
Server CI / Postgres (shard 2) (push) Has been skipped
Tools CI / check-style (mmgotool) (push) Failing after 3s
Server CI / Elasticsearch v8 Compatibility (push) Has been skipped
Server CI / OpenSearch v2 Compatibility (push) Has been skipped
Server CI / Postgres FIPS (shard 0) (push) Has been skipped
Tools CI / Test (mattermost-govet) (push) Failing after 3s
Server CI / Postgres FIPS (shard 1) (push) Has been skipped
Server CI / Postgres FIPS (shard 2) (push) Has been skipped
Tools CI / Test (mmgotool) (push) Failing after 3s
Server CI / Postgres FIPS (shard 3) (push) Has been skipped
Server CI / Run mmctl tests (push) Has been skipped
Server CI / Run mmctl tests (FIPS) (push) Has been skipped
Server CI / Merge Postgres Test Results (push) Has been skipped
Server CI / Merge Postgres FIPS Test Results (push) Has been skipped
Server CI / Check backport migrations (push) Has been skipped
Server CI / Check generated files (push) Has been skipped
Server CI / check-style (push) Has been skipped
Server CI / Vet API (push) Has been skipped
Server CI / Build mattermost server app (push) Has been skipped
Server CI / Server CI Complete (push) Failing after 2s
Server CI / Server CI Artifacts (push) Has been skipped
Server CI / Sentry Upload (push) Has been skipped
Server CI / Server CI Report (push) Failing after 5m1s
Server CI Nightly Race / Race Detector (push) Has been skipped
CodeQL / Analyze (go) (push) Has been cancelled
CodeQL / Analyze (javascript) (push) Has been cancelled
Webapp CI Cache Warm / Warm Webapp node_modules Cache (push) Has been cancelled
E2E CI Cache Warm / Warm E2E npm Registry Cache (push) Has been cancelled
API / build (push) Has been cancelled
Web App CI / check-types (push) Has been cancelled
Web App CI / check-lint (push) Has been cancelled
Web App CI / check-i18n (push) Has been cancelled
Web App CI / check-external-links (push) Has been cancelled
Web App CI / test (platform) (push) Has been cancelled
Web App CI / test (mattermost-redux) (push) Has been cancelled
Web App CI / test (channels shard 1/4) (push) Has been cancelled
Web App CI / test (channels shard 2/4) (push) Has been cancelled
Web App CI / test (channels shard 3/4) (push) Has been cancelled
Web App CI / test (channels shard 4/4) (push) Has been cancelled
Web App CI / upload-coverage (push) Has been cancelled
Web App CI / build (push) Has been cancelled
YAML Lint / yamllint (push) Has been cancelled
Push mirrored docker images / cd/Push mirrored docker images (push) Has been cancelled
docs-cd / Build and deploy docs (push) Has been cancelled
Docs CI / build (push) Has been cancelled
E2E Cache Warm: Cypress deps / Warm Cypress deps cache (push) Has been cancelled
Scorecards supply-chain security / Scorecard analysis (push) Has been cancelled
Server CI Weekly / Postgres FIPS (push) Has been skipped
Server CI Weekly / Run mmctl tests (FIPS) (push) Has been skipped
Server CI Weekly / Postgres with binary parameters (push) Has been skipped
Server CI Weekly / Build Environment (push) Failing after 1s
Server CI Cache Warm / Warm Go Module and Build Cache (push) Failing after 2s
Server CI Nightly Race / Build Environment (push) Failing after 1s
Some checks failed
Server CI / Postgres (shard 0) (push) Has been skipped
Server CI / Postgres (shard 3) (push) Has been skipped
BuildEnv Docker Image / build-image (push) Failing after 4s
BuildEnv Docker Image / build-image-fips (push) Failing after 4s
Migration Review & Release Notes / Generate migration review & release notes (push) Failing after 3s
Server CI / Build Environment (push) Failing after 3s
Server CI / Postgres (shard 1) (push) Has been skipped
Tools CI / check-style (mattermost-govet) (push) Failing after 4s
Server CI / Postgres (shard 2) (push) Has been skipped
Tools CI / check-style (mmgotool) (push) Failing after 3s
Server CI / Elasticsearch v8 Compatibility (push) Has been skipped
Server CI / OpenSearch v2 Compatibility (push) Has been skipped
Server CI / Postgres FIPS (shard 0) (push) Has been skipped
Tools CI / Test (mattermost-govet) (push) Failing after 3s
Server CI / Postgres FIPS (shard 1) (push) Has been skipped
Server CI / Postgres FIPS (shard 2) (push) Has been skipped
Tools CI / Test (mmgotool) (push) Failing after 3s
Server CI / Postgres FIPS (shard 3) (push) Has been skipped
Server CI / Run mmctl tests (push) Has been skipped
Server CI / Run mmctl tests (FIPS) (push) Has been skipped
Server CI / Merge Postgres Test Results (push) Has been skipped
Server CI / Merge Postgres FIPS Test Results (push) Has been skipped
Server CI / Check backport migrations (push) Has been skipped
Server CI / Check generated files (push) Has been skipped
Server CI / check-style (push) Has been skipped
Server CI / Vet API (push) Has been skipped
Server CI / Build mattermost server app (push) Has been skipped
Server CI / Server CI Complete (push) Failing after 2s
Server CI / Server CI Artifacts (push) Has been skipped
Server CI / Sentry Upload (push) Has been skipped
Server CI / Server CI Report (push) Failing after 5m1s
Server CI Nightly Race / Race Detector (push) Has been skipped
CodeQL / Analyze (go) (push) Has been cancelled
CodeQL / Analyze (javascript) (push) Has been cancelled
Webapp CI Cache Warm / Warm Webapp node_modules Cache (push) Has been cancelled
E2E CI Cache Warm / Warm E2E npm Registry Cache (push) Has been cancelled
API / build (push) Has been cancelled
Web App CI / check-types (push) Has been cancelled
Web App CI / check-lint (push) Has been cancelled
Web App CI / check-i18n (push) Has been cancelled
Web App CI / check-external-links (push) Has been cancelled
Web App CI / test (platform) (push) Has been cancelled
Web App CI / test (mattermost-redux) (push) Has been cancelled
Web App CI / test (channels shard 1/4) (push) Has been cancelled
Web App CI / test (channels shard 2/4) (push) Has been cancelled
Web App CI / test (channels shard 3/4) (push) Has been cancelled
Web App CI / test (channels shard 4/4) (push) Has been cancelled
Web App CI / upload-coverage (push) Has been cancelled
Web App CI / build (push) Has been cancelled
YAML Lint / yamllint (push) Has been cancelled
Push mirrored docker images / cd/Push mirrored docker images (push) Has been cancelled
docs-cd / Build and deploy docs (push) Has been cancelled
Docs CI / build (push) Has been cancelled
E2E Cache Warm: Cypress deps / Warm Cypress deps cache (push) Has been cancelled
Scorecards supply-chain security / Scorecard analysis (push) Has been cancelled
Server CI Weekly / Postgres FIPS (push) Has been skipped
Server CI Weekly / Run mmctl tests (FIPS) (push) Has been skipped
Server CI Weekly / Postgres with binary parameters (push) Has been skipped
Server CI Weekly / Build Environment (push) Failing after 1s
Server CI Cache Warm / Warm Go Module and Build Cache (push) Failing after 2s
Server CI Nightly Race / Build Environment (push) Failing after 1s
This commit is contained in:
10
server/platform/shared/filestore/.mockery.yaml
Normal file
10
server/platform/shared/filestore/.mockery.yaml
Normal file
@@ -0,0 +1,10 @@
|
||||
with-expecter: false
|
||||
packages:
|
||||
github.com/mattermost/mattermost/server/v8/platform/shared/filestore:
|
||||
config:
|
||||
all: true
|
||||
dir: "platform/shared/filestore/mocks"
|
||||
filename: "{{.InterfaceName}}.go"
|
||||
mockname: "{{.InterfaceName}}"
|
||||
outpkg: "mocks"
|
||||
note: "Regenerate this file using `make filestore-mocks`."
|
||||
829
server/platform/shared/filestore/azurestore.go
Normal file
829
server/platform/shared/filestore/azurestore.go
Normal file
@@ -0,0 +1,829 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package filestore
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
|
||||
"github.com/google/uuid"
|
||||
"github.com/mattermost/mattermost/server/public/model"
|
||||
"github.com/mattermost/mattermost/server/public/shared/httpservice"
|
||||
"github.com/mattermost/mattermost/server/public/shared/mlog"
|
||||
)
|
||||
|
||||
// azureBlockSize is the chunk size used when staging block blob uploads.
|
||||
// Matches the Azure SDK's default block size for UploadStream and keeps each
|
||||
// StageBlock call well under the per-block REST limit (4000 MiB).
|
||||
const azureBlockSize = 4 * 1024 * 1024
|
||||
|
||||
// AzureFileBackend stores files in Azure Blob Storage. Two authentication
|
||||
// modes are supported: shared key (an account access key configured by the
|
||||
// admin) and Microsoft Entra ID via DefaultAzureCredential (managed identity,
|
||||
// service principal, workload identity, or az login - whichever the host
|
||||
// environment provides).
|
||||
type AzureFileBackend struct {
|
||||
client *azblob.Client
|
||||
// sharedKey holds the account credential when authMode is shared key, and is
|
||||
// nil under default credential. GeneratePublicLink needs it to sign a Service
|
||||
// SAS by hand, since GetSASURL does not allow to change the ContentDisposition
|
||||
// if not at the blob level, and the parsing it does of the URL to retrieve
|
||||
// container and blob names does not support custom endpoints.
|
||||
sharedKey *azblob.SharedKeyCredential
|
||||
authMode string
|
||||
container string
|
||||
pathPrefix string
|
||||
timeout time.Duration
|
||||
presignExpires time.Duration
|
||||
ssl bool
|
||||
}
|
||||
|
||||
var _ FileBackendWithLinkGenerator = (*AzureFileBackend)(nil)
|
||||
|
||||
func NewAzureFileBackend(settings FileBackendSettings) (*AzureFileBackend, error) {
|
||||
if err := settings.CheckMandatoryAzureFields(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
scheme := "https"
|
||||
if !settings.AzureSSL {
|
||||
scheme = "http"
|
||||
}
|
||||
|
||||
serviceURL, err := buildAzureServiceURL(settings.AzureCloud, scheme, settings.AzureStorageAccount, settings.AzureEndpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client, sharedKey, err := newAzureClient(settings, serviceURL, buildAzureClientOptions(settings))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Config.IsValid rejects non-positive timeouts before they reach this
|
||||
// constructor, but direct callers (tests, library users that build a
|
||||
// FileBackendSettings by hand) can still slip a zero or negative value
|
||||
// in. Fall back to a sane default in that case, and log loudly enough
|
||||
// for the substitution to show up if it ever happens in production.
|
||||
timeout := time.Duration(settings.AzureRequestTimeoutMilliseconds) * time.Millisecond
|
||||
if timeout <= 0 {
|
||||
mlog.Warn("AzureRequestTimeoutMilliseconds is non-positive; falling back to 30s default",
|
||||
mlog.Int("value", int(settings.AzureRequestTimeoutMilliseconds)))
|
||||
timeout = 30 * time.Second
|
||||
}
|
||||
|
||||
return &AzureFileBackend{
|
||||
client: client,
|
||||
sharedKey: sharedKey,
|
||||
authMode: settings.AzureAuthMode,
|
||||
container: settings.AzureContainer,
|
||||
pathPrefix: settings.AzurePathPrefix,
|
||||
timeout: timeout,
|
||||
presignExpires: time.Duration(settings.AzurePresignExpiresSeconds) * time.Second,
|
||||
ssl: settings.AzureSSL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b *AzureFileBackend) DriverName() string {
|
||||
return driverAzure
|
||||
}
|
||||
|
||||
// newAzureClient builds an azblob client for the configured authentication
|
||||
// mode. Shared key uses NewClientWithSharedKeyCredential; default credential
|
||||
// uses NewClient with DefaultAzureCredential, which discovers managed
|
||||
// identity, workload identity, service principal env vars, and az login in
|
||||
// that order at runtime.
|
||||
//
|
||||
// The shared-key credential is returned alongside the client when shared-key
|
||||
// auth is in use, so callers (GeneratePublicLink in particular) can sign
|
||||
// Service SAS tokens without round-tripping to Entra ID. It is nil in the
|
||||
// default-credential path; that path obtains a user-delegation credential
|
||||
// lazily at link-generation time.
|
||||
func newAzureClient(settings FileBackendSettings, serviceURL string, clientOptions *azblob.ClientOptions) (*azblob.Client, *azblob.SharedKeyCredential, error) {
|
||||
switch settings.AzureAuthMode {
|
||||
case model.AzureAuthModeDefaultCredential:
|
||||
cred, err := azidentity.NewDefaultAzureCredential(nil)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to create azure default credential: %w", err)
|
||||
}
|
||||
client, err := azblob.NewClient(serviceURL, cred, clientOptions)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to create azure blob client: %w", err)
|
||||
}
|
||||
return client, nil, nil
|
||||
case model.AzureAuthModeSharedKey:
|
||||
cred, err := azblob.NewSharedKeyCredential(settings.AzureStorageAccount, settings.AzureAccessKey)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to create azure shared key credential: %w", err)
|
||||
}
|
||||
client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, cred, clientOptions)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to create azure blob client: %w", err)
|
||||
}
|
||||
return client, cred, nil
|
||||
default:
|
||||
return nil, nil, fmt.Errorf("unknown azure auth mode %q", settings.AzureAuthMode)
|
||||
}
|
||||
}
|
||||
|
||||
// buildAzureClientOptions selects the HTTP transport for the Azure SDK. The
|
||||
// custom cloud routes requests through httpservice's transport, honoring
|
||||
// AllowedUntrustedInternalConnections. The commercial and government clouds use
|
||||
// a fixed Azure host (which may resolve to a private address under Private
|
||||
// Link), so they keep the SDK default transport and only override it to honor
|
||||
// SkipVerify, mirroring the S3 backend.
|
||||
func buildAzureClientOptions(settings FileBackendSettings) *azblob.ClientOptions {
|
||||
if settings.AzureCloud == model.AzureCloudCustom {
|
||||
return &azblob.ClientOptions{
|
||||
ClientOptions: azcore.ClientOptions{
|
||||
Transport: &http.Client{
|
||||
Transport: httpservice.NewTransportForInternalConnections(settings.SkipVerify, settings.AllowedUntrustedInternalConnections),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
if settings.SkipVerify {
|
||||
return &azblob.ClientOptions{
|
||||
ClientOptions: azcore.ClientOptions{
|
||||
Transport: &http.Client{
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildAzureServiceURL renders the Blob service URL that the SDK signs
|
||||
// requests against. The cloud value selects the topology:
|
||||
//
|
||||
// - commercial -> vhost-style against blob.core.windows.net, e.g.
|
||||
// https://{account}.blob.core.windows.net/.
|
||||
// - government -> vhost-style against blob.core.usgovcloudapi.net,
|
||||
// e.g. https://{account}.blob.core.usgovcloudapi.net/.
|
||||
// - custom -> the admin-provided endpoint is the full service URL,
|
||||
// including scheme and storage account (vhost-style for production
|
||||
// Azure, path-style for Azurite or reverse proxies). Mattermost
|
||||
// does not modify the URL.
|
||||
//
|
||||
// Empty cloud is treated as commercial so existing configs that pre-date
|
||||
// this field keep working. Shared-key auth signs against the URL host,
|
||||
// so for custom deployments the admin is responsible for ensuring the
|
||||
// host actually serves the storage account named in the credential.
|
||||
func buildAzureServiceURL(cloud, scheme, account, endpoint string) (string, error) {
|
||||
switch cloud {
|
||||
case model.AzureCloudCommercial, "":
|
||||
if !model.IsValidAzureStorageAccountName(account) {
|
||||
return "", fmt.Errorf("invalid azure storage account name %q", account)
|
||||
}
|
||||
return fmt.Sprintf("%s://%s.blob.core.windows.net/", scheme, account), nil
|
||||
case model.AzureCloudGovernment:
|
||||
if !model.IsValidAzureStorageAccountName(account) {
|
||||
return "", fmt.Errorf("invalid azure storage account name %q", account)
|
||||
}
|
||||
return fmt.Sprintf("%s://%s.blob.core.usgovcloudapi.net/", scheme, account), nil
|
||||
case model.AzureCloudCustom:
|
||||
if endpoint == "" {
|
||||
return "", errors.New("AzureCloud=custom requires AzureEndpoint to be set")
|
||||
}
|
||||
// The admin owns this URL end to end, but we still reject inputs
|
||||
// that the SDK is guaranteed to fail on later (missing scheme,
|
||||
// missing host, a scheme other than http/https) so the
|
||||
// failure mode is a clear configuration error at startup rather
|
||||
// than an opaque SDK error on the first blob request.
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("AzureEndpoint is not a valid URL: %w", err)
|
||||
}
|
||||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return "", fmt.Errorf("AzureEndpoint must use http or https, got %q", endpoint)
|
||||
}
|
||||
if parsed.Host == "" {
|
||||
return "", fmt.Errorf("AzureEndpoint must include a host, got %q", endpoint)
|
||||
}
|
||||
return endpoint, nil
|
||||
default:
|
||||
return "", fmt.Errorf("unknown AzureCloud value %q", cloud)
|
||||
}
|
||||
}
|
||||
|
||||
// prefix joins the configured pathPrefix and the caller-supplied path.
|
||||
// Using a plain path.Join, a value like "foo/../../secret" can escape
|
||||
// the prefix entirely, so we compute the join and verify the result is
|
||||
// the prefix directory itself or a descendant of it. The descendant check
|
||||
// requires a path-separator boundary so a prefix of "mattermost" does not
|
||||
// match a sibling like "mattermost-evil/...". If the joined path escapes,
|
||||
// we fall back to joining the prefix with path.Base, which may drop any
|
||||
// intermediate directories the caller intended.
|
||||
func (b *AzureFileBackend) prefix(p string) string {
|
||||
joined := path.Join(b.pathPrefix, p)
|
||||
if b.pathPrefix == "" {
|
||||
return joined
|
||||
}
|
||||
|
||||
cleanPrefix := strings.TrimSuffix(path.Clean(b.pathPrefix), "/")
|
||||
if joined == cleanPrefix || strings.HasPrefix(joined, cleanPrefix+"/") {
|
||||
return joined
|
||||
}
|
||||
return path.Join(cleanPrefix, path.Base(p))
|
||||
}
|
||||
|
||||
func (b *AzureFileBackend) newBlobClient(p string) *blob.Client {
|
||||
return b.client.ServiceClient().NewContainerClient(b.container).NewBlobClient(b.prefix(p))
|
||||
}
|
||||
|
||||
func (b *AzureFileBackend) newBlockBlobClient(p string) *blockblob.Client {
|
||||
return b.client.ServiceClient().NewContainerClient(b.container).NewBlockBlobClient(b.prefix(p))
|
||||
}
|
||||
|
||||
func (b *AzureFileBackend) newContainerClient() *container.Client {
|
||||
return b.client.ServiceClient().NewContainerClient(b.container)
|
||||
}
|
||||
|
||||
// TestConnection probes the configured container and reports the outcome
|
||||
// using the typed errors shared with the other backends. Container
|
||||
// creation is deliberately out of scope here - callers (Server.Start)
|
||||
// decide whether to provision a missing container via MakeContainer.
|
||||
// That separation keeps a typo in the System Console from silently
|
||||
// provisioning an unwanted container, and matches the S3 contract where
|
||||
// TestConnection returns FileBackendNoBucketError and MakeBucket is an
|
||||
// explicit call.
|
||||
func (b *AzureFileBackend) TestConnection() error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
|
||||
_, err := b.newContainerClient().GetProperties(ctx, nil)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if bloberror.HasCode(err, bloberror.ContainerNotFound) {
|
||||
return &FileBackendNoBucketError{Err: fmt.Errorf("azure container %q does not exist: %w", b.container, err)}
|
||||
}
|
||||
if isAzureAuthError(err) {
|
||||
return &FileBackendAuthError{Err: fmt.Errorf("unable to authenticate against azure blob storage: %w", err)}
|
||||
}
|
||||
return fmt.Errorf("unable to connect to azure blob storage: %w", err)
|
||||
}
|
||||
|
||||
// MakeContainer creates the configured container. Mirrors S3FileBackend.MakeBucket
|
||||
// so callers can opt into container provisioning explicitly. An already-existing
|
||||
// container is treated as success so that concurrent boots (two nodes racing
|
||||
// through TestConnection plus MakeContainer) both converge cleanly.
|
||||
func (b *AzureFileBackend) MakeContainer() error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
|
||||
if _, err := b.newContainerClient().Create(ctx, nil); err != nil {
|
||||
if bloberror.HasCode(err, bloberror.ContainerAlreadyExists) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("unable to create azure container %q: %w", b.container, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *AzureFileBackend) Reader(p string) (ReadCloseSeeker, error) {
|
||||
// Arm the deadline *before* the first network call, then hand the same
|
||||
// timer to the returned reader on success. The previous code only set up
|
||||
// the timer on the happy path, which left GetProperties running against a
|
||||
// no-deadline context.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
timer := time.AfterFunc(b.timeout, cancel)
|
||||
blobClient := b.newBlobClient(p)
|
||||
|
||||
props, err := blobClient.GetProperties(ctx, nil)
|
||||
if err != nil {
|
||||
timer.Stop()
|
||||
cancel()
|
||||
return nil, fmt.Errorf("unable to read file %q: %w", p, err)
|
||||
}
|
||||
if props.ContentLength == nil {
|
||||
timer.Stop()
|
||||
cancel()
|
||||
return nil, fmt.Errorf("missing content length for %q", p)
|
||||
}
|
||||
|
||||
return &azureRangeReader{
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
timer: timer,
|
||||
blobClient: blobClient,
|
||||
size: *props.ContentLength,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b *AzureFileBackend) ReadFile(p string) ([]byte, error) {
|
||||
r, err := b.Reader(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer r.Close()
|
||||
return io.ReadAll(r)
|
||||
}
|
||||
|
||||
func (b *AzureFileBackend) FileExists(p string) (bool, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
|
||||
_, err := b.newBlobClient(p).GetProperties(ctx, nil)
|
||||
if err != nil {
|
||||
if bloberror.HasCode(err, bloberror.BlobNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("unable to check existence of %q: %w", p, err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (b *AzureFileBackend) FileSize(p string) (int64, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
|
||||
props, err := b.newBlobClient(p).GetProperties(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("unable to get size of %q: %w", p, err)
|
||||
}
|
||||
|
||||
return model.SafeDereference(props.ContentLength), nil
|
||||
}
|
||||
|
||||
func (b *AzureFileBackend) FileModTime(p string) (time.Time, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
|
||||
props, err := b.newBlobClient(p).GetProperties(ctx, nil)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("unable to get modification time of %q: %w", p, err)
|
||||
}
|
||||
|
||||
return model.SafeDereference(props.LastModified), nil
|
||||
}
|
||||
|
||||
// CopyFile copies via StartCopyFromURL and polls the resulting blob's copy
|
||||
// status until it succeeds, matching the synchronous semantics that the
|
||||
// FileBackend interface (and the S3 driver via ComposeObject) provides.
|
||||
func (b *AzureFileBackend) CopyFile(oldPath, newPath string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
|
||||
src := b.newBlobClient(oldPath).URL()
|
||||
dst := b.newBlockBlobClient(newPath)
|
||||
if _, err := dst.StartCopyFromURL(ctx, src, nil); err != nil {
|
||||
return fmt.Errorf("unable to copy %q to %q: %w", oldPath, newPath, err)
|
||||
}
|
||||
|
||||
// Poll until the copy reports success. For server-to-server copies within
|
||||
// the same account this is typically synchronous, but the API is
|
||||
// asynchronous in general, so we wait.
|
||||
for {
|
||||
props, err := dst.GetProperties(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read copy status for %q: %w", newPath, err)
|
||||
}
|
||||
if props.CopyStatus == nil {
|
||||
return nil
|
||||
}
|
||||
switch *props.CopyStatus {
|
||||
case blob.CopyStatusTypeSuccess:
|
||||
return nil
|
||||
case blob.CopyStatusTypeFailed, blob.CopyStatusTypeAborted:
|
||||
desc := model.SafeDereference(props.CopyStatusDescription)
|
||||
return fmt.Errorf("azure copy from %q to %q ended in status %q: %q", oldPath, newPath, *props.CopyStatus, desc)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("azure copy from %q to %q did not complete in time: %w", oldPath, newPath, ctx.Err())
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *AzureFileBackend) MoveFile(oldPath, newPath string) error {
|
||||
if err := b.CopyFile(oldPath, newPath); err != nil {
|
||||
return err
|
||||
}
|
||||
return b.RemoveFile(oldPath)
|
||||
}
|
||||
|
||||
func (b *AzureFileBackend) WriteFile(fr io.Reader, p string) (int64, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
return b.WriteFileContext(ctx, fr, p)
|
||||
}
|
||||
|
||||
// stageBlocks reads fr in azureBlockSize chunks and stages each chunk as a
|
||||
// block under a fresh ID. Returns the IDs of the newly staged blocks (in
|
||||
// order) and the total byte count. The caller is responsible for committing
|
||||
// the block list.
|
||||
func (b *AzureFileBackend) stageBlocks(ctx context.Context, bb *blockblob.Client, fr io.Reader, p string) ([]string, int64, error) {
|
||||
buf := make([]byte, azureBlockSize)
|
||||
var ids []string
|
||||
var total int64
|
||||
|
||||
for {
|
||||
n, err := io.ReadFull(fr, buf)
|
||||
if n > 0 {
|
||||
id, idErr := newAzureBlockID()
|
||||
if idErr != nil {
|
||||
return nil, 0, fmt.Errorf("failed to generate azure block id: %w", idErr)
|
||||
}
|
||||
if _, sbErr := bb.StageBlock(ctx, id, &readSeekNopCloser{Reader: bytes.NewReader(buf[:n])}, nil); sbErr != nil {
|
||||
return nil, 0, fmt.Errorf("unable to stage block for %q: %w", p, sbErr)
|
||||
}
|
||||
ids = append(ids, id)
|
||||
total += int64(n)
|
||||
}
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("failed to read input: %w", err)
|
||||
}
|
||||
}
|
||||
return ids, total, nil
|
||||
}
|
||||
|
||||
// WriteFileContext stages the body in fixed-size blocks and commits a fresh
|
||||
// block list. It deliberately does not use the SDK's UploadStream helper:
|
||||
// UploadStream's small-payload fast path falls back to single-shot PutBlob,
|
||||
// which leaves the resulting blob with no committed block list. A subsequent
|
||||
// AppendFile that calls CommitBlockList on that blob would then clobber its
|
||||
// content. Routing every WriteFile through StageBlock + CommitBlockList keeps
|
||||
// AppendFile correct regardless of payload size.
|
||||
//
|
||||
// The caller's context governs the entire upload - no inner timeout is added.
|
||||
// TryWriteFileContext (filesstore.go) relies on this to let long-running
|
||||
// callers like message-export bulk writes opt out of the per-operation
|
||||
// timeout that WriteFile applies by default.
|
||||
func (b *AzureFileBackend) WriteFileContext(ctx context.Context, fr io.Reader, p string) (int64, error) {
|
||||
bb := b.newBlockBlobClient(p)
|
||||
blockIDs, total, err := b.stageBlocks(ctx, bb, fr, p)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if len(blockIDs) == 0 {
|
||||
// Empty input - still need to materialize an empty blob with a
|
||||
// committed block list so AppendFile can target it.
|
||||
id, idErr := newAzureBlockID()
|
||||
if idErr != nil {
|
||||
return 0, fmt.Errorf("failed to generate azure block id: %w", idErr)
|
||||
}
|
||||
if _, sbErr := bb.StageBlock(ctx, id, &readSeekNopCloser{Reader: bytes.NewReader(nil)}, nil); sbErr != nil {
|
||||
return 0, fmt.Errorf("unable to stage empty block for %q: %w", p, sbErr)
|
||||
}
|
||||
blockIDs = append(blockIDs, id)
|
||||
}
|
||||
|
||||
if _, err := bb.CommitBlockList(ctx, blockIDs, nil); err != nil {
|
||||
return 0, fmt.Errorf("unable to commit block list for %q: %w", p, err)
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// AppendFile stages the new chunk as one or more blocks and commits the
|
||||
// existing committed block list plus the newly staged IDs. Each AppendFile
|
||||
// call uploads the new bytes exactly once - no re-download, no
|
||||
// re-concatenate, no re-upload of the prior contents. The S3-style contract
|
||||
// is preserved: returns an error if the target blob does not yet exist;
|
||||
// returns the number of bytes appended (not the resulting total size).
|
||||
//
|
||||
// Refuses to append to a blob that has content but no committed block list
|
||||
// (i.e. was uploaded via Put Blob by another tool - Azure portal, azcopy,
|
||||
// a migration script). Committing a new block list against such a blob
|
||||
// would replace the existing content with only the appended bytes, so
|
||||
// failing loud beats silent data loss.
|
||||
func (b *AzureFileBackend) AppendFile(fr io.Reader, p string) (int64, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
|
||||
bb := b.newBlockBlobClient(p)
|
||||
|
||||
listResp, err := bb.GetBlockList(ctx, blockblob.BlockListTypeCommitted, nil)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("unable to find file %q to append data: %w", p, err)
|
||||
}
|
||||
|
||||
var existingIDs []string
|
||||
if listResp.BlockList.CommittedBlocks != nil {
|
||||
for _, blk := range listResp.BlockList.CommittedBlocks {
|
||||
if blk.Name != nil {
|
||||
existingIDs = append(existingIDs, *blk.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(existingIDs) == 0 {
|
||||
props, propsErr := bb.GetProperties(ctx, nil)
|
||||
if propsErr != nil {
|
||||
return 0, fmt.Errorf("unable to inspect %q before append: %w", p, propsErr)
|
||||
}
|
||||
if model.SafeDereference(props.ContentLength) > 0 {
|
||||
return 0, fmt.Errorf("refusing to append to %q: blob has content but no committed block list (likely written via Put Blob by another tool)", p)
|
||||
}
|
||||
}
|
||||
|
||||
newIDs, total, err := b.stageBlocks(ctx, bb, fr, p)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if _, err := bb.CommitBlockList(ctx, append(existingIDs, newIDs...), nil); err != nil {
|
||||
return 0, fmt.Errorf("unable to commit block list for %q: %w", p, err)
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// GeneratePublicLink returns a time-limited, read-only URL to the blob at path
|
||||
// using a Shared Access Signature. The SAS is auth-mode aware:
|
||||
//
|
||||
// - shared-key auth signs a Service SAS in-process with the stored
|
||||
// SharedKeyCredential.
|
||||
// - default-credential auth fetches a user-delegation key from Entra ID
|
||||
// (one round trip per call) and signs a user-delegation SAS with it.
|
||||
//
|
||||
// This is intended for the export-download flow (App.GeneratePresignURLForExport
|
||||
// and the /exportlink slash command). End users never reach this code path.
|
||||
func (b *AzureFileBackend) GeneratePublicLink(path string) (string, time.Duration, error) {
|
||||
if b.presignExpires <= 0 {
|
||||
return "", 0, errors.New("azure presign expiration is not configured")
|
||||
}
|
||||
|
||||
prefixed := b.prefix(path)
|
||||
|
||||
// Back-date the start by a small fixed amount to absorb minor clock skew
|
||||
// between this host and Azure, matching the azure-sdk-for-go SAS examples:
|
||||
// https://github.com/Azure/azure-sdk-for-go/blob/65c3b792856d9ad7ce0b59c127ce299358e41a01/sdk/storage/azblob/sas/examples_test.go#L71
|
||||
// https://github.com/Azure/azure-sdk-for-go/blob/65c3b792856d9ad7ce0b59c127ce299358e41a01/sdk/storage/azblob/service/examples_test.go#L315
|
||||
const clockSkew = 10 * time.Second
|
||||
start := time.Now().UTC().Add(-clockSkew)
|
||||
expiry := start.Add(b.presignExpires)
|
||||
|
||||
protocol := sas.ProtocolHTTPSandHTTP
|
||||
if b.ssl {
|
||||
protocol = sas.ProtocolHTTPS
|
||||
}
|
||||
|
||||
values := sas.BlobSignatureValues{
|
||||
Protocol: protocol,
|
||||
StartTime: start,
|
||||
ExpiryTime: expiry,
|
||||
Permissions: (&sas.BlobPermissions{Read: true}).String(),
|
||||
ContainerName: b.container,
|
||||
BlobName: prefixed,
|
||||
// Request browsers to download the file instead of rendering it
|
||||
ContentDisposition: "attachment",
|
||||
}
|
||||
|
||||
var (
|
||||
qps sas.QueryParameters
|
||||
err error
|
||||
)
|
||||
switch b.authMode {
|
||||
case model.AzureAuthModeSharedKey:
|
||||
if b.sharedKey == nil {
|
||||
return "", 0, errors.New("shared key credential is unavailable")
|
||||
}
|
||||
qps, err = values.SignWithSharedKey(b.sharedKey)
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("unable to sign service SAS for %q: %w", path, err)
|
||||
}
|
||||
case model.AzureAuthModeDefaultCredential:
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
udc, udcErr := b.client.ServiceClient().GetUserDelegationCredential(ctx, service.KeyInfo{
|
||||
Start: new(start.Format(sas.TimeFormat)),
|
||||
Expiry: new(expiry.Format(sas.TimeFormat)),
|
||||
}, nil)
|
||||
if udcErr != nil {
|
||||
return "", 0, fmt.Errorf("unable to obtain user delegation key for %q: %w", path, udcErr)
|
||||
}
|
||||
qps, err = values.SignWithUserDelegation(udc)
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("unable to sign user-delegation SAS for %q: %w", path, err)
|
||||
}
|
||||
default:
|
||||
return "", 0, fmt.Errorf("unknown azure auth mode %q", b.authMode)
|
||||
}
|
||||
|
||||
return b.newBlobClient(path).URL() + "?" + qps.Encode(), b.presignExpires, nil
|
||||
}
|
||||
|
||||
func (b *AzureFileBackend) RemoveFile(p string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
|
||||
_, err := b.newBlobClient(p).Delete(ctx, nil)
|
||||
if err != nil && !bloberror.HasCode(err, bloberror.BlobNotFound) {
|
||||
return fmt.Errorf("unable to remove file %q: %w", p, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *AzureFileBackend) ListDirectory(p string) ([]string, error) {
|
||||
prefix := b.prefix(p)
|
||||
if prefix != "" && !strings.HasSuffix(prefix, "/") {
|
||||
prefix += "/"
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
|
||||
pager := b.newContainerClient().NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{
|
||||
Prefix: &prefix,
|
||||
})
|
||||
|
||||
var entries []string
|
||||
for pager.More() {
|
||||
page, err := pager.NextPage(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to list directory %q: %w", p, err)
|
||||
}
|
||||
for _, item := range page.Segment.BlobItems {
|
||||
if item.Name == nil {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimPrefix(*item.Name, b.pathPrefix)
|
||||
name = strings.TrimPrefix(name, "/")
|
||||
entries = append(entries, name)
|
||||
}
|
||||
for _, item := range page.Segment.BlobPrefixes {
|
||||
if item.Name == nil {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimPrefix(*item.Name, b.pathPrefix)
|
||||
name = strings.TrimPrefix(name, "/")
|
||||
name = strings.TrimSuffix(name, "/")
|
||||
entries = append(entries, name)
|
||||
}
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func (b *AzureFileBackend) ListDirectoryRecursively(p string) ([]string, error) {
|
||||
prefix := b.prefix(p)
|
||||
if prefix != "" && !strings.HasSuffix(prefix, "/") {
|
||||
prefix += "/"
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
|
||||
pager := b.newContainerClient().NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
|
||||
Prefix: &prefix,
|
||||
})
|
||||
|
||||
var entries []string
|
||||
for pager.More() {
|
||||
page, err := pager.NextPage(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to list directory %q recursively: %w", p, err)
|
||||
}
|
||||
for _, item := range page.Segment.BlobItems {
|
||||
if item.Name == nil {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimPrefix(*item.Name, b.pathPrefix)
|
||||
name = strings.TrimPrefix(name, "/")
|
||||
entries = append(entries, name)
|
||||
}
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func (b *AzureFileBackend) RemoveDirectory(p string) error {
|
||||
files, err := b.ListDirectoryRecursively(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, f := range files {
|
||||
if err := b.RemoveFile(f); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *AzureFileBackend) ZipReader(p string, deflate bool) (io.ReadCloser, error) {
|
||||
method := zip.Store
|
||||
if deflate {
|
||||
method = zip.Deflate
|
||||
}
|
||||
|
||||
pr, pw := io.Pipe()
|
||||
go func() {
|
||||
zw := zip.NewWriter(pw)
|
||||
err := b.writeZip(zw, p, method)
|
||||
if cerr := zw.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
pw.CloseWithError(err)
|
||||
}()
|
||||
return pr, nil
|
||||
}
|
||||
|
||||
func (b *AzureFileBackend) writeZip(zw *zip.Writer, p string, method uint16) error {
|
||||
exists, err := b.FileExists(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if exists {
|
||||
return b.writeZipEntry(zw, p, path.Base(p), method)
|
||||
}
|
||||
|
||||
files, err := b.ListDirectoryRecursively(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
prefix := strings.TrimSuffix(p, "/") + "/"
|
||||
for _, f := range files {
|
||||
rel := strings.TrimPrefix(f, prefix)
|
||||
if err := b.writeZipEntry(zw, f, rel, method); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *AzureFileBackend) writeZipEntry(zw *zip.Writer, blobPath, name string, method uint16) error {
|
||||
r, err := b.Reader(blobPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer r.Close()
|
||||
header := &zip.FileHeader{Name: name, Method: method}
|
||||
header.SetMode(0644)
|
||||
w, err := zw.CreateHeader(header)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(w, r)
|
||||
return err
|
||||
}
|
||||
|
||||
// readSeekNopCloser adapts a Reader+Seeker into a ReadSeekCloser without
|
||||
// closing the underlying source. The Azure SDK's StageBlock signature
|
||||
// requires a ReadSeekCloser.
|
||||
type readSeekNopCloser struct {
|
||||
io.Reader
|
||||
}
|
||||
|
||||
func (r *readSeekNopCloser) Seek(offset int64, whence int) (int64, error) {
|
||||
return r.Reader.(io.Seeker).Seek(offset, whence)
|
||||
}
|
||||
|
||||
func (r *readSeekNopCloser) Close() error { return nil }
|
||||
|
||||
// newAzureBlockID returns a fresh base64-encoded 16-byte random block ID,
|
||||
// generated with github.com/google/uuid - the same library azblob uses
|
||||
// internally for the block IDs it produces in UploadStream. All committed
|
||||
// blocks in a single blob must share the same decoded length, so callers
|
||||
// must use this for both WriteFile and AppendFile staging.
|
||||
//
|
||||
// Per https://learn.microsoft.com/en-us/rest/api/storageservices/put-block:
|
||||
//
|
||||
// For a given blob, all block IDs must be the same length. If a block is
|
||||
// uploaded with a block ID of a different length than the block IDs for any
|
||||
// existing uncommitted blocks, the service returns error response code 400
|
||||
// (Bad Request).
|
||||
func newAzureBlockID() (string, error) {
|
||||
u, err := uuid.NewRandom()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(u[:]), nil
|
||||
}
|
||||
|
||||
func isAzureAuthError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
return bloberror.HasCode(err, bloberror.AuthenticationFailed) ||
|
||||
bloberror.HasCode(err, bloberror.AuthorizationFailure) ||
|
||||
bloberror.HasCode(err, bloberror.InvalidAuthenticationInfo)
|
||||
}
|
||||
160
server/platform/shared/filestore/azurestore_rangereader.go
Normal file
160
server/platform/shared/filestore/azurestore_rangereader.go
Normal file
@@ -0,0 +1,160 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package filestore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
||||
pkgerr "github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// blobDownloader is the subset of *blob.Client used by azureRangeReader.
|
||||
// Defined as an interface so tests can substitute a fake without standing up
|
||||
// a real Azure client.
|
||||
type blobDownloader interface {
|
||||
DownloadStream(ctx context.Context, opts *blob.DownloadStreamOptions) (blob.DownloadStreamResponse, error)
|
||||
}
|
||||
|
||||
// azureRangeReader is a seekable reader over an Azure blob, backed by HTTP
|
||||
// Range requests. A stream is opened lazily on the first Read at the current
|
||||
// offset; Seek closes any open stream so the next Read re-opens it from the
|
||||
// new offset. The context is cancelled either by Close or by a timer set to
|
||||
// the backend's configured timeout, matching the S3 driver's behavior.
|
||||
//
|
||||
// Callers constructing this struct directly must set ctx, cancel and timer;
|
||||
// the methods below assume all three are non-nil.
|
||||
type azureRangeReader struct {
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
timer *time.Timer
|
||||
blobClient blobDownloader
|
||||
size int64
|
||||
offset int64
|
||||
body io.ReadCloser
|
||||
}
|
||||
|
||||
// Compile-time guarantees that azureRangeReader satisfies the interfaces the
|
||||
// app layer relies on. zip.NewReader requires io.ReaderAt for archive
|
||||
// readers (e.g. the bulk-import worker), and the import worker also
|
||||
// type-asserts to a CancelTimeout interface for long-running operations.
|
||||
var (
|
||||
_ ReadCloseSeeker = (*azureRangeReader)(nil)
|
||||
_ io.ReaderAt = (*azureRangeReader)(nil)
|
||||
)
|
||||
|
||||
func (r *azureRangeReader) Read(p []byte) (int, error) {
|
||||
if r.offset >= r.size {
|
||||
return 0, io.EOF
|
||||
}
|
||||
if r.body == nil {
|
||||
resp, err := r.blobClient.DownloadStream(r.ctx, &blob.DownloadStreamOptions{
|
||||
Range: blob.HTTPRange{Offset: r.offset, Count: 0},
|
||||
})
|
||||
if err != nil {
|
||||
return 0, pkgerr.Wrap(err, "failed to open azure range stream")
|
||||
}
|
||||
r.body = resp.Body
|
||||
}
|
||||
n, err := r.body.Read(p)
|
||||
r.offset += int64(n)
|
||||
if err == nil {
|
||||
return n, nil
|
||||
}
|
||||
// Close+drop the body so the caller (or a retry) doesn't read more
|
||||
// from a half-consumed stream, and so Close stays idempotent.
|
||||
r.body.Close()
|
||||
r.body = nil
|
||||
if err == io.EOF && r.offset < r.size {
|
||||
// The remote stream ended before we reached the blob's content
|
||||
// length. Surface that as a truncation rather than a clean EOF
|
||||
// so the caller doesn't accept a partial blob as complete.
|
||||
return n, io.ErrUnexpectedEOF
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (r *azureRangeReader) Seek(offset int64, whence int) (int64, error) {
|
||||
var abs int64
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
abs = offset
|
||||
case io.SeekCurrent:
|
||||
abs = r.offset + offset
|
||||
case io.SeekEnd:
|
||||
abs = r.size + offset
|
||||
default:
|
||||
return 0, pkgerr.Errorf("invalid whence: %d", whence)
|
||||
}
|
||||
if abs < 0 {
|
||||
return 0, pkgerr.Errorf("negative position: %d", abs)
|
||||
}
|
||||
if abs == r.offset {
|
||||
return abs, nil
|
||||
}
|
||||
if r.body != nil {
|
||||
r.body.Close()
|
||||
r.body = nil
|
||||
}
|
||||
r.offset = abs
|
||||
return abs, nil
|
||||
}
|
||||
|
||||
// ReadAt reads len(p) bytes starting at offset off. Each call issues a
|
||||
// dedicated ranged DownloadStream - calls do not affect the cursor that Read
|
||||
// uses, matching the io.ReaderAt contract. This is what the bulk-import
|
||||
// worker needs to feed zip.NewReader on Azure-backed deployments.
|
||||
func (r *azureRangeReader) ReadAt(p []byte, off int64) (int, error) {
|
||||
if off < 0 {
|
||||
return 0, pkgerr.Errorf("negative offset: %d", off)
|
||||
}
|
||||
if off >= r.size {
|
||||
return 0, io.EOF
|
||||
}
|
||||
count := int64(len(p))
|
||||
if remaining := r.size - off; count > remaining {
|
||||
count = remaining
|
||||
}
|
||||
resp, err := r.blobClient.DownloadStream(r.ctx, &blob.DownloadStreamOptions{
|
||||
Range: blob.HTTPRange{Offset: off, Count: count},
|
||||
})
|
||||
if err != nil {
|
||||
return 0, pkgerr.Wrap(err, "failed to open azure range stream")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
n, err := io.ReadFull(resp.Body, p[:count])
|
||||
// io.ReadFull returns ErrUnexpectedEOF when the stream terminates
|
||||
// before count bytes arrive. Only collapse it to io.EOF when we
|
||||
// actually filled the buffer and consumed the blob to the end -
|
||||
// otherwise it is a real truncation that needs to surface so
|
||||
// callers like zip.NewReader do not accept partial content.
|
||||
if err == io.ErrUnexpectedEOF && int64(n) == count && off+int64(n) == r.size {
|
||||
return n, io.EOF
|
||||
}
|
||||
if err == nil && off+int64(n) == r.size {
|
||||
return n, io.EOF
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// CancelTimeout stops the timer that bounds this reader's lifetime, so
|
||||
// long-running consumers (e.g. the bulk-import worker, which can run far
|
||||
// past the default per-operation timeout) can opt out of the automatic
|
||||
// cancellation. Returns false if the timer has already fired.
|
||||
func (r *azureRangeReader) CancelTimeout() bool {
|
||||
return r.timer.Stop()
|
||||
}
|
||||
|
||||
func (r *azureRangeReader) Close() error {
|
||||
if r.timer != nil {
|
||||
r.timer.Stop()
|
||||
}
|
||||
r.cancel()
|
||||
if r.body != nil {
|
||||
return r.body.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
361
server/platform/shared/filestore/azurestore_rangereader_test.go
Normal file
361
server/platform/shared/filestore/azurestore_rangereader_test.go
Normal file
@@ -0,0 +1,361 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package filestore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// trackingReadCloser wraps a Reader and records whether Close was called.
|
||||
type trackingReadCloser struct {
|
||||
io.Reader
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (t *trackingReadCloser) Close() error {
|
||||
t.closed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// fakeDownloader serves bytes from an in-memory blob, records every
|
||||
// DownloadStream call's Range, and hands out trackingReadClosers so tests
|
||||
// can assert close-on-Seek behavior. An optional err short-circuits responses.
|
||||
type fakeDownloader struct {
|
||||
data []byte
|
||||
calls []blob.HTTPRange
|
||||
bodies []*trackingReadCloser
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeDownloader) DownloadStream(_ context.Context, opts *blob.DownloadStreamOptions) (blob.DownloadStreamResponse, error) {
|
||||
if f.err != nil {
|
||||
return blob.DownloadStreamResponse{}, f.err
|
||||
}
|
||||
var rng blob.HTTPRange
|
||||
if opts != nil {
|
||||
rng = opts.Range
|
||||
}
|
||||
f.calls = append(f.calls, rng)
|
||||
|
||||
start := min(max(rng.Offset, 0), int64(len(f.data)))
|
||||
end := int64(len(f.data))
|
||||
if rng.Count > 0 && start+rng.Count < end {
|
||||
end = start + rng.Count
|
||||
}
|
||||
body := &trackingReadCloser{Reader: bytes.NewReader(f.data[start:end])}
|
||||
f.bodies = append(f.bodies, body)
|
||||
|
||||
return blob.DownloadStreamResponse{
|
||||
DownloadResponse: blob.DownloadResponse{Body: body},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// newTestReader returns an azureRangeReader wired to the given fake, with a
|
||||
// long-lived timer so it never fires during the test. Caller must Close it.
|
||||
func newTestReader(t *testing.T, fake *fakeDownloader, size int64) *azureRangeReader {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
timer := time.AfterFunc(time.Hour, cancel)
|
||||
return &azureRangeReader{
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
timer: timer,
|
||||
blobClient: fake,
|
||||
size: size,
|
||||
}
|
||||
}
|
||||
|
||||
func TestRead(t *testing.T) {
|
||||
t.Run("returns EOF at end of blob without downloading", func(t *testing.T) {
|
||||
fake := &fakeDownloader{data: []byte("hello")}
|
||||
r := newTestReader(t, fake, int64(len(fake.data)))
|
||||
defer r.Close()
|
||||
|
||||
_, err := r.Seek(0, io.SeekEnd)
|
||||
require.NoError(t, err)
|
||||
|
||||
n, err := r.Read(make([]byte, 4))
|
||||
require.Equal(t, 0, n)
|
||||
require.Equal(t, io.EOF, err)
|
||||
require.Empty(t, fake.calls, "no download should be issued past end of blob")
|
||||
})
|
||||
|
||||
t.Run("opens stream at current offset", func(t *testing.T) {
|
||||
fake := &fakeDownloader{data: []byte("hello world")}
|
||||
r := newTestReader(t, fake, int64(len(fake.data)))
|
||||
defer r.Close()
|
||||
|
||||
_, err := r.Seek(6, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
|
||||
buf := make([]byte, 5)
|
||||
n, err := io.ReadFull(r, buf)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 5, n)
|
||||
require.Equal(t, "world", string(buf))
|
||||
|
||||
require.Len(t, fake.calls, 1)
|
||||
require.Equal(t, blob.HTTPRange{Offset: 6, Count: 0}, fake.calls[0])
|
||||
})
|
||||
|
||||
t.Run("sequential reads reuse the open stream", func(t *testing.T) {
|
||||
fake := &fakeDownloader{data: []byte("abcdefghij")}
|
||||
r := newTestReader(t, fake, int64(len(fake.data)))
|
||||
defer r.Close()
|
||||
|
||||
buf := make([]byte, 4)
|
||||
_, err := io.ReadFull(r, buf)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "abcd", string(buf))
|
||||
|
||||
_, err = io.ReadFull(r, buf)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "efgh", string(buf))
|
||||
|
||||
require.Len(t, fake.calls, 1, "sequential reads must reuse the open stream")
|
||||
})
|
||||
|
||||
t.Run("propagates download errors", func(t *testing.T) {
|
||||
wantErr := errors.New("boom")
|
||||
fake := &fakeDownloader{data: []byte("xyz"), err: wantErr}
|
||||
r := newTestReader(t, fake, int64(len(fake.data)))
|
||||
defer r.Close()
|
||||
|
||||
_, err := r.Read(make([]byte, 1))
|
||||
require.ErrorIs(t, err, wantErr)
|
||||
})
|
||||
|
||||
t.Run("surfaces truncation when stream EOFs before the blob ends", func(t *testing.T) {
|
||||
// Promised size is larger than what the fake actually serves,
|
||||
// so the body eventually returns io.EOF while r.offset < r.size.
|
||||
// bytes.Reader returns its content + nil first, then 0 + EOF on
|
||||
// the next call, so we drain the bytes before the truncation
|
||||
// is observable.
|
||||
fake := &fakeDownloader{data: []byte("hello")}
|
||||
r := newTestReader(t, fake, int64(len(fake.data))+10)
|
||||
defer r.Close()
|
||||
|
||||
buf := make([]byte, 16)
|
||||
n, err := r.Read(buf)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 5, n)
|
||||
|
||||
// Second call hits EOF from the body before we've reached r.size,
|
||||
// so the reader must surface that as a truncation.
|
||||
n, err = r.Read(buf)
|
||||
require.Equal(t, 0, n)
|
||||
require.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
require.Nil(t, r.body, "body must be released after a truncation error")
|
||||
})
|
||||
}
|
||||
|
||||
func TestReadAt(t *testing.T) {
|
||||
t.Run("reads at the given offset without disturbing the cursor", func(t *testing.T) {
|
||||
fake := &fakeDownloader{data: []byte("abcdefghij")}
|
||||
r := newTestReader(t, fake, int64(len(fake.data)))
|
||||
defer r.Close()
|
||||
|
||||
// Advance the streaming cursor first.
|
||||
_, err := io.ReadFull(r, make([]byte, 3))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(3), r.offset)
|
||||
|
||||
buf := make([]byte, 4)
|
||||
n, err := r.ReadAt(buf, 5)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 4, n)
|
||||
require.Equal(t, "fghi", string(buf))
|
||||
require.Equal(t, int64(3), r.offset, "ReadAt must not touch the streaming offset")
|
||||
})
|
||||
|
||||
t.Run("returns io.EOF when the read lands exactly at the end of the blob", func(t *testing.T) {
|
||||
fake := &fakeDownloader{data: []byte("abcdefghij")}
|
||||
r := newTestReader(t, fake, int64(len(fake.data)))
|
||||
defer r.Close()
|
||||
|
||||
buf := make([]byte, 3)
|
||||
n, err := r.ReadAt(buf, 7)
|
||||
require.Equal(t, io.EOF, err)
|
||||
require.Equal(t, 3, n)
|
||||
require.Equal(t, "hij", string(buf))
|
||||
})
|
||||
|
||||
t.Run("returns io.EOF when off is past the size", func(t *testing.T) {
|
||||
fake := &fakeDownloader{data: []byte("abcdefghij")}
|
||||
r := newTestReader(t, fake, int64(len(fake.data)))
|
||||
defer r.Close()
|
||||
|
||||
n, err := r.ReadAt(make([]byte, 4), 100)
|
||||
require.Equal(t, 0, n)
|
||||
require.Equal(t, io.EOF, err)
|
||||
require.Empty(t, fake.calls, "no download should be issued past end of blob")
|
||||
})
|
||||
|
||||
t.Run("rejects negative offsets", func(t *testing.T) {
|
||||
r := newTestReader(t, &fakeDownloader{}, 10)
|
||||
defer r.Close()
|
||||
|
||||
_, err := r.ReadAt(make([]byte, 1), -1)
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("propagates download errors", func(t *testing.T) {
|
||||
wantErr := errors.New("boom")
|
||||
fake := &fakeDownloader{data: []byte("xyz"), err: wantErr}
|
||||
r := newTestReader(t, fake, int64(len(fake.data)))
|
||||
defer r.Close()
|
||||
|
||||
_, err := r.ReadAt(make([]byte, 1), 0)
|
||||
require.ErrorIs(t, err, wantErr)
|
||||
})
|
||||
|
||||
t.Run("surfaces truncation when stream falls short of the requested count", func(t *testing.T) {
|
||||
// Promised size exceeds the fake's actual data so ReadFull
|
||||
// sees the body terminate before count bytes arrived. That
|
||||
// must surface as ErrUnexpectedEOF, not a clean EOF.
|
||||
fake := &fakeDownloader{data: []byte("hello")}
|
||||
r := newTestReader(t, fake, int64(len(fake.data))+5)
|
||||
defer r.Close()
|
||||
|
||||
buf := make([]byte, 10)
|
||||
n, err := r.ReadAt(buf, 0)
|
||||
require.Equal(t, 5, n)
|
||||
require.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCancelTimeout(t *testing.T) {
|
||||
fake := &fakeDownloader{data: []byte("abc")}
|
||||
r := newTestReader(t, fake, int64(len(fake.data)))
|
||||
defer r.Close()
|
||||
|
||||
require.True(t, r.CancelTimeout(), "first stop should succeed")
|
||||
require.False(t, r.CancelTimeout(), "second stop must report the timer was already stopped")
|
||||
}
|
||||
|
||||
func TestSeek(t *testing.T) {
|
||||
t.Run("absolute from start", func(t *testing.T) {
|
||||
fake := &fakeDownloader{data: bytes.Repeat([]byte("x"), 32)}
|
||||
r := newTestReader(t, fake, int64(len(fake.data)))
|
||||
defer r.Close()
|
||||
|
||||
pos, err := r.Seek(10, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(10), pos)
|
||||
})
|
||||
|
||||
t.Run("relative to current position", func(t *testing.T) {
|
||||
fake := &fakeDownloader{data: bytes.Repeat([]byte("x"), 32)}
|
||||
r := newTestReader(t, fake, int64(len(fake.data)))
|
||||
defer r.Close()
|
||||
|
||||
_, err := r.Seek(10, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
|
||||
pos, err := r.Seek(5, io.SeekCurrent)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(15), pos)
|
||||
})
|
||||
|
||||
t.Run("relative to end", func(t *testing.T) {
|
||||
fake := &fakeDownloader{data: bytes.Repeat([]byte("x"), 32)}
|
||||
r := newTestReader(t, fake, int64(len(fake.data)))
|
||||
defer r.Close()
|
||||
|
||||
pos, err := r.Seek(-4, io.SeekEnd)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(28), pos)
|
||||
})
|
||||
|
||||
t.Run("rejects invalid whence", func(t *testing.T) {
|
||||
r := newTestReader(t, &fakeDownloader{}, 0)
|
||||
defer r.Close()
|
||||
|
||||
_, err := r.Seek(0, 99)
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("rejects negative absolute position", func(t *testing.T) {
|
||||
r := newTestReader(t, &fakeDownloader{}, 10)
|
||||
defer r.Close()
|
||||
|
||||
_, err := r.Seek(-1, io.SeekStart)
|
||||
require.Error(t, err)
|
||||
|
||||
_, err = r.Seek(-20, io.SeekEnd)
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("same offset leaves the open stream untouched", func(t *testing.T) {
|
||||
fake := &fakeDownloader{data: []byte("abcdefgh")}
|
||||
r := newTestReader(t, fake, int64(len(fake.data)))
|
||||
defer r.Close()
|
||||
|
||||
_, err := io.ReadFull(r, make([]byte, 3))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, fake.bodies, 1)
|
||||
openBody := fake.bodies[0]
|
||||
|
||||
pos, err := r.Seek(3, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(3), pos)
|
||||
require.False(t, openBody.closed, "same-offset seek must not close the open stream")
|
||||
|
||||
_, err = io.ReadFull(r, make([]byte, 3))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, fake.calls, 1, "same-offset seek must not trigger a new download")
|
||||
})
|
||||
|
||||
t.Run("different offset closes the open stream and the next read reopens", func(t *testing.T) {
|
||||
fake := &fakeDownloader{data: []byte("abcdefghij")}
|
||||
r := newTestReader(t, fake, int64(len(fake.data)))
|
||||
defer r.Close()
|
||||
|
||||
_, err := io.ReadFull(r, make([]byte, 2))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, fake.bodies, 1)
|
||||
firstBody := fake.bodies[0]
|
||||
|
||||
_, err = r.Seek(7, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
require.True(t, firstBody.closed, "seek to a new offset must close the open stream")
|
||||
|
||||
buf := make([]byte, 3)
|
||||
_, err = io.ReadFull(r, buf)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "hij", string(buf))
|
||||
|
||||
require.Len(t, fake.calls, 2)
|
||||
require.Equal(t, int64(7), fake.calls[1].Offset)
|
||||
})
|
||||
}
|
||||
|
||||
func TestClose(t *testing.T) {
|
||||
t.Run("cancels context and closes the open body", func(t *testing.T) {
|
||||
fake := &fakeDownloader{data: []byte("abcdef")}
|
||||
r := newTestReader(t, fake, int64(len(fake.data)))
|
||||
|
||||
_, err := io.ReadFull(r, make([]byte, 3))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, fake.bodies, 1)
|
||||
|
||||
require.NoError(t, r.Close())
|
||||
require.True(t, fake.bodies[0].closed)
|
||||
require.ErrorIs(t, r.ctx.Err(), context.Canceled)
|
||||
})
|
||||
|
||||
t.Run("works when no stream was opened", func(t *testing.T) {
|
||||
r := newTestReader(t, &fakeDownloader{}, 10)
|
||||
require.NoError(t, r.Close())
|
||||
require.ErrorIs(t, r.ctx.Err(), context.Canceled)
|
||||
})
|
||||
}
|
||||
466
server/platform/shared/filestore/azurestore_test.go
Normal file
466
server/platform/shared/filestore/azurestore_test.go
Normal file
@@ -0,0 +1,466 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package filestore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mattermost/mattermost/server/public/model"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
)
|
||||
|
||||
func TestBuildAzureServiceURL(t *testing.T) {
|
||||
const account = "acmemattermost"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
cloud string
|
||||
scheme string
|
||||
account string // defaults to the package-level account const when empty
|
||||
endpoint string
|
||||
expected string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "commercial cloud, default scheme",
|
||||
cloud: model.AzureCloudCommercial,
|
||||
scheme: "https",
|
||||
expected: "https://acmemattermost.blob.core.windows.net/",
|
||||
},
|
||||
{
|
||||
name: "commercial cloud rejects an account name with a hash character",
|
||||
cloud: model.AzureCloudCommercial,
|
||||
scheme: "https",
|
||||
account: "account#",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "commercial cloud rejects an account name with a slash character",
|
||||
cloud: model.AzureCloudCommercial,
|
||||
scheme: "https",
|
||||
account: "account/",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "government cloud rejects a malformed account name",
|
||||
cloud: model.AzureCloudGovernment,
|
||||
scheme: "https",
|
||||
account: "account#",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "empty cloud falls back to commercial for legacy configs",
|
||||
cloud: "",
|
||||
scheme: "https",
|
||||
expected: "https://acmemattermost.blob.core.windows.net/",
|
||||
},
|
||||
{
|
||||
name: "government cloud uses the usgovcloudapi suffix",
|
||||
cloud: model.AzureCloudGovernment,
|
||||
scheme: "https",
|
||||
expected: "https://acmemattermost.blob.core.usgovcloudapi.net/",
|
||||
},
|
||||
{
|
||||
name: "custom cloud returns the endpoint verbatim (vhost-style)",
|
||||
cloud: model.AzureCloudCustom,
|
||||
scheme: "https",
|
||||
endpoint: "https://acmemattermost.blob.core.windows.net/",
|
||||
expected: "https://acmemattermost.blob.core.windows.net/",
|
||||
},
|
||||
{
|
||||
name: "custom cloud returns the endpoint verbatim (Azurite path-style)",
|
||||
cloud: model.AzureCloudCustom,
|
||||
scheme: "http",
|
||||
endpoint: "http://localhost:10000/devstoreaccount1/",
|
||||
expected: "http://localhost:10000/devstoreaccount1/",
|
||||
},
|
||||
{
|
||||
name: "custom cloud preserves arbitrary paths the admin provides",
|
||||
cloud: model.AzureCloudCustom,
|
||||
scheme: "https",
|
||||
endpoint: "https://blob-proxy.internal.example.com/some/prefix/account/",
|
||||
expected: "https://blob-proxy.internal.example.com/some/prefix/account/",
|
||||
},
|
||||
{
|
||||
name: "custom cloud rejects an empty endpoint",
|
||||
cloud: model.AzureCloudCustom,
|
||||
scheme: "https",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "custom cloud rejects an endpoint missing the scheme",
|
||||
cloud: model.AzureCloudCustom,
|
||||
scheme: "https",
|
||||
endpoint: "acmemattermost.blob.core.windows.net/",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "custom cloud rejects an endpoint with no host",
|
||||
cloud: model.AzureCloudCustom,
|
||||
scheme: "https",
|
||||
endpoint: "https:///acmemattermost/",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "custom cloud rejects a non-HTTP scheme",
|
||||
cloud: model.AzureCloudCustom,
|
||||
scheme: "https",
|
||||
endpoint: "ftp://blob.example.com/",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "unknown cloud value is rejected",
|
||||
cloud: "azuregermany",
|
||||
scheme: "https",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
acct := account
|
||||
if tt.account != "" {
|
||||
acct = tt.account
|
||||
}
|
||||
got, err := buildAzureServiceURL(tt.cloud, tt.scheme, acct, tt.endpoint)
|
||||
if tt.wantErr {
|
||||
require.Error(t, err)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tt.expected, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAzureFileBackendPrefix(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
prefix string
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{name: "no prefix, plain path", prefix: "", input: "team/channel/file", expected: "team/channel/file"},
|
||||
{name: "no prefix, with dot-dot", prefix: "", input: "../escape", expected: "../escape"},
|
||||
{name: "prefix, plain path", prefix: "mattermost", input: "team/channel/file", expected: "mattermost/team/channel/file"},
|
||||
{name: "prefix, exact root", prefix: "mattermost", input: "", expected: "mattermost"},
|
||||
{name: "prefix, dot-dot escapes", prefix: "mattermost", input: "../escape", expected: "mattermost/escape"},
|
||||
{name: "prefix, nested dot-dot escapes", prefix: "mattermost", input: "sub/../../escape", expected: "mattermost/escape"},
|
||||
{name: "prefix, dot-dot in middle stays inside", prefix: "mattermost", input: "a/../b", expected: "mattermost/b"},
|
||||
{name: "prefix with trailing slash, dot-dot escapes", prefix: "mattermost/", input: "../escape", expected: "mattermost/escape"},
|
||||
{name: "prefix boundary collision must not escape", prefix: "mattermost", input: "../mattermost-evil/file", expected: "mattermost/file"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
b := &AzureFileBackend{pathPrefix: tt.prefix}
|
||||
require.Equal(t, tt.expected, b.prefix(tt.input))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// azuriteWellKnownAccount and azuriteWellKnownKey are Azurite's published
|
||||
// development credentials. They are not secrets - they are documented in the
|
||||
// Azurite README and ship hardcoded in every Azurite distribution.
|
||||
const (
|
||||
azuriteWellKnownAccount = "devstoreaccount1"
|
||||
azuriteWellKnownKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
|
||||
)
|
||||
|
||||
// TestAzureFileBackendAppendRefusesNonBlockBlob exercises the safety
|
||||
// check in AppendFile: when a blob exists with content but no committed
|
||||
// block list (i.e. it was uploaded via Put Blob by another tool), the
|
||||
// backend must refuse the append rather than silently destroy the
|
||||
// existing content.
|
||||
func TestAzureFileBackendAppendRefusesNonBlockBlob(t *testing.T) {
|
||||
be := newAzuriteBackend(t)
|
||||
|
||||
path := "append-refusal-test.bin"
|
||||
t.Cleanup(func() { _ = be.RemoveFile(path) })
|
||||
|
||||
// Write the blob via the high-level Upload helper, which calls the
|
||||
// Put Blob REST endpoint and leaves the committed-block list empty.
|
||||
original := []byte("planted-by-another-tool")
|
||||
bb := be.newBlockBlobClient(path)
|
||||
_, err := bb.Upload(context.Background(), nopReadSeekCloser{bytes.NewReader(original)}, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = be.AppendFile(bytes.NewReader([]byte("would-overwrite")), path)
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "no committed block list")
|
||||
|
||||
// The original content must still be intact.
|
||||
got, err := be.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, original, got)
|
||||
}
|
||||
|
||||
// TestAzureFileBackendMakeContainerIdempotent ensures that calling
|
||||
// MakeContainer twice on the same backend is a no-op the second time.
|
||||
// Two nodes can race through TestConnection plus MakeContainer at boot;
|
||||
// the loser must converge instead of returning an error.
|
||||
func TestAzureFileBackendMakeContainerIdempotent(t *testing.T) {
|
||||
be := newAzuriteBackend(t)
|
||||
|
||||
require.NoError(t, be.MakeContainer())
|
||||
require.NoError(t, be.MakeContainer())
|
||||
}
|
||||
|
||||
type nopReadSeekCloser struct {
|
||||
*bytes.Reader
|
||||
}
|
||||
|
||||
func (nopReadSeekCloser) Close() error { return nil }
|
||||
|
||||
// newAzuriteBackend builds an Azure backend pointed at the Azurite emulator
|
||||
// and ensures the container exists. Standalone Azure tests should use this
|
||||
// instead of calling NewAzureFileBackend + TestConnection directly; the
|
||||
// shared FileBackendTestSuite handles provisioning itself in SetupTest.
|
||||
func newAzuriteBackend(t *testing.T) *AzureFileBackend {
|
||||
t.Helper()
|
||||
be, err := NewAzureFileBackend(azuriteSettings(t))
|
||||
require.NoError(t, err)
|
||||
|
||||
var noBucket *FileBackendNoBucketError
|
||||
if err := be.TestConnection(); errors.As(err, &noBucket) {
|
||||
require.NoError(t, be.MakeContainer())
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
return be
|
||||
}
|
||||
|
||||
func azuriteSettings(t *testing.T) FileBackendSettings {
|
||||
t.Helper()
|
||||
host := os.Getenv("CI_AZURITE_HOST")
|
||||
if host == "" {
|
||||
host = "localhost"
|
||||
}
|
||||
port := os.Getenv("CI_AZURITE_PORT")
|
||||
if port == "" {
|
||||
port = "10000"
|
||||
}
|
||||
return FileBackendSettings{
|
||||
DriverName: driverAzure,
|
||||
AzureStorageAccount: azuriteWellKnownAccount,
|
||||
AzureAuthMode: model.AzureAuthModeSharedKey,
|
||||
AzureAccessKey: azuriteWellKnownKey,
|
||||
AzureContainer: "mattermost-test",
|
||||
AzureCloud: model.AzureCloudCustom,
|
||||
AzureEndpoint: "http://" + net.JoinHostPort(host, port) + "/" + azuriteWellKnownAccount + "/",
|
||||
// The emulator runs on an internal address, so its host must be allowed.
|
||||
AllowedUntrustedInternalConnections: host,
|
||||
AzureRequestTimeoutMilliseconds: 30000,
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewAzureFileBackendAuthMode(t *testing.T) {
|
||||
base := FileBackendSettings{
|
||||
DriverName: driverAzure,
|
||||
AzureStorageAccount: "anaccount",
|
||||
AzureContainer: "acontainer",
|
||||
AzureEndpoint: "localhost:10000",
|
||||
AzureSSL: false,
|
||||
AzureRequestTimeoutMilliseconds: 30000,
|
||||
}
|
||||
|
||||
t.Run("shared_key constructs a client", func(t *testing.T) {
|
||||
s := base
|
||||
s.AzureAuthMode = model.AzureAuthModeSharedKey
|
||||
s.AzureAccessKey = azuriteWellKnownKey
|
||||
|
||||
be, err := NewAzureFileBackend(s)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, be.client)
|
||||
})
|
||||
|
||||
t.Run("default_credential constructs a client without an access key", func(t *testing.T) {
|
||||
s := base
|
||||
s.AzureAuthMode = model.AzureAuthModeDefaultCredential
|
||||
// Intentionally no AzureAccessKey - default credential reads
|
||||
// identity from the host environment, not config.
|
||||
|
||||
be, err := NewAzureFileBackend(s)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, be.client)
|
||||
})
|
||||
|
||||
t.Run("empty AuthMode is rejected", func(t *testing.T) {
|
||||
s := base
|
||||
s.AzureAuthMode = ""
|
||||
s.AzureAccessKey = azuriteWellKnownKey
|
||||
|
||||
_, err := NewAzureFileBackend(s)
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "unknown azure auth mode")
|
||||
})
|
||||
|
||||
t.Run("unknown AuthMode is rejected", func(t *testing.T) {
|
||||
s := base
|
||||
s.AzureAuthMode = "oauth2"
|
||||
s.AzureAccessKey = azuriteWellKnownKey
|
||||
|
||||
_, err := NewAzureFileBackend(s)
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "unknown azure auth mode")
|
||||
})
|
||||
}
|
||||
|
||||
func TestCheckMandatoryAzureFieldsAuthMode(t *testing.T) {
|
||||
base := FileBackendSettings{
|
||||
AzureStorageAccount: "anaccount",
|
||||
AzureContainer: "acontainer",
|
||||
}
|
||||
|
||||
t.Run("shared_key requires access key", func(t *testing.T) {
|
||||
s := base
|
||||
s.AzureAuthMode = model.AzureAuthModeSharedKey
|
||||
s.AzureAccessKey = ""
|
||||
require.Error(t, s.CheckMandatoryAzureFields())
|
||||
|
||||
s.AzureAccessKey = "somekey"
|
||||
require.NoError(t, s.CheckMandatoryAzureFields())
|
||||
})
|
||||
|
||||
t.Run("default_credential does not require access key", func(t *testing.T) {
|
||||
s := base
|
||||
s.AzureAuthMode = model.AzureAuthModeDefaultCredential
|
||||
s.AzureAccessKey = ""
|
||||
require.NoError(t, s.CheckMandatoryAzureFields())
|
||||
})
|
||||
}
|
||||
|
||||
func TestAzureFileBackendTestSuite(t *testing.T) {
|
||||
suite.Run(t, &FileBackendTestSuite{settings: azuriteSettings(t)})
|
||||
}
|
||||
|
||||
// TestAzureGeneratePublicLink exercises the Service SAS path against Azurite.
|
||||
// The user-delegation SAS path requires Entra ID and cannot run in CI; it is
|
||||
// covered by the manual verification recipe in the PR description.
|
||||
func TestAzureGeneratePublicLink(t *testing.T) {
|
||||
t.Run("refuses to issue a link when presign expiration is unset", func(t *testing.T) {
|
||||
settings := azuriteSettings(t)
|
||||
// Deliberately leave AzurePresignExpiresSeconds at zero - the
|
||||
// shape an admin would hit if they enabled the export-direct-
|
||||
// download feature without ever configuring the field.
|
||||
be, err := NewAzureFileBackend(settings)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, _, err = be.GeneratePublicLink("any/path")
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "presign expiration is not configured")
|
||||
})
|
||||
|
||||
t.Run("shared-key SAS round-trips through Azurite", func(t *testing.T) {
|
||||
settings := azuriteSettings(t)
|
||||
settings.AzurePresignExpiresSeconds = 300
|
||||
be, err := NewAzureFileBackend(settings)
|
||||
require.NoError(t, err)
|
||||
|
||||
var noBucket *FileBackendNoBucketError
|
||||
if connErr := be.TestConnection(); errors.As(connErr, &noBucket) {
|
||||
require.NoError(t, be.MakeContainer())
|
||||
} else {
|
||||
require.NoError(t, connErr)
|
||||
}
|
||||
|
||||
path := "tests/presign-" + model.NewId() + ".bin"
|
||||
payload := bytes.Repeat([]byte("mattermost"), 4096) // 40 KiB
|
||||
_, writeErr := be.WriteFile(bytes.NewReader(payload), path)
|
||||
require.NoError(t, writeErr)
|
||||
t.Cleanup(func() { _ = be.RemoveFile(path) })
|
||||
|
||||
link, expiry, err := be.GeneratePublicLink(path)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, link)
|
||||
require.Equal(t, 300*time.Second, expiry)
|
||||
|
||||
// SAS query parameters Azurite expects to see.
|
||||
u, err := url.Parse(link)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, u.Query().Get("sig"))
|
||||
require.Equal(t, "r", u.Query().Get("sp"))
|
||||
require.Equal(t, "attachment", u.Query().Get("rscd"))
|
||||
|
||||
resp, err := http.Get(link)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = resp.Body.Close() })
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
require.Equal(t, "attachment", resp.Header.Get("Content-Disposition"))
|
||||
|
||||
got, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, payload, got)
|
||||
})
|
||||
|
||||
t.Run("tampered SAS is rejected by the storage service", func(t *testing.T) {
|
||||
settings := azuriteSettings(t)
|
||||
settings.AzurePresignExpiresSeconds = 300
|
||||
be, err := NewAzureFileBackend(settings)
|
||||
require.NoError(t, err)
|
||||
|
||||
var noBucket *FileBackendNoBucketError
|
||||
if connErr := be.TestConnection(); errors.As(connErr, &noBucket) {
|
||||
require.NoError(t, be.MakeContainer())
|
||||
} else {
|
||||
require.NoError(t, connErr)
|
||||
}
|
||||
|
||||
path := "tests/tamper-" + model.NewId() + ".bin"
|
||||
_, writeErr := be.WriteFile(bytes.NewReader([]byte("contents")), path)
|
||||
require.NoError(t, writeErr)
|
||||
t.Cleanup(func() { _ = be.RemoveFile(path) })
|
||||
|
||||
link, _, err := be.GeneratePublicLink(path)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Flip the last character of the signature so the link no longer
|
||||
// matches what Azurite signed.
|
||||
u, err := url.Parse(link)
|
||||
require.NoError(t, err)
|
||||
q := u.Query()
|
||||
sig := q.Get("sig")
|
||||
require.NotEmpty(t, sig)
|
||||
swapped := sig[:len(sig)-1] + flipBase64Char(sig[len(sig)-1])
|
||||
q.Set("sig", swapped)
|
||||
u.RawQuery = q.Encode()
|
||||
|
||||
resp, err := http.Get(u.String())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = resp.Body.Close() })
|
||||
require.Equal(t, http.StatusForbidden, resp.StatusCode)
|
||||
})
|
||||
|
||||
t.Run("unknown auth mode is rejected", func(t *testing.T) {
|
||||
// Build a backend by hand so we can exercise the default-branch
|
||||
// guard. Going through NewAzureFileBackend would be rejected by
|
||||
// CheckMandatoryAzureFields first.
|
||||
be := &AzureFileBackend{
|
||||
authMode: "totally-not-real",
|
||||
presignExpires: time.Minute,
|
||||
}
|
||||
_, _, err := be.GeneratePublicLink("any/path")
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "unknown azure auth mode")
|
||||
})
|
||||
}
|
||||
|
||||
// flipBase64Char swaps a base64 character for another in the same alphabet
|
||||
// to produce a still-valid-but-wrong signature character. Used only by the
|
||||
// tamper test above.
|
||||
func flipBase64Char(c byte) string {
|
||||
if c == 'A' {
|
||||
return "B"
|
||||
}
|
||||
return "A"
|
||||
}
|
||||
44
server/platform/shared/filestore/errors.go
Normal file
44
server/platform/shared/filestore/errors.go
Normal file
@@ -0,0 +1,44 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package filestore
|
||||
|
||||
// FileBackendAuthError is returned when testing a connection and authentication
|
||||
// against the file storage backend fails. Backends should wrap the underlying
|
||||
// auth failure in this type so the admin Test Connection flow can surface a
|
||||
// useful message regardless of which driver is configured.
|
||||
type FileBackendAuthError struct {
|
||||
// Err is the underlying driver error, if any.
|
||||
Err error
|
||||
// DetailedError is a human-readable message describing the failure.
|
||||
// Kept for compatibility with the previous S3-specific type.
|
||||
DetailedError string
|
||||
}
|
||||
|
||||
func (e *FileBackendAuthError) Error() string {
|
||||
if e.DetailedError != "" {
|
||||
return e.DetailedError
|
||||
}
|
||||
if e.Err != nil {
|
||||
return e.Err.Error()
|
||||
}
|
||||
return "authentication failed"
|
||||
}
|
||||
|
||||
func (e *FileBackendAuthError) Unwrap() error { return e.Err }
|
||||
|
||||
// FileBackendNoBucketError is returned when testing a connection and the
|
||||
// configured bucket / container does not exist.
|
||||
type FileBackendNoBucketError struct {
|
||||
// Err is the underlying driver error, if any.
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *FileBackendNoBucketError) Error() string {
|
||||
if e.Err != nil {
|
||||
return e.Err.Error()
|
||||
}
|
||||
return "no such bucket or container"
|
||||
}
|
||||
|
||||
func (e *FileBackendNoBucketError) Unwrap() error { return e.Err }
|
||||
250
server/platform/shared/filestore/filesstore.go
Normal file
250
server/platform/shared/filestore/filesstore.go
Normal file
@@ -0,0 +1,250 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package filestore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/mattermost/mattermost/server/public/model"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
driverS3 = "amazons3"
|
||||
driverLocal = "local"
|
||||
driverAzure = "azureblob"
|
||||
)
|
||||
|
||||
type ReadCloseSeeker interface {
|
||||
io.ReadCloser
|
||||
io.Seeker
|
||||
}
|
||||
|
||||
type FileBackend interface {
|
||||
DriverName() string
|
||||
TestConnection() error
|
||||
|
||||
Reader(path string) (ReadCloseSeeker, error)
|
||||
ReadFile(path string) ([]byte, error)
|
||||
FileExists(path string) (bool, error)
|
||||
FileSize(path string) (int64, error)
|
||||
CopyFile(oldPath, newPath string) error
|
||||
MoveFile(oldPath, newPath string) error
|
||||
WriteFile(fr io.Reader, path string) (int64, error)
|
||||
AppendFile(fr io.Reader, path string) (int64, error)
|
||||
RemoveFile(path string) error
|
||||
FileModTime(path string) (time.Time, error)
|
||||
|
||||
ListDirectory(path string) ([]string, error)
|
||||
ListDirectoryRecursively(path string) ([]string, error)
|
||||
RemoveDirectory(path string) error
|
||||
ZipReader(path string, deflate bool) (io.ReadCloser, error)
|
||||
}
|
||||
|
||||
type FileBackendWithLinkGenerator interface {
|
||||
GeneratePublicLink(path string) (string, time.Duration, error)
|
||||
}
|
||||
|
||||
type FileBackendSettings struct {
|
||||
DriverName string
|
||||
Directory string
|
||||
AmazonS3AccessKeyId string
|
||||
AmazonS3SecretAccessKey string
|
||||
AmazonS3Bucket string
|
||||
AmazonS3PathPrefix string
|
||||
AmazonS3Region string
|
||||
AmazonS3Endpoint string
|
||||
AmazonS3SSL bool
|
||||
AmazonS3SignV2 bool
|
||||
AmazonS3SSE bool
|
||||
AmazonS3Trace bool
|
||||
SkipVerify bool
|
||||
AmazonS3RequestTimeoutMilliseconds int64
|
||||
AmazonS3PresignExpiresSeconds int64
|
||||
AmazonS3UploadPartSizeBytes int64
|
||||
AmazonS3StorageClass string
|
||||
AzureStorageAccount string
|
||||
AzureAuthMode string
|
||||
AzureAccessKey string
|
||||
AzureContainer string
|
||||
AzurePathPrefix string
|
||||
AzureCloud string
|
||||
AzureEndpoint string
|
||||
AzureSSL bool
|
||||
AzureRequestTimeoutMilliseconds int64
|
||||
AzurePresignExpiresSeconds int64
|
||||
// AllowedUntrustedInternalConnections carries the value of the
|
||||
// ServiceSettings.AllowedUntrustedInternalConnections config setting, used
|
||||
// by the Azure custom-cloud backend.
|
||||
AllowedUntrustedInternalConnections string
|
||||
}
|
||||
|
||||
func NewFileBackendSettingsFromConfig(fileSettings *model.FileSettings, enableComplianceFeature bool, skipVerify bool, allowedUntrustedInternalConnections string) FileBackendSettings {
|
||||
if *fileSettings.DriverName == model.ImageDriverLocal {
|
||||
return FileBackendSettings{
|
||||
DriverName: *fileSettings.DriverName,
|
||||
Directory: *fileSettings.Directory,
|
||||
}
|
||||
}
|
||||
if *fileSettings.DriverName == model.ImageDriverAzure {
|
||||
return FileBackendSettings{
|
||||
DriverName: *fileSettings.DriverName,
|
||||
AzureStorageAccount: *fileSettings.AzureStorageAccount,
|
||||
AzureAuthMode: *fileSettings.AzureAuthMode,
|
||||
AzureAccessKey: *fileSettings.AzureAccessKey,
|
||||
AzureContainer: *fileSettings.AzureContainer,
|
||||
AzurePathPrefix: *fileSettings.AzurePathPrefix,
|
||||
AzureCloud: *fileSettings.AzureCloud,
|
||||
AzureEndpoint: *fileSettings.AzureEndpoint,
|
||||
AzureSSL: fileSettings.AzureSSL == nil || *fileSettings.AzureSSL,
|
||||
AzureRequestTimeoutMilliseconds: *fileSettings.AzureRequestTimeoutMilliseconds,
|
||||
SkipVerify: skipVerify,
|
||||
AllowedUntrustedInternalConnections: allowedUntrustedInternalConnections,
|
||||
}
|
||||
}
|
||||
return FileBackendSettings{
|
||||
DriverName: *fileSettings.DriverName,
|
||||
AmazonS3AccessKeyId: *fileSettings.AmazonS3AccessKeyId,
|
||||
AmazonS3SecretAccessKey: *fileSettings.AmazonS3SecretAccessKey,
|
||||
AmazonS3Bucket: *fileSettings.AmazonS3Bucket,
|
||||
AmazonS3PathPrefix: *fileSettings.AmazonS3PathPrefix,
|
||||
AmazonS3Region: *fileSettings.AmazonS3Region,
|
||||
AmazonS3Endpoint: *fileSettings.AmazonS3Endpoint,
|
||||
AmazonS3SSL: fileSettings.AmazonS3SSL == nil || *fileSettings.AmazonS3SSL,
|
||||
AmazonS3SignV2: fileSettings.AmazonS3SignV2 != nil && *fileSettings.AmazonS3SignV2,
|
||||
AmazonS3SSE: fileSettings.AmazonS3SSE != nil && *fileSettings.AmazonS3SSE && enableComplianceFeature,
|
||||
AmazonS3Trace: fileSettings.AmazonS3Trace != nil && *fileSettings.AmazonS3Trace,
|
||||
AmazonS3RequestTimeoutMilliseconds: *fileSettings.AmazonS3RequestTimeoutMilliseconds,
|
||||
SkipVerify: skipVerify,
|
||||
AmazonS3UploadPartSizeBytes: *fileSettings.AmazonS3UploadPartSizeBytes,
|
||||
AmazonS3StorageClass: *fileSettings.AmazonS3StorageClass,
|
||||
}
|
||||
}
|
||||
|
||||
func NewExportFileBackendSettingsFromConfig(fileSettings *model.FileSettings, enableComplianceFeature bool, skipVerify bool, allowedUntrustedInternalConnections string) FileBackendSettings {
|
||||
if *fileSettings.ExportDriverName == model.ImageDriverLocal {
|
||||
return FileBackendSettings{
|
||||
DriverName: *fileSettings.ExportDriverName,
|
||||
Directory: *fileSettings.ExportDirectory,
|
||||
}
|
||||
}
|
||||
if *fileSettings.ExportDriverName == model.ImageDriverAzure {
|
||||
return FileBackendSettings{
|
||||
DriverName: *fileSettings.ExportDriverName,
|
||||
AzureStorageAccount: *fileSettings.ExportAzureStorageAccount,
|
||||
AzureAuthMode: *fileSettings.ExportAzureAuthMode,
|
||||
AzureAccessKey: *fileSettings.ExportAzureAccessKey,
|
||||
AzureContainer: *fileSettings.ExportAzureContainer,
|
||||
AzurePathPrefix: *fileSettings.ExportAzurePathPrefix,
|
||||
AzureCloud: *fileSettings.ExportAzureCloud,
|
||||
AzureEndpoint: *fileSettings.ExportAzureEndpoint,
|
||||
AzureSSL: fileSettings.ExportAzureSSL == nil || *fileSettings.ExportAzureSSL,
|
||||
AzureRequestTimeoutMilliseconds: *fileSettings.ExportAzureRequestTimeoutMilliseconds,
|
||||
AzurePresignExpiresSeconds: *fileSettings.ExportAzurePresignExpiresSeconds,
|
||||
SkipVerify: skipVerify,
|
||||
AllowedUntrustedInternalConnections: allowedUntrustedInternalConnections,
|
||||
}
|
||||
}
|
||||
return FileBackendSettings{
|
||||
DriverName: *fileSettings.ExportDriverName,
|
||||
AmazonS3AccessKeyId: *fileSettings.ExportAmazonS3AccessKeyId,
|
||||
AmazonS3SecretAccessKey: *fileSettings.ExportAmazonS3SecretAccessKey,
|
||||
AmazonS3Bucket: *fileSettings.ExportAmazonS3Bucket,
|
||||
AmazonS3PathPrefix: *fileSettings.ExportAmazonS3PathPrefix,
|
||||
AmazonS3Region: *fileSettings.ExportAmazonS3Region,
|
||||
AmazonS3Endpoint: *fileSettings.ExportAmazonS3Endpoint,
|
||||
AmazonS3SSL: fileSettings.ExportAmazonS3SSL == nil || *fileSettings.ExportAmazonS3SSL,
|
||||
AmazonS3SignV2: fileSettings.ExportAmazonS3SignV2 != nil && *fileSettings.ExportAmazonS3SignV2,
|
||||
AmazonS3SSE: fileSettings.ExportAmazonS3SSE != nil && *fileSettings.ExportAmazonS3SSE && enableComplianceFeature,
|
||||
AmazonS3Trace: fileSettings.ExportAmazonS3Trace != nil && *fileSettings.ExportAmazonS3Trace,
|
||||
AmazonS3RequestTimeoutMilliseconds: *fileSettings.ExportAmazonS3RequestTimeoutMilliseconds,
|
||||
AmazonS3PresignExpiresSeconds: *fileSettings.ExportAmazonS3PresignExpiresSeconds,
|
||||
AmazonS3UploadPartSizeBytes: *fileSettings.ExportAmazonS3UploadPartSizeBytes,
|
||||
AmazonS3StorageClass: *fileSettings.ExportAmazonS3StorageClass,
|
||||
SkipVerify: skipVerify,
|
||||
}
|
||||
}
|
||||
|
||||
func (settings *FileBackendSettings) CheckMandatoryS3Fields() error {
|
||||
if settings.AmazonS3Bucket == "" {
|
||||
return errors.New("missing s3 bucket settings")
|
||||
}
|
||||
|
||||
// if S3 endpoint is not set call the set defaults to set that
|
||||
if settings.AmazonS3Endpoint == "" {
|
||||
settings.AmazonS3Endpoint = "s3.amazonaws.com"
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (settings *FileBackendSettings) CheckMandatoryAzureFields() error {
|
||||
if settings.AzureStorageAccount == "" {
|
||||
return errors.New("missing azure storage account setting")
|
||||
}
|
||||
if settings.AzureContainer == "" {
|
||||
return errors.New("missing azure container setting")
|
||||
}
|
||||
// AzureAccessKey is only meaningful for shared-key auth. Default credential
|
||||
// reads identity from the host environment (managed identity / workload
|
||||
// identity / service principal env vars / az login), so an empty access key
|
||||
// is the expected configuration in that mode.
|
||||
if settings.AzureAuthMode == model.AzureAuthModeSharedKey && settings.AzureAccessKey == "" {
|
||||
return errors.New("missing azure access key setting")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewFileBackend creates a new file backend
|
||||
func NewFileBackend(settings FileBackendSettings) (FileBackend, error) {
|
||||
return newFileBackend(settings, true)
|
||||
}
|
||||
|
||||
// NewExportFileBackend creates a new file backend for exports, that will not attempt to use bifrost.
|
||||
func NewExportFileBackend(settings FileBackendSettings) (FileBackend, error) {
|
||||
return newFileBackend(settings, false)
|
||||
}
|
||||
|
||||
func newFileBackend(settings FileBackendSettings, canBeCloud bool) (FileBackend, error) {
|
||||
switch settings.DriverName {
|
||||
case driverS3:
|
||||
newBackendFn := NewS3FileBackend
|
||||
if !canBeCloud {
|
||||
newBackendFn = NewS3FileBackendWithoutBifrost
|
||||
}
|
||||
backend, err := newBackendFn(settings)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "unable to connect to the s3 backend")
|
||||
}
|
||||
return backend, nil
|
||||
case driverLocal:
|
||||
return &LocalFileBackend{
|
||||
directory: settings.Directory,
|
||||
}, nil
|
||||
case driverAzure:
|
||||
backend, err := NewAzureFileBackend(settings)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "unable to connect to the azure backend")
|
||||
}
|
||||
return backend, nil
|
||||
}
|
||||
return nil, errors.New("no valid filestorage driver found")
|
||||
}
|
||||
|
||||
// TryWriteFileContext checks if the file backend supports context writes and passes the context in that case.
|
||||
// Should the file backend not support contexts, it just calls WriteFile instead. This can be used to disable
|
||||
// the timeouts for long writes (like exports).
|
||||
func TryWriteFileContext(ctx context.Context, fb FileBackend, fr io.Reader, path string) (int64, error) {
|
||||
type ContextWriter interface {
|
||||
WriteFileContext(context.Context, io.Reader, string) (int64, error)
|
||||
}
|
||||
|
||||
if cw, ok := fb.(ContextWriter); ok {
|
||||
return cw.WriteFileContext(ctx, fr, path)
|
||||
}
|
||||
|
||||
return fb.WriteFile(fr, path)
|
||||
}
|
||||
1329
server/platform/shared/filestore/filesstore_test.go
Normal file
1329
server/platform/shared/filestore/filesstore_test.go
Normal file
File diff suppressed because it is too large
Load Diff
347
server/platform/shared/filestore/localstore.go
Normal file
347
server/platform/shared/filestore/localstore.go
Normal file
@@ -0,0 +1,347 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package filestore
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/mattermost/mattermost/server/public/shared/mlog"
|
||||
)
|
||||
|
||||
const (
|
||||
TestFilePath = "/testfile"
|
||||
MaxRecursionDepth = 50
|
||||
)
|
||||
|
||||
type LocalFileBackend struct {
|
||||
directory string
|
||||
}
|
||||
|
||||
// copyFile will copy a file from src path to dst path.
|
||||
// Overwrites any existing files at dst.
|
||||
// Permissions are copied from file at src to the new file at dst.
|
||||
func copyFile(src, dst string) (err error) {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer in.Close()
|
||||
|
||||
if err = os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil {
|
||||
return
|
||||
}
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if e := out.Close(); e != nil {
|
||||
err = e
|
||||
}
|
||||
}()
|
||||
|
||||
_, err = io.Copy(out, in)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
err = out.Sync()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
stat, err := os.Stat(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = os.Chmod(dst, stat.Mode())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) DriverName() string {
|
||||
return driverLocal
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) TestConnection() error {
|
||||
f := bytes.NewReader([]byte("testingwrite"))
|
||||
if _, err := writeFileLocally(f, filepath.Join(b.directory, TestFilePath)); err != nil {
|
||||
return errors.Wrap(err, "unable to write to the local filesystem storage")
|
||||
}
|
||||
os.Remove(filepath.Join(b.directory, TestFilePath))
|
||||
mlog.Debug("Able to write files to local storage.")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) Reader(path string) (ReadCloseSeeker, error) {
|
||||
f, err := os.Open(filepath.Join(b.directory, path))
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "unable to open file %s", path)
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) ReadFile(path string) ([]byte, error) {
|
||||
f, err := os.ReadFile(filepath.Join(b.directory, path))
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "unable to read file %s", path)
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) FileExists(path string) (bool, error) {
|
||||
_, err := os.Stat(filepath.Join(b.directory, path))
|
||||
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return false, errors.Wrapf(err, "unable to know if file %s exists", path)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) FileSize(path string) (int64, error) {
|
||||
info, err := os.Stat(filepath.Join(b.directory, path))
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "unable to get file size for %s", path)
|
||||
}
|
||||
return info.Size(), nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) FileModTime(path string) (time.Time, error) {
|
||||
info, err := os.Stat(filepath.Join(b.directory, path))
|
||||
if err != nil {
|
||||
return time.Time{}, errors.Wrapf(err, "unable to get modification time for file %s", path)
|
||||
}
|
||||
return info.ModTime(), nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) CopyFile(oldPath, newPath string) error {
|
||||
if err := copyFile(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil {
|
||||
return errors.Wrapf(err, "unable to copy file from %s to %s", oldPath, newPath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) MoveFile(oldPath, newPath string) error {
|
||||
if err := os.MkdirAll(filepath.Dir(filepath.Join(b.directory, newPath)), 0750); err != nil {
|
||||
return errors.Wrapf(err, "unable to create the new destination directory %s", filepath.Dir(newPath))
|
||||
}
|
||||
|
||||
if err := os.Rename(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil {
|
||||
return errors.Wrapf(err, "unable to move the file to %s to the destination directory", newPath)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) WriteFile(fr io.Reader, path string) (int64, error) {
|
||||
return writeFileLocally(fr, filepath.Join(b.directory, path))
|
||||
}
|
||||
|
||||
func writeFileLocally(fr io.Reader, path string) (int64, error) {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil {
|
||||
directory, _ := filepath.Abs(filepath.Dir(path))
|
||||
return 0, errors.Wrapf(err, "unable to create the directory %s for the file %s", directory, path)
|
||||
}
|
||||
fw, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "unable to open the file %s to write the data", path)
|
||||
}
|
||||
defer fw.Close()
|
||||
written, err := io.Copy(fw, fr)
|
||||
if err != nil {
|
||||
return written, errors.Wrapf(err, "unable write the data in the file %s", path)
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) AppendFile(fr io.Reader, path string) (int64, error) {
|
||||
fp := filepath.Join(b.directory, path)
|
||||
if _, err := os.Stat(fp); err != nil {
|
||||
return 0, errors.Wrapf(err, "unable to find the file %s to append the data", path)
|
||||
}
|
||||
fw, err := os.OpenFile(fp, os.O_WRONLY|os.O_APPEND, 0600)
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "unable to open the file %s to append the data", path)
|
||||
}
|
||||
defer fw.Close()
|
||||
written, err := io.Copy(fw, fr)
|
||||
if err != nil {
|
||||
return written, errors.Wrapf(err, "unable append the data in the file %s", path)
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) RemoveFile(path string) error {
|
||||
if err := os.Remove(filepath.Join(b.directory, path)); err != nil {
|
||||
return errors.Wrapf(err, "unable to remove the file %s", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// basePath: path to get to the file but won't be added to the end result
|
||||
// path: basePath+path current directory we are looking at
|
||||
// maxDepth: parameter to prevent infinite recursion, once this is reached we won't look any further
|
||||
func appendRecursively(basePath, path string, maxDepth int) ([]string, error) {
|
||||
results := []string{}
|
||||
dirEntries, err := os.ReadDir(filepath.Join(basePath, path))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return results, nil
|
||||
}
|
||||
return results, errors.Wrapf(err, "unable to list the directory %s", path)
|
||||
}
|
||||
for _, dirEntry := range dirEntries {
|
||||
entryName := dirEntry.Name()
|
||||
entryPath := filepath.Join(path, entryName)
|
||||
if entryName == "." || entryName == ".." || entryPath == path {
|
||||
continue
|
||||
}
|
||||
if dirEntry.IsDir() {
|
||||
if maxDepth <= 0 {
|
||||
mlog.Warn("Max depth reached, skipping any further directories", mlog.Int("depth", maxDepth), mlog.String("path", entryPath))
|
||||
results = append(results, entryPath)
|
||||
continue // we'll ignore it if max depth is reached.
|
||||
}
|
||||
nestedResults, err := appendRecursively(basePath, entryPath, maxDepth-1)
|
||||
if err != nil {
|
||||
return results, err
|
||||
}
|
||||
results = append(results, nestedResults...)
|
||||
} else {
|
||||
results = append(results, entryPath)
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) ListDirectory(path string) ([]string, error) {
|
||||
results := []string{}
|
||||
dirEntries, err := os.ReadDir(filepath.Join(b.directory, path))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
// ideally os.ErrNotExist should've been returned but to keep the
|
||||
// consistency, leaving it as is before.
|
||||
return results, nil
|
||||
}
|
||||
// same here, ideally we shouldn't return the empty slice
|
||||
return results, errors.Wrapf(err, "unable to list the directory %s", path)
|
||||
}
|
||||
for _, dirEntry := range dirEntries {
|
||||
results = append(results, filepath.Join(path, dirEntry.Name()))
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) ListDirectoryRecursively(path string) ([]string, error) {
|
||||
return appendRecursively(b.directory, path, MaxRecursionDepth)
|
||||
}
|
||||
|
||||
func (b *LocalFileBackend) RemoveDirectory(path string) error {
|
||||
if err := os.RemoveAll(filepath.Join(b.directory, path)); err != nil {
|
||||
return errors.Wrapf(err, "unable to remove the directory %s", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ZipReader will create a zip of path. If path is a single file, it will zip the single file.
|
||||
// If deflate is true, the contents will be compressed. It will stream the zip to io.ReadCloser.
|
||||
func (b *LocalFileBackend) ZipReader(path string, deflate bool) (io.ReadCloser, error) {
|
||||
deflateMethod := zip.Store
|
||||
if deflate {
|
||||
deflateMethod = zip.Deflate
|
||||
}
|
||||
|
||||
fullPath := filepath.Join(b.directory, path)
|
||||
baseInfo, err := os.Stat(fullPath)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "unable to stat path %s", path)
|
||||
}
|
||||
|
||||
pr, pw := io.Pipe()
|
||||
|
||||
go func() {
|
||||
defer pw.Close()
|
||||
|
||||
zipWriter := zip.NewWriter(pw)
|
||||
defer zipWriter.Close()
|
||||
|
||||
err = filepath.Walk(fullPath, func(filePath string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Handle single file case
|
||||
baseDir := fullPath
|
||||
if !baseInfo.IsDir() {
|
||||
baseDir = filepath.Dir(baseDir)
|
||||
}
|
||||
|
||||
// Get the relative path from the base directory
|
||||
relPath, err := filepath.Rel(baseDir, filePath)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "unable to get relative path for %s", filePath)
|
||||
}
|
||||
|
||||
// Skip the root directory itself
|
||||
if relPath == "." {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create zip header
|
||||
header, err := zip.FileInfoHeader(info)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "unable to create zip header for %s", relPath)
|
||||
}
|
||||
|
||||
// Ensure consistent forward slashes in paths
|
||||
header.Name = filepath.ToSlash(relPath)
|
||||
|
||||
// Skip directories - we don't need to create entries for them
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create file entry
|
||||
header.Method = deflateMethod
|
||||
header.SetMode(0644) // rw-r--r-- permissions
|
||||
writer, err := zipWriter.CreateHeader(header)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "unable to create zip entry for %s", relPath)
|
||||
}
|
||||
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "unable to open file %s", filePath)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
if _, err := io.Copy(writer, file); err != nil {
|
||||
return errors.Wrapf(err, "unable to copy file content for %s", relPath)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
pw.CloseWithError(errors.Wrap(err, "error walking directory"))
|
||||
}
|
||||
}()
|
||||
|
||||
return pr, nil
|
||||
}
|
||||
430
server/platform/shared/filestore/mocks/FileBackend.go
Normal file
430
server/platform/shared/filestore/mocks/FileBackend.go
Normal file
@@ -0,0 +1,430 @@
|
||||
// Code generated by mockery v2.53.4. DO NOT EDIT.
|
||||
|
||||
// Regenerate this file using `make filestore-mocks`.
|
||||
|
||||
package mocks
|
||||
|
||||
import (
|
||||
io "io"
|
||||
time "time"
|
||||
|
||||
filestore "github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// FileBackend is an autogenerated mock type for the FileBackend type
|
||||
type FileBackend struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
// AppendFile provides a mock function with given fields: fr, path
|
||||
func (_m *FileBackend) AppendFile(fr io.Reader, path string) (int64, error) {
|
||||
ret := _m.Called(fr, path)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for AppendFile")
|
||||
}
|
||||
|
||||
var r0 int64
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(io.Reader, string) (int64, error)); ok {
|
||||
return rf(fr, path)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(io.Reader, string) int64); ok {
|
||||
r0 = rf(fr, path)
|
||||
} else {
|
||||
r0 = ret.Get(0).(int64)
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(io.Reader, string) error); ok {
|
||||
r1 = rf(fr, path)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// CopyFile provides a mock function with given fields: oldPath, newPath
|
||||
func (_m *FileBackend) CopyFile(oldPath string, newPath string) error {
|
||||
ret := _m.Called(oldPath, newPath)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for CopyFile")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(string, string) error); ok {
|
||||
r0 = rf(oldPath, newPath)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// DriverName provides a mock function with no fields
|
||||
func (_m *FileBackend) DriverName() string {
|
||||
ret := _m.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for DriverName")
|
||||
}
|
||||
|
||||
var r0 string
|
||||
if rf, ok := ret.Get(0).(func() string); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
r0 = ret.Get(0).(string)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// FileExists provides a mock function with given fields: path
|
||||
func (_m *FileBackend) FileExists(path string) (bool, error) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for FileExists")
|
||||
}
|
||||
|
||||
var r0 bool
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(string) (bool, error)); ok {
|
||||
return rf(path)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(string) bool); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
r0 = ret.Get(0).(bool)
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(string) error); ok {
|
||||
r1 = rf(path)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// FileModTime provides a mock function with given fields: path
|
||||
func (_m *FileBackend) FileModTime(path string) (time.Time, error) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for FileModTime")
|
||||
}
|
||||
|
||||
var r0 time.Time
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(string) (time.Time, error)); ok {
|
||||
return rf(path)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(string) time.Time); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
r0 = ret.Get(0).(time.Time)
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(string) error); ok {
|
||||
r1 = rf(path)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// FileSize provides a mock function with given fields: path
|
||||
func (_m *FileBackend) FileSize(path string) (int64, error) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for FileSize")
|
||||
}
|
||||
|
||||
var r0 int64
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(string) (int64, error)); ok {
|
||||
return rf(path)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(string) int64); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
r0 = ret.Get(0).(int64)
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(string) error); ok {
|
||||
r1 = rf(path)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// ListDirectory provides a mock function with given fields: path
|
||||
func (_m *FileBackend) ListDirectory(path string) ([]string, error) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for ListDirectory")
|
||||
}
|
||||
|
||||
var r0 []string
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(string) ([]string, error)); ok {
|
||||
return rf(path)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(string) []string); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).([]string)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(string) error); ok {
|
||||
r1 = rf(path)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// ListDirectoryRecursively provides a mock function with given fields: path
|
||||
func (_m *FileBackend) ListDirectoryRecursively(path string) ([]string, error) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for ListDirectoryRecursively")
|
||||
}
|
||||
|
||||
var r0 []string
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(string) ([]string, error)); ok {
|
||||
return rf(path)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(string) []string); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).([]string)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(string) error); ok {
|
||||
r1 = rf(path)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MoveFile provides a mock function with given fields: oldPath, newPath
|
||||
func (_m *FileBackend) MoveFile(oldPath string, newPath string) error {
|
||||
ret := _m.Called(oldPath, newPath)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for MoveFile")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(string, string) error); ok {
|
||||
r0 = rf(oldPath, newPath)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// ReadFile provides a mock function with given fields: path
|
||||
func (_m *FileBackend) ReadFile(path string) ([]byte, error) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for ReadFile")
|
||||
}
|
||||
|
||||
var r0 []byte
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(string) ([]byte, error)); ok {
|
||||
return rf(path)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(string) []byte); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).([]byte)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(string) error); ok {
|
||||
r1 = rf(path)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// Reader provides a mock function with given fields: path
|
||||
func (_m *FileBackend) Reader(path string) (filestore.ReadCloseSeeker, error) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Reader")
|
||||
}
|
||||
|
||||
var r0 filestore.ReadCloseSeeker
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(string) (filestore.ReadCloseSeeker, error)); ok {
|
||||
return rf(path)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(string) filestore.ReadCloseSeeker); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(filestore.ReadCloseSeeker)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(string) error); ok {
|
||||
r1 = rf(path)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// RemoveDirectory provides a mock function with given fields: path
|
||||
func (_m *FileBackend) RemoveDirectory(path string) error {
|
||||
ret := _m.Called(path)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for RemoveDirectory")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(string) error); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// RemoveFile provides a mock function with given fields: path
|
||||
func (_m *FileBackend) RemoveFile(path string) error {
|
||||
ret := _m.Called(path)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for RemoveFile")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func(string) error); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// TestConnection provides a mock function with no fields
|
||||
func (_m *FileBackend) TestConnection() error {
|
||||
ret := _m.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for TestConnection")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func() error); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// WriteFile provides a mock function with given fields: fr, path
|
||||
func (_m *FileBackend) WriteFile(fr io.Reader, path string) (int64, error) {
|
||||
ret := _m.Called(fr, path)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for WriteFile")
|
||||
}
|
||||
|
||||
var r0 int64
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(io.Reader, string) (int64, error)); ok {
|
||||
return rf(fr, path)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(io.Reader, string) int64); ok {
|
||||
r0 = rf(fr, path)
|
||||
} else {
|
||||
r0 = ret.Get(0).(int64)
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(io.Reader, string) error); ok {
|
||||
r1 = rf(fr, path)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// ZipReader provides a mock function with given fields: path, deflate
|
||||
func (_m *FileBackend) ZipReader(path string, deflate bool) (io.ReadCloser, error) {
|
||||
ret := _m.Called(path, deflate)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for ZipReader")
|
||||
}
|
||||
|
||||
var r0 io.ReadCloser
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(string, bool) (io.ReadCloser, error)); ok {
|
||||
return rf(path, deflate)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(string, bool) io.ReadCloser); ok {
|
||||
r0 = rf(path, deflate)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(io.ReadCloser)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(string, bool) error); ok {
|
||||
r1 = rf(path, deflate)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// NewFileBackend creates a new instance of FileBackend. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func NewFileBackend(t interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}) *FileBackend {
|
||||
mock := &FileBackend{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
// Code generated by mockery v2.53.4. DO NOT EDIT.
|
||||
|
||||
// Regenerate this file using `make filestore-mocks`.
|
||||
|
||||
package mocks
|
||||
|
||||
import (
|
||||
time "time"
|
||||
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// FileBackendWithLinkGenerator is an autogenerated mock type for the FileBackendWithLinkGenerator type
|
||||
type FileBackendWithLinkGenerator struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
// GeneratePublicLink provides a mock function with given fields: path
|
||||
func (_m *FileBackendWithLinkGenerator) GeneratePublicLink(path string) (string, time.Duration, error) {
|
||||
ret := _m.Called(path)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for GeneratePublicLink")
|
||||
}
|
||||
|
||||
var r0 string
|
||||
var r1 time.Duration
|
||||
var r2 error
|
||||
if rf, ok := ret.Get(0).(func(string) (string, time.Duration, error)); ok {
|
||||
return rf(path)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(string) string); ok {
|
||||
r0 = rf(path)
|
||||
} else {
|
||||
r0 = ret.Get(0).(string)
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(string) time.Duration); ok {
|
||||
r1 = rf(path)
|
||||
} else {
|
||||
r1 = ret.Get(1).(time.Duration)
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(2).(func(string) error); ok {
|
||||
r2 = rf(path)
|
||||
} else {
|
||||
r2 = ret.Error(2)
|
||||
}
|
||||
|
||||
return r0, r1, r2
|
||||
}
|
||||
|
||||
// NewFileBackendWithLinkGenerator creates a new instance of FileBackendWithLinkGenerator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func NewFileBackendWithLinkGenerator(t interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}) *FileBackendWithLinkGenerator {
|
||||
mock := &FileBackendWithLinkGenerator{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
||||
100
server/platform/shared/filestore/mocks/ReadCloseSeeker.go
Normal file
100
server/platform/shared/filestore/mocks/ReadCloseSeeker.go
Normal file
@@ -0,0 +1,100 @@
|
||||
// Code generated by mockery v2.53.4. DO NOT EDIT.
|
||||
|
||||
// Regenerate this file using `make filestore-mocks`.
|
||||
|
||||
package mocks
|
||||
|
||||
import mock "github.com/stretchr/testify/mock"
|
||||
|
||||
// ReadCloseSeeker is an autogenerated mock type for the ReadCloseSeeker type
|
||||
type ReadCloseSeeker struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
// Close provides a mock function with no fields
|
||||
func (_m *ReadCloseSeeker) Close() error {
|
||||
ret := _m.Called()
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Close")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func() error); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// Read provides a mock function with given fields: p
|
||||
func (_m *ReadCloseSeeker) Read(p []byte) (int, error) {
|
||||
ret := _m.Called(p)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Read")
|
||||
}
|
||||
|
||||
var r0 int
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func([]byte) (int, error)); ok {
|
||||
return rf(p)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func([]byte) int); ok {
|
||||
r0 = rf(p)
|
||||
} else {
|
||||
r0 = ret.Get(0).(int)
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func([]byte) error); ok {
|
||||
r1 = rf(p)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// Seek provides a mock function with given fields: offset, whence
|
||||
func (_m *ReadCloseSeeker) Seek(offset int64, whence int) (int64, error) {
|
||||
ret := _m.Called(offset, whence)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Seek")
|
||||
}
|
||||
|
||||
var r0 int64
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(int64, int) (int64, error)); ok {
|
||||
return rf(offset, whence)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(int64, int) int64); ok {
|
||||
r0 = rf(offset, whence)
|
||||
} else {
|
||||
r0 = ret.Get(0).(int64)
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(int64, int) error); ok {
|
||||
r1 = rf(offset, whence)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// NewReadCloseSeeker creates a new instance of ReadCloseSeeker. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func NewReadCloseSeeker(t interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}) *ReadCloseSeeker {
|
||||
mock := &ReadCloseSeeker{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
||||
60
server/platform/shared/filestore/mocks/blobDownloader.go
Normal file
60
server/platform/shared/filestore/mocks/blobDownloader.go
Normal file
@@ -0,0 +1,60 @@
|
||||
// Code generated by mockery v2.53.4. DO NOT EDIT.
|
||||
|
||||
// Regenerate this file using `make filestore-mocks`.
|
||||
|
||||
package mocks
|
||||
|
||||
import (
|
||||
context "context"
|
||||
|
||||
blob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
||||
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// blobDownloader is an autogenerated mock type for the blobDownloader type
|
||||
type blobDownloader struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
// DownloadStream provides a mock function with given fields: ctx, opts
|
||||
func (_m *blobDownloader) DownloadStream(ctx context.Context, opts *blob.DownloadStreamOptions) (blob.DownloadStreamResponse, error) {
|
||||
ret := _m.Called(ctx, opts)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for DownloadStream")
|
||||
}
|
||||
|
||||
var r0 blob.DownloadStreamResponse
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *blob.DownloadStreamOptions) (blob.DownloadStreamResponse, error)); ok {
|
||||
return rf(ctx, opts)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *blob.DownloadStreamOptions) blob.DownloadStreamResponse); ok {
|
||||
r0 = rf(ctx, opts)
|
||||
} else {
|
||||
r0 = ret.Get(0).(blob.DownloadStreamResponse)
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *blob.DownloadStreamOptions) error); ok {
|
||||
r1 = rf(ctx, opts)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// newBlobDownloader creates a new instance of blobDownloader. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func newBlobDownloader(t interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}) *blobDownloader {
|
||||
mock := &blobDownloader{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
||||
122
server/platform/shared/filestore/mocks/s3CopyClient.go
Normal file
122
server/platform/shared/filestore/mocks/s3CopyClient.go
Normal file
@@ -0,0 +1,122 @@
|
||||
// Code generated by mockery v2.53.4. DO NOT EDIT.
|
||||
|
||||
// Regenerate this file using `make filestore-mocks`.
|
||||
|
||||
package mocks
|
||||
|
||||
import (
|
||||
context "context"
|
||||
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// s3CopyClient is an autogenerated mock type for the s3CopyClient type
|
||||
type s3CopyClient struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
// ComposeObject provides a mock function with given fields: ctx, dst, srcs
|
||||
func (_m *s3CopyClient) ComposeObject(ctx context.Context, dst minio.CopyDestOptions, srcs ...minio.CopySrcOptions) (minio.UploadInfo, error) {
|
||||
_va := make([]interface{}, len(srcs))
|
||||
for _i := range srcs {
|
||||
_va[_i] = srcs[_i]
|
||||
}
|
||||
var _ca []interface{}
|
||||
_ca = append(_ca, ctx, dst)
|
||||
_ca = append(_ca, _va...)
|
||||
ret := _m.Called(_ca...)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for ComposeObject")
|
||||
}
|
||||
|
||||
var r0 minio.UploadInfo
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, minio.CopyDestOptions, ...minio.CopySrcOptions) (minio.UploadInfo, error)); ok {
|
||||
return rf(ctx, dst, srcs...)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, minio.CopyDestOptions, ...minio.CopySrcOptions) minio.UploadInfo); ok {
|
||||
r0 = rf(ctx, dst, srcs...)
|
||||
} else {
|
||||
r0 = ret.Get(0).(minio.UploadInfo)
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, minio.CopyDestOptions, ...minio.CopySrcOptions) error); ok {
|
||||
r1 = rf(ctx, dst, srcs...)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// CopyObject provides a mock function with given fields: ctx, dst, src
|
||||
func (_m *s3CopyClient) CopyObject(ctx context.Context, dst minio.CopyDestOptions, src minio.CopySrcOptions) (minio.UploadInfo, error) {
|
||||
ret := _m.Called(ctx, dst, src)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for CopyObject")
|
||||
}
|
||||
|
||||
var r0 minio.UploadInfo
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, minio.CopyDestOptions, minio.CopySrcOptions) (minio.UploadInfo, error)); ok {
|
||||
return rf(ctx, dst, src)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, minio.CopyDestOptions, minio.CopySrcOptions) minio.UploadInfo); ok {
|
||||
r0 = rf(ctx, dst, src)
|
||||
} else {
|
||||
r0 = ret.Get(0).(minio.UploadInfo)
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, minio.CopyDestOptions, minio.CopySrcOptions) error); ok {
|
||||
r1 = rf(ctx, dst, src)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// StatObject provides a mock function with given fields: ctx, bucketName, objectName, opts
|
||||
func (_m *s3CopyClient) StatObject(ctx context.Context, bucketName string, objectName string, opts minio.GetObjectOptions) (minio.ObjectInfo, error) {
|
||||
ret := _m.Called(ctx, bucketName, objectName, opts)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for StatObject")
|
||||
}
|
||||
|
||||
var r0 minio.ObjectInfo
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, string, string, minio.GetObjectOptions) (minio.ObjectInfo, error)); ok {
|
||||
return rf(ctx, bucketName, objectName, opts)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, string, string, minio.GetObjectOptions) minio.ObjectInfo); ok {
|
||||
r0 = rf(ctx, bucketName, objectName, opts)
|
||||
} else {
|
||||
r0 = ret.Get(0).(minio.ObjectInfo)
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, string, string, minio.GetObjectOptions) error); ok {
|
||||
r1 = rf(ctx, bucketName, objectName, opts)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// newS3CopyClient creates a new instance of s3CopyClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func newS3CopyClient(t interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}) *s3CopyClient {
|
||||
mock := &s3CopyClient{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
||||
61
server/platform/shared/filestore/s3_overrides.go
Normal file
61
server/platform/shared/filestore/s3_overrides.go
Normal file
@@ -0,0 +1,61 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package filestore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
// customTransport is used to point the request to a different server.
|
||||
// This is helpful in situations where a different service is handling AWS S3 requests
|
||||
// from multiple Mattermost applications, and the Mattermost service itself does not
|
||||
// have any S3 credentials.
|
||||
type customTransport struct {
|
||||
host string
|
||||
scheme string
|
||||
client http.Client
|
||||
}
|
||||
|
||||
// RoundTrip implements the http.Roundtripper interface.
|
||||
func (t *customTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
// Roundtrippers should not modify the original request.
|
||||
newReq := req.Clone(context.Background())
|
||||
*newReq.URL = *req.URL
|
||||
req.URL.Scheme = t.scheme
|
||||
req.URL.Host = t.host
|
||||
return t.client.Do(req)
|
||||
}
|
||||
|
||||
// customProvider is a dummy credentials provider for the minio client to work
|
||||
// without actually providing credentials. This is needed with a custom transport
|
||||
// in cases where the minio client does not actually have credentials with itself,
|
||||
// rather needs responses from another entity.
|
||||
//
|
||||
// It satisfies the credentials.Provider interface.
|
||||
type customProvider struct {
|
||||
isSignV2 bool
|
||||
}
|
||||
|
||||
// RetrieveWithCredContext simply calls Retrieve, because we don't need to use
|
||||
// the param.
|
||||
func (cp customProvider) RetrieveWithCredContext(_ *credentials.CredContext) (credentials.Value, error) {
|
||||
return cp.Retrieve()
|
||||
}
|
||||
|
||||
// Retrieve just returns empty credentials.
|
||||
func (cp customProvider) Retrieve() (credentials.Value, error) {
|
||||
sign := credentials.SignatureV4
|
||||
if cp.isSignV2 {
|
||||
sign = credentials.SignatureV2
|
||||
}
|
||||
return credentials.Value{
|
||||
SignerType: sign,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// IsExpired always returns false.
|
||||
func (cp customProvider) IsExpired() bool { return false }
|
||||
877
server/platform/shared/filestore/s3store.go
Normal file
877
server/platform/shared/filestore/s3store.go
Normal file
@@ -0,0 +1,877 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package filestore
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
s3 "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio-go/v7/pkg/encrypt"
|
||||
"github.com/minio/minio-go/v7/pkg/s3utils"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/mattermost/mattermost/server/public/shared/mlog"
|
||||
)
|
||||
|
||||
// S3FileBackend contains all necessary information to communicate with
|
||||
// an AWS S3 compatible API backend.
|
||||
type S3FileBackend struct {
|
||||
endpoint string
|
||||
accessKey string
|
||||
secretKey string
|
||||
secure bool
|
||||
signV2 bool
|
||||
region string
|
||||
bucket string
|
||||
pathPrefix string
|
||||
encrypt bool
|
||||
trace bool
|
||||
client *s3.Client
|
||||
skipVerify bool
|
||||
timeout time.Duration
|
||||
presignExpires time.Duration
|
||||
isCloud bool // field to indicate whether this is running under Mattermost cloud or not.
|
||||
uploadPartSize int64
|
||||
storageClass string
|
||||
}
|
||||
|
||||
// S3FileBackendAuthError and S3FileBackendNoBucketError are aliases for the
|
||||
// generic backend errors. They are kept so external code (plugins,
|
||||
// historically-typed consumers) continues to compile.
|
||||
type (
|
||||
S3FileBackendAuthError = FileBackendAuthError
|
||||
S3FileBackendNoBucketError = FileBackendNoBucketError
|
||||
)
|
||||
|
||||
const (
|
||||
// This is not exported by minio. See: https://github.com/minio/minio-go/issues/1339
|
||||
bucketNotFound = "NoSuchBucket"
|
||||
invalidBucket = "InvalidBucketName"
|
||||
)
|
||||
|
||||
var (
|
||||
// Ensure that the ReaderAt interface is implemented.
|
||||
_ io.ReaderAt = (*s3WithCancel)(nil)
|
||||
_ FileBackendWithLinkGenerator = (*S3FileBackend)(nil)
|
||||
)
|
||||
|
||||
func getContentType(ext string) string {
|
||||
mimeType := mime.TypeByExtension(strings.ToLower(ext))
|
||||
if mimeType == "" {
|
||||
mimeType = "application/octet-stream"
|
||||
}
|
||||
return mimeType
|
||||
}
|
||||
|
||||
// NewS3FileBackend returns an instance of an S3FileBackend and determine if we are in Mattermost cloud or not.
|
||||
func NewS3FileBackend(settings FileBackendSettings) (*S3FileBackend, error) {
|
||||
return newS3FileBackend(settings, os.Getenv("MM_CLOUD_FILESTORE_BIFROST") != "")
|
||||
}
|
||||
|
||||
// NewS3FileBackendWithoutBifrost returns an instance of an S3FileBackend that will not use bifrost.
|
||||
func NewS3FileBackendWithoutBifrost(settings FileBackendSettings) (*S3FileBackend, error) {
|
||||
return newS3FileBackend(settings, false)
|
||||
}
|
||||
|
||||
func newS3FileBackend(settings FileBackendSettings, isCloud bool) (*S3FileBackend, error) {
|
||||
timeout := time.Duration(settings.AmazonS3RequestTimeoutMilliseconds) * time.Millisecond
|
||||
backend := &S3FileBackend{
|
||||
endpoint: settings.AmazonS3Endpoint,
|
||||
accessKey: settings.AmazonS3AccessKeyId,
|
||||
secretKey: settings.AmazonS3SecretAccessKey,
|
||||
secure: settings.AmazonS3SSL,
|
||||
signV2: settings.AmazonS3SignV2,
|
||||
region: settings.AmazonS3Region,
|
||||
bucket: settings.AmazonS3Bucket,
|
||||
pathPrefix: settings.AmazonS3PathPrefix,
|
||||
encrypt: settings.AmazonS3SSE,
|
||||
trace: settings.AmazonS3Trace,
|
||||
skipVerify: settings.SkipVerify,
|
||||
timeout: timeout,
|
||||
presignExpires: time.Duration(settings.AmazonS3PresignExpiresSeconds) * time.Second,
|
||||
uploadPartSize: settings.AmazonS3UploadPartSizeBytes,
|
||||
storageClass: settings.AmazonS3StorageClass,
|
||||
}
|
||||
cli, err := backend.s3New(isCloud)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
backend.client = cli
|
||||
backend.isCloud = isCloud
|
||||
return backend, nil
|
||||
}
|
||||
|
||||
type s3Trace struct {
|
||||
}
|
||||
|
||||
func (*s3Trace) Write(in []byte) (int, error) {
|
||||
mlog.Debug(string(in))
|
||||
return len(in), nil
|
||||
}
|
||||
|
||||
// Similar to s3.New() but allows initialization of signature v2 or signature v4 client.
|
||||
// If signV2 input is false, function always returns signature v4.
|
||||
//
|
||||
// Additionally this function also takes a user defined region, if set
|
||||
// disables automatic region lookup.
|
||||
func (b *S3FileBackend) s3New(isCloud bool) (*s3.Client, error) {
|
||||
var creds *credentials.Credentials
|
||||
|
||||
if isCloud {
|
||||
creds = credentials.New(customProvider{isSignV2: b.signV2})
|
||||
} else if b.accessKey == "" && b.secretKey == "" {
|
||||
creds = credentials.NewIAM("")
|
||||
} else if b.signV2 {
|
||||
creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV2)
|
||||
} else {
|
||||
creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV4)
|
||||
}
|
||||
|
||||
opts := s3.Options{
|
||||
Creds: creds,
|
||||
Secure: b.secure,
|
||||
Region: b.region,
|
||||
}
|
||||
|
||||
tr, err := s3.DefaultTransport(b.secure)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if b.skipVerify {
|
||||
tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
|
||||
}
|
||||
opts.Transport = tr
|
||||
|
||||
// If this is a cloud installation, we override the default transport.
|
||||
if isCloud {
|
||||
scheme := "http"
|
||||
if b.secure {
|
||||
scheme = "https"
|
||||
}
|
||||
newTransport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
newTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: b.skipVerify}
|
||||
opts.Transport = &customTransport{
|
||||
host: b.endpoint,
|
||||
scheme: scheme,
|
||||
client: http.Client{Transport: newTransport},
|
||||
}
|
||||
}
|
||||
|
||||
s3Clnt, err := s3.New(b.endpoint, &opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if b.trace {
|
||||
s3Clnt.TraceOn(&s3Trace{})
|
||||
}
|
||||
|
||||
if tr, ok := opts.Transport.(*http.Transport); ok {
|
||||
if tr.TLSClientConfig == nil {
|
||||
tr.TLSClientConfig = &tls.Config{}
|
||||
}
|
||||
tr.TLSClientConfig.MinVersion = tls.VersionTLS12
|
||||
}
|
||||
|
||||
return s3Clnt, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) DriverName() string {
|
||||
return driverS3
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) TestConnection() error {
|
||||
exists := true
|
||||
var err error
|
||||
// If a path prefix is present, we attempt to test the bucket by listing objects under the path
|
||||
// and just checking the first response. This is because the BucketExists call is only at a bucket level
|
||||
// and sometimes the user might only be allowed access to the specified path prefix.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
if b.pathPrefix != "" {
|
||||
obj := <-b.client.ListObjects(ctx, b.bucket, s3.ListObjectsOptions{Prefix: b.pathPrefix})
|
||||
if obj.Err != nil {
|
||||
typedErr := s3.ToErrorResponse(obj.Err)
|
||||
if typedErr.Code != bucketNotFound && typedErr.Code != invalidBucket {
|
||||
return &S3FileBackendAuthError{DetailedError: fmt.Sprintf("unable to list objects in the S3 bucket: %v", typedErr)}
|
||||
}
|
||||
exists = false
|
||||
}
|
||||
} else {
|
||||
exists, err = b.client.BucketExists(ctx, b.bucket)
|
||||
if err != nil {
|
||||
typedErr := s3.ToErrorResponse(err)
|
||||
if typedErr.Code != bucketNotFound && typedErr.Code != invalidBucket {
|
||||
return &S3FileBackendAuthError{DetailedError: fmt.Sprintf("unable to check if the S3 bucket exists: %v", typedErr)}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !exists {
|
||||
return &S3FileBackendNoBucketError{}
|
||||
}
|
||||
mlog.Debug("Connection to S3 or minio is good. Bucket exists.")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) MakeBucket() error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
err := b.client.MakeBucket(ctx, b.bucket, s3.MakeBucketOptions{Region: b.region})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "unable to create the s3 bucket")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// s3WithCancel is a wrapper struct which cancels the context
|
||||
// when the object is closed.
|
||||
type s3WithCancel struct {
|
||||
*s3.Object
|
||||
timer *time.Timer
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (sc *s3WithCancel) Close() error {
|
||||
sc.timer.Stop()
|
||||
sc.cancel()
|
||||
return sc.Object.Close()
|
||||
}
|
||||
|
||||
// CancelTimeout attempts to cancel the timeout for this reader. It allows calling
|
||||
// code to ignore the timeout in case of longer running operations. The methods returns
|
||||
// false if the timeout has already fired.
|
||||
func (sc *s3WithCancel) CancelTimeout() bool {
|
||||
return sc.timer.Stop()
|
||||
}
|
||||
|
||||
// Caller must close the first return value
|
||||
func (b *S3FileBackend) Reader(path string) (ReadCloseSeeker, error) {
|
||||
path, err := b.prefixedPath(path)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "unable to prefix path %s", path)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
minioObject, err := b.client.GetObject(ctx, b.bucket, path, s3.GetObjectOptions{})
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, errors.Wrapf(err, "unable to open file %s", path)
|
||||
}
|
||||
|
||||
sc := &s3WithCancel{
|
||||
Object: minioObject,
|
||||
timer: time.AfterFunc(b.timeout, cancel),
|
||||
cancel: cancel,
|
||||
}
|
||||
|
||||
return sc, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) ReadFile(path string) ([]byte, error) {
|
||||
encodedPath, err := b.prefixedPath(path)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "unable to prefix path %s", path)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
minioObject, err := b.client.GetObject(ctx, b.bucket, encodedPath, s3.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "unable to open file %s", encodedPath)
|
||||
}
|
||||
|
||||
defer minioObject.Close()
|
||||
f, err := io.ReadAll(minioObject)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "unable to read file %s", encodedPath)
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) FileExists(path string) (bool, error) {
|
||||
path, err := b.prefixedPath(path)
|
||||
if err != nil {
|
||||
return false, errors.Wrapf(err, "unable to prefix path %s", path)
|
||||
}
|
||||
|
||||
return b._fileExists(path)
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) _fileExists(path string) (bool, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
_, err := b.client.StatObject(ctx, b.bucket, path, s3.StatObjectOptions{})
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
var s3Err s3.ErrorResponse
|
||||
if errors.As(err, &s3Err); s3Err.Code == "NoSuchKey" {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return false, errors.Wrapf(err, "unable to know if file %s exists", path)
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) FileSize(path string) (int64, error) {
|
||||
path, err := b.prefixedPath(path)
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "unable to prefix path %s", path)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
info, err := b.client.StatObject(ctx, b.bucket, path, s3.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "unable to get file size for %s", path)
|
||||
}
|
||||
|
||||
return info.Size, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) FileModTime(path string) (time.Time, error) {
|
||||
path, err := b.prefixedPath(path)
|
||||
if err != nil {
|
||||
return time.Time{}, errors.Wrapf(err, "unable to prefix path %s", path)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
info, err := b.client.StatObject(ctx, b.bucket, path, s3.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return time.Time{}, errors.Wrapf(err, "unable to get modification time for file %s", path)
|
||||
}
|
||||
|
||||
return info.LastModified, nil
|
||||
}
|
||||
|
||||
// maxS3SingleCopySize is the largest source object (5GiB) that S3 allows for a
|
||||
// single server-side copy (CopyObject). Sources larger than this must be copied
|
||||
// with a server-side multipart copy (UploadPartCopy).
|
||||
const maxS3SingleCopySize = 5 * 1024 * 1024 * 1024
|
||||
|
||||
// s3CopyClient is the subset of the S3 client used to route a server-side copy.
|
||||
// It exists so copyObjectWithClient can be unit tested with a mock client.
|
||||
type s3CopyClient interface {
|
||||
StatObject(ctx context.Context, bucketName, objectName string, opts s3.StatObjectOptions) (s3.ObjectInfo, error)
|
||||
CopyObject(ctx context.Context, dst s3.CopyDestOptions, src s3.CopySrcOptions) (s3.UploadInfo, error)
|
||||
ComposeObject(ctx context.Context, dst s3.CopyDestOptions, srcs ...s3.CopySrcOptions) (s3.UploadInfo, error)
|
||||
}
|
||||
|
||||
// copyObject performs a server-side copy of a single object from srcOpts to
|
||||
// dstOpts, using the backend's request timeout.
|
||||
func (b *S3FileBackend) copyObject(srcOpts s3.CopySrcOptions, dstOpts s3.CopyDestOptions) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
|
||||
return copyObjectWithClient(ctx, b.client, srcOpts, dstOpts)
|
||||
}
|
||||
|
||||
// copyObjectWithClient uses CopyObject for sources up to 5GiB, and falls back to
|
||||
// ComposeObject — which performs a server-side multipart copy — for larger
|
||||
// sources, which S3's single-operation CopyObject rejects. The size is checked
|
||||
// explicitly rather than relying on ComposeObject's own single-copy fast path,
|
||||
// which is only taken when the source range Start is -1 (a value its input
|
||||
// validation rejects, so it is unreachable here).
|
||||
func copyObjectWithClient(ctx context.Context, client s3CopyClient, srcOpts s3.CopySrcOptions, dstOpts s3.CopyDestOptions) error {
|
||||
stat, err := client.StatObject(ctx, srcOpts.Bucket, srcOpts.Object, s3.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "unable to stat source file %s", srcOpts.Object)
|
||||
}
|
||||
|
||||
if stat.Size > maxS3SingleCopySize {
|
||||
_, err = client.ComposeObject(ctx, dstOpts, srcOpts)
|
||||
} else {
|
||||
_, err = client.CopyObject(ctx, dstOpts, srcOpts)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) CopyFile(oldPath, newPath string) error {
|
||||
oldPath, err := b.prefixedPath(oldPath)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "unable to prefix path %s", oldPath)
|
||||
}
|
||||
newPath = filepath.Join(b.pathPrefix, newPath)
|
||||
srcOpts := s3.CopySrcOptions{
|
||||
Bucket: b.bucket,
|
||||
Object: oldPath,
|
||||
}
|
||||
if b.encrypt {
|
||||
srcOpts.Encryption = encrypt.NewSSE()
|
||||
}
|
||||
|
||||
dstOpts := s3.CopyDestOptions{
|
||||
Bucket: b.bucket,
|
||||
Object: newPath,
|
||||
}
|
||||
if b.encrypt {
|
||||
dstOpts.Encryption = encrypt.NewSSE()
|
||||
}
|
||||
|
||||
if err := b.copyObject(srcOpts, dstOpts); err != nil {
|
||||
return errors.Wrapf(err, "unable to copy file from %s to %s", oldPath, newPath)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DecodeFilePathIfNeeded is a special method to URL decode all older
|
||||
// file paths. It is only needed for the migration, and will be removed
|
||||
// as soon as the migration is complete.
|
||||
func (b *S3FileBackend) DecodeFilePathIfNeeded(path string) error {
|
||||
// Encode and check if file path changes.
|
||||
// If there is no change, then there is no need to do anything.
|
||||
if path == s3utils.EncodePath(path) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check if encoded path exists.
|
||||
exists, err := b.lookupOriginalPath(s3utils.EncodePath(path))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !exists {
|
||||
return nil
|
||||
}
|
||||
|
||||
// If yes, then it needs to be migrated.
|
||||
// This is basically a copy of MoveFile without the path encoding.
|
||||
// We avoid any further refactoring because this method will be removed anyways.
|
||||
oldPath := filepath.Join(b.pathPrefix, s3utils.EncodePath(path))
|
||||
newPath := filepath.Join(b.pathPrefix, path)
|
||||
srcOpts := s3.CopySrcOptions{
|
||||
Bucket: b.bucket,
|
||||
Object: oldPath,
|
||||
}
|
||||
if b.encrypt {
|
||||
srcOpts.Encryption = encrypt.NewSSE()
|
||||
}
|
||||
|
||||
dstOpts := s3.CopyDestOptions{
|
||||
Bucket: b.bucket,
|
||||
Object: newPath,
|
||||
}
|
||||
if b.encrypt {
|
||||
dstOpts.Encryption = encrypt.NewSSE()
|
||||
}
|
||||
|
||||
if err := b.copyObject(srcOpts, dstOpts); err != nil {
|
||||
return errors.Wrapf(err, "unable to copy the file to %s to the new destination", newPath)
|
||||
}
|
||||
|
||||
ctx2, cancel2 := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel2()
|
||||
if err := b.client.RemoveObject(ctx2, b.bucket, oldPath, s3.RemoveObjectOptions{}); err != nil {
|
||||
return errors.Wrapf(err, "unable to remove the file old file %s", oldPath)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) MoveFile(oldPath, newPath string) error {
|
||||
oldPath, err := b.prefixedPath(oldPath)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "unable to prefix path %s", oldPath)
|
||||
}
|
||||
newPath = filepath.Join(b.pathPrefix, newPath)
|
||||
srcOpts := s3.CopySrcOptions{
|
||||
Bucket: b.bucket,
|
||||
Object: oldPath,
|
||||
}
|
||||
if b.encrypt {
|
||||
srcOpts.Encryption = encrypt.NewSSE()
|
||||
}
|
||||
|
||||
dstOpts := s3.CopyDestOptions{
|
||||
Bucket: b.bucket,
|
||||
Object: newPath,
|
||||
}
|
||||
if b.encrypt {
|
||||
dstOpts.Encryption = encrypt.NewSSE()
|
||||
}
|
||||
|
||||
if err := b.copyObject(srcOpts, dstOpts); err != nil {
|
||||
return errors.Wrapf(err, "unable to copy the file to %s to the new destination", newPath)
|
||||
}
|
||||
|
||||
ctx2, cancel2 := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel2()
|
||||
if err := b.client.RemoveObject(ctx2, b.bucket, oldPath, s3.RemoveObjectOptions{}); err != nil {
|
||||
return errors.Wrapf(err, "unable to remove the file old file %s", oldPath)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) WriteFile(fr io.Reader, path string) (int64, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
|
||||
return b.WriteFileContext(ctx, fr, path)
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) WriteFileContext(ctx context.Context, fr io.Reader, path string) (int64, error) {
|
||||
var contentType string
|
||||
path = filepath.Join(b.pathPrefix, path)
|
||||
ext := filepath.Ext(path)
|
||||
contentType = getContentType(ext)
|
||||
|
||||
options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize, b.storageClass)
|
||||
|
||||
objSize := int64(-1)
|
||||
if b.isCloud {
|
||||
options.DisableContentSha256 = true
|
||||
} else {
|
||||
// We pass an object size only in situations where bifrost is not
|
||||
// used. Bifrost needs to run in HTTPS, which is not yet deployed.
|
||||
switch t := fr.(type) {
|
||||
case *bytes.Buffer:
|
||||
objSize = int64(t.Len())
|
||||
case *os.File:
|
||||
if s, err2 := t.Stat(); err2 == nil {
|
||||
objSize = s.Size()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
info, err := b.client.PutObject(ctx, b.bucket, path, fr, objSize, options)
|
||||
if err != nil {
|
||||
return info.Size, errors.Wrapf(err, "unable write the data in the file %s", path)
|
||||
}
|
||||
|
||||
return info.Size, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) AppendFile(fr io.Reader, path string) (int64, error) {
|
||||
fp, err := b.prefixedPath(path)
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "unable to prefix path %s", path)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
if _, err2 := b.client.StatObject(ctx, b.bucket, fp, s3.StatObjectOptions{}); err2 != nil {
|
||||
return 0, errors.Wrapf(err2, "unable to find the file %s to append the data", path)
|
||||
}
|
||||
|
||||
contentType := getContentType(filepath.Ext(fp))
|
||||
|
||||
options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize, b.storageClass)
|
||||
sse := options.ServerSideEncryption
|
||||
partName := fp + ".part"
|
||||
ctx2, cancel2 := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel2()
|
||||
objSize := -1
|
||||
if b.isCloud {
|
||||
options.DisableContentSha256 = true
|
||||
}
|
||||
// We pass an object size only in situations where bifrost is not
|
||||
// used. Bifrost needs to run in HTTPS, which is not yet deployed.
|
||||
if buf, ok := fr.(*bytes.Buffer); ok && !b.isCloud {
|
||||
objSize = buf.Len()
|
||||
}
|
||||
info, err := b.client.PutObject(ctx2, b.bucket, partName, fr, int64(objSize), options)
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "unable append the data in the file %s", path)
|
||||
}
|
||||
defer func() {
|
||||
ctx4, cancel4 := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel4()
|
||||
b.client.RemoveObject(ctx4, b.bucket, partName, s3.RemoveObjectOptions{})
|
||||
}()
|
||||
|
||||
src1Opts := s3.CopySrcOptions{
|
||||
Bucket: b.bucket,
|
||||
Object: fp,
|
||||
}
|
||||
src2Opts := s3.CopySrcOptions{
|
||||
Bucket: b.bucket,
|
||||
Object: partName,
|
||||
}
|
||||
dstOpts := s3.CopyDestOptions{
|
||||
Bucket: b.bucket,
|
||||
Object: fp,
|
||||
Encryption: sse,
|
||||
}
|
||||
ctx3, cancel3 := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel3()
|
||||
_, err = b.client.ComposeObject(ctx3, dstOpts, src1Opts, src2Opts)
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "unable append the data in the file %s", path)
|
||||
}
|
||||
return info.Size, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) RemoveFile(path string) error {
|
||||
path, err := b.prefixedPath(path)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "unable to prefix path %s", path)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
if err := b.client.RemoveObject(ctx, b.bucket, path, s3.RemoveObjectOptions{}); err != nil {
|
||||
return errors.Wrapf(err, "unable to remove the file %s", path)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) listDirectory(path string, recursion bool) ([]string, error) {
|
||||
path, err := b.prefixedPath(path)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "unable to prefix path %s", path)
|
||||
}
|
||||
if !strings.HasSuffix(path, "/") && path != "" {
|
||||
// s3Clnt returns only the path itself when "/" is not present
|
||||
// appending "/" to make it consistent across all filestores
|
||||
path = path + "/"
|
||||
}
|
||||
|
||||
opts := s3.ListObjectsOptions{
|
||||
Prefix: path,
|
||||
Recursive: recursion,
|
||||
}
|
||||
var paths []string
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
var count int
|
||||
for object := range b.client.ListObjects(ctx, b.bucket, opts) {
|
||||
if object.Err != nil {
|
||||
return nil, errors.Wrapf(object.Err, "unable to list the directory %s", path)
|
||||
}
|
||||
// We strip the path prefix that gets applied,
|
||||
// so that it remains transparent to the application.
|
||||
object.Key = strings.TrimPrefix(object.Key, b.pathPrefix)
|
||||
trimmed := strings.Trim(object.Key, "/")
|
||||
if trimmed != "" {
|
||||
paths = append(paths, trimmed)
|
||||
}
|
||||
count++
|
||||
}
|
||||
// Check if only one item was returned and it matches the path prefix
|
||||
if count == 1 && len(paths) > 0 && strings.TrimRight(path, "/") == paths[0] {
|
||||
// Return a fs.PathError to maintain consistency
|
||||
return nil, &fs.PathError{Op: "readdir", Path: path, Err: fs.ErrNotExist}
|
||||
}
|
||||
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) ListDirectory(path string) ([]string, error) {
|
||||
return b.listDirectory(path, false)
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) ListDirectoryRecursively(path string) ([]string, error) {
|
||||
return b.listDirectory(path, true)
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) RemoveDirectory(path string) error {
|
||||
path, err := b.prefixedPath(path)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "unable to prefix path %s", path)
|
||||
}
|
||||
opts := s3.ListObjectsOptions{
|
||||
Prefix: path,
|
||||
Recursive: true,
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
|
||||
// List all objects in the directory
|
||||
for object := range b.client.ListObjects(ctx, b.bucket, opts) {
|
||||
if object.Err != nil {
|
||||
return errors.Wrapf(object.Err, "unable to list the directory %s", path)
|
||||
}
|
||||
|
||||
// Remove each object individually to avoid MD5 usage
|
||||
ctx2, cancel2 := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel2()
|
||||
err := b.client.RemoveObject(ctx2, b.bucket, object.Key, s3.RemoveObjectOptions{})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "unable to remove object %s from directory %s", object.Key, path)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ZipReader will create a zip of path. If path is a single file, it will zip the single file.
|
||||
// If deflate is true, the contents will be compressed. It will stream the zip to io.ReadCloser.
|
||||
func (b *S3FileBackend) ZipReader(path string, deflate bool) (io.ReadCloser, error) {
|
||||
deflateMethod := zip.Store
|
||||
if deflate {
|
||||
deflateMethod = zip.Deflate
|
||||
}
|
||||
|
||||
path, err := b.prefixedPath(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pr, pw := io.Pipe()
|
||||
|
||||
go func() {
|
||||
defer pw.Close()
|
||||
|
||||
zipWriter := zip.NewWriter(pw)
|
||||
defer zipWriter.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
|
||||
// Is path a single file?
|
||||
object, err := b.client.StatObject(ctx, b.bucket, path, s3.StatObjectOptions{})
|
||||
if err == nil {
|
||||
// We want the zipped file to be at the root of the zip. E.g., given a path of
|
||||
// "path/to/file.sh" we want the zip to have one file: "file.sh", not "path/to/file.sh".
|
||||
stripPath := filepath.Dir(path)
|
||||
if stripPath != "" {
|
||||
stripPath += "/"
|
||||
}
|
||||
if err = b._copyObjectToZipWriter(zipWriter, object, stripPath, deflateMethod); err != nil {
|
||||
pw.CloseWithError(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Is path a directory?
|
||||
path = path + "/"
|
||||
opts := s3.ListObjectsOptions{
|
||||
Prefix: path,
|
||||
Recursive: true,
|
||||
}
|
||||
ctx2, cancel2 := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel2()
|
||||
|
||||
for object := range b.client.ListObjects(ctx2, b.bucket, opts) {
|
||||
if object.Err != nil {
|
||||
pw.CloseWithError(errors.Wrapf(object.Err, "unable to list the directory %s", path))
|
||||
return
|
||||
}
|
||||
|
||||
if err = b._copyObjectToZipWriter(zipWriter, object, path, deflateMethod); err != nil {
|
||||
pw.CloseWithError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return pr, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) _copyObjectToZipWriter(zipWriter *zip.Writer, object s3.ObjectInfo, stripPath string, deflateMethod uint16) error {
|
||||
// We strip the path prefix that gets applied,
|
||||
// so that it remains transparent to the application.
|
||||
object.Key = strings.TrimPrefix(object.Key, b.pathPrefix)
|
||||
|
||||
// We strip the path prefix + path so the zip file is relative to the root of the requested path
|
||||
relPath := strings.TrimPrefix(object.Key, stripPath)
|
||||
header := &zip.FileHeader{
|
||||
Name: relPath,
|
||||
Method: deflateMethod,
|
||||
Modified: object.LastModified,
|
||||
}
|
||||
header.SetMode(0644) // rw-r--r-- permissions
|
||||
|
||||
writer, err := zipWriter.CreateHeader(header)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "unable to create zip entry for %s", object.Key)
|
||||
}
|
||||
|
||||
reader, err := b.Reader(object.Key)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "unable to create reader for %s", object.Key)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
_, err = io.Copy(writer, reader)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "unable to copy content for %s", object.Key)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) GeneratePublicLink(path string) (string, time.Duration, error) {
|
||||
path, err := b.prefixedPath(path)
|
||||
if err != nil {
|
||||
return "", 0, errors.Wrapf(err, "unable to prefix path %s", path)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
|
||||
defer cancel()
|
||||
|
||||
reqParams := make(url.Values)
|
||||
reqParams.Set("response-content-disposition", "attachment")
|
||||
|
||||
req, err := b.client.PresignedGetObject(ctx, b.bucket, path, b.presignExpires, reqParams)
|
||||
if err != nil {
|
||||
return "", 0, errors.Wrapf(err, "unable to generate public link for %s", path)
|
||||
}
|
||||
|
||||
return req.String(), b.presignExpires, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) lookupOriginalPath(s string) (bool, error) {
|
||||
exists, err := b._fileExists(filepath.Join(b.pathPrefix, s))
|
||||
if err != nil {
|
||||
var s3Err s3.ErrorResponse
|
||||
// Sometimes S3 will not allow to access other paths.
|
||||
// In that case, we consider them as not exists.
|
||||
if errors.As(err, &s3Err); s3Err.Code == "AccessDenied" {
|
||||
return false, nil
|
||||
}
|
||||
return false, errors.Wrapf(err, "unable to check for file path %s", s)
|
||||
}
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
func (b *S3FileBackend) prefixedPath(s string) (string, error) {
|
||||
if b.isCloud {
|
||||
// We do a lookup of the original path for compatibility purposes.
|
||||
exists, err := b.lookupOriginalPath(s)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// If it exists, then we don't want to encode it
|
||||
// because it's an old path.
|
||||
if !exists {
|
||||
// If the request is routed via bifrost, then we need to encode the path
|
||||
// to avoid signature validation errors.
|
||||
// This happens because in bifrost, we are signing the URL outside the SDK
|
||||
// and therefore the signature sent from the bifrost client
|
||||
// will contain the encoded path, whereas the original path is sent
|
||||
// un-encoded.
|
||||
// More info at: https://github.com/aws/aws-sdk-go-v2/blob/1e4148ac334a4ea7abe31bd984a31dc761bb289d/aws/signer/v4/v4.go#L20
|
||||
s = s3utils.EncodePath(s)
|
||||
}
|
||||
}
|
||||
return filepath.Join(b.pathPrefix, s), nil
|
||||
}
|
||||
|
||||
func s3PutOptions(encrypted bool, contentType string, uploadPartSize int64, storageClass string) s3.PutObjectOptions {
|
||||
options := s3.PutObjectOptions{}
|
||||
if encrypted {
|
||||
options.ServerSideEncryption = encrypt.NewSSE()
|
||||
}
|
||||
options.ContentType = contentType
|
||||
options.PartSize = uint64(uploadPartSize)
|
||||
options.StorageClass = storageClass
|
||||
|
||||
return options
|
||||
}
|
||||
79
server/platform/shared/filestore/s3store_copyobject_test.go
Normal file
79
server/platform/shared/filestore/s3store_copyobject_test.go
Normal file
@@ -0,0 +1,79 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package filestore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
s3 "github.com/minio/minio-go/v7"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// ensure the real S3 client satisfies the interface used by copyObjectWithClient.
|
||||
var _ s3CopyClient = (*s3.Client)(nil)
|
||||
|
||||
// mockS3CopyClient is a minimal s3CopyClient that records which copy operation
|
||||
// was invoked and lets StatObject's result be controlled per test.
|
||||
type mockS3CopyClient struct {
|
||||
statInfo s3.ObjectInfo
|
||||
statErr error
|
||||
|
||||
copyCalled bool
|
||||
composeCalled bool
|
||||
}
|
||||
|
||||
func (m *mockS3CopyClient) StatObject(_ context.Context, _, _ string, _ s3.StatObjectOptions) (s3.ObjectInfo, error) {
|
||||
return m.statInfo, m.statErr
|
||||
}
|
||||
|
||||
func (m *mockS3CopyClient) CopyObject(_ context.Context, _ s3.CopyDestOptions, _ s3.CopySrcOptions) (s3.UploadInfo, error) {
|
||||
m.copyCalled = true
|
||||
return s3.UploadInfo{}, nil
|
||||
}
|
||||
|
||||
func (m *mockS3CopyClient) ComposeObject(_ context.Context, _ s3.CopyDestOptions, _ ...s3.CopySrcOptions) (s3.UploadInfo, error) {
|
||||
m.composeCalled = true
|
||||
return s3.UploadInfo{}, nil
|
||||
}
|
||||
|
||||
var (
|
||||
testCopySrc = s3.CopySrcOptions{Bucket: "bucket", Object: "src"}
|
||||
testCopyDst = s3.CopyDestOptions{Bucket: "bucket", Object: "dst"}
|
||||
)
|
||||
|
||||
func Test_copyObject_UsesCompose_ForLarge(t *testing.T) {
|
||||
mock := &mockS3CopyClient{statInfo: s3.ObjectInfo{Size: maxS3SingleCopySize + 1}}
|
||||
|
||||
err := copyObjectWithClient(context.Background(), mock, testCopySrc, testCopyDst)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.True(t, mock.composeCalled, "expected ComposeObject (multipart copy) for sources larger than 5GiB")
|
||||
assert.False(t, mock.copyCalled, "expected CopyObject not to be used for sources larger than 5GiB")
|
||||
}
|
||||
|
||||
func Test_copyObject_UsesCopy_ForSmall(t *testing.T) {
|
||||
// A source exactly at the 5GiB limit must still use the single CopyObject.
|
||||
mock := &mockS3CopyClient{statInfo: s3.ObjectInfo{Size: maxS3SingleCopySize}}
|
||||
|
||||
err := copyObjectWithClient(context.Background(), mock, testCopySrc, testCopyDst)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.True(t, mock.copyCalled, "expected CopyObject for sources up to 5GiB")
|
||||
assert.False(t, mock.composeCalled, "expected ComposeObject not to be used for sources up to 5GiB")
|
||||
}
|
||||
|
||||
func Test_copyObject_PropagatesStatError(t *testing.T) {
|
||||
statErr := errors.New("stat failed")
|
||||
mock := &mockS3CopyClient{statErr: statErr}
|
||||
|
||||
err := copyObjectWithClient(context.Background(), mock, testCopySrc, testCopyDst)
|
||||
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, statErr, "StatObject error must be propagated")
|
||||
assert.False(t, mock.copyCalled, "no copy should be attempted when StatObject fails")
|
||||
assert.False(t, mock.composeCalled, "no compose should be attempted when StatObject fails")
|
||||
}
|
||||
456
server/platform/shared/filestore/s3store_test.go
Normal file
456
server/platform/shared/filestore/s3store_test.go
Normal file
@@ -0,0 +1,456 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package filestore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http/httptest"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mattermost/mattermost/server/public/model"
|
||||
s3 "github.com/minio/minio-go/v7"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCheckMandatoryS3Fields(t *testing.T) {
|
||||
cfg := FileBackendSettings{}
|
||||
|
||||
err := cfg.CheckMandatoryS3Fields()
|
||||
require.Error(t, err)
|
||||
require.Equal(t, err.Error(), "missing s3 bucket settings", "should've failed with missing s3 bucket")
|
||||
|
||||
cfg.AmazonS3Bucket = "test-mm"
|
||||
err = cfg.CheckMandatoryS3Fields()
|
||||
require.NoError(t, err)
|
||||
|
||||
cfg.AmazonS3Endpoint = ""
|
||||
err = cfg.CheckMandatoryS3Fields()
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, "s3.amazonaws.com", cfg.AmazonS3Endpoint, "should've set the endpoint to the default")
|
||||
}
|
||||
|
||||
func TestMakeBucket(t *testing.T) {
|
||||
s3Host := os.Getenv("CI_MINIO_HOST")
|
||||
if s3Host == "" {
|
||||
s3Host = "localhost"
|
||||
}
|
||||
|
||||
s3Port := os.Getenv("CI_MINIO_PORT")
|
||||
if s3Port == "" {
|
||||
s3Port = "9000"
|
||||
}
|
||||
|
||||
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
|
||||
|
||||
// Generate a random bucket name
|
||||
b := make([]byte, 30)
|
||||
rand.Read(b)
|
||||
bucketName := base64.StdEncoding.EncodeToString(b)
|
||||
bucketName = strings.ToLower(bucketName)
|
||||
bucketName = strings.Replace(bucketName, "+", "", -1)
|
||||
bucketName = strings.Replace(bucketName, "/", "", -1)
|
||||
|
||||
cfg := FileBackendSettings{
|
||||
DriverName: model.ImageDriverS3,
|
||||
AmazonS3AccessKeyId: model.MinioAccessKey,
|
||||
AmazonS3SecretAccessKey: model.MinioSecretKey,
|
||||
AmazonS3Bucket: bucketName,
|
||||
AmazonS3Endpoint: s3Endpoint,
|
||||
AmazonS3Region: "",
|
||||
AmazonS3PathPrefix: "",
|
||||
AmazonS3SSL: false,
|
||||
SkipVerify: false,
|
||||
AmazonS3RequestTimeoutMilliseconds: 5000,
|
||||
}
|
||||
|
||||
fileBackend, err := NewS3FileBackend(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = fileBackend.MakeBucket()
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestTimeout(t *testing.T) {
|
||||
s3Host := os.Getenv("CI_MINIO_HOST")
|
||||
if s3Host == "" {
|
||||
s3Host = "localhost"
|
||||
}
|
||||
|
||||
s3Port := os.Getenv("CI_MINIO_PORT")
|
||||
if s3Port == "" {
|
||||
s3Port = "9000"
|
||||
}
|
||||
|
||||
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
|
||||
|
||||
// Generate a random bucket name
|
||||
b := make([]byte, 30)
|
||||
rand.Read(b)
|
||||
bucketName := base64.StdEncoding.EncodeToString(b)
|
||||
bucketName = strings.ToLower(bucketName)
|
||||
bucketName = strings.Replace(bucketName, "+", "", -1)
|
||||
bucketName = strings.Replace(bucketName, "/", "", -1)
|
||||
|
||||
cfg := FileBackendSettings{
|
||||
DriverName: model.ImageDriverS3,
|
||||
AmazonS3AccessKeyId: model.MinioAccessKey,
|
||||
AmazonS3SecretAccessKey: model.MinioSecretKey,
|
||||
AmazonS3Bucket: bucketName,
|
||||
AmazonS3Endpoint: s3Endpoint,
|
||||
AmazonS3Region: "",
|
||||
AmazonS3PathPrefix: "",
|
||||
AmazonS3SSL: false,
|
||||
SkipVerify: false,
|
||||
AmazonS3RequestTimeoutMilliseconds: 0,
|
||||
}
|
||||
|
||||
t.Run("MakeBucket", func(t *testing.T) {
|
||||
fileBackend, err := NewS3FileBackend(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = fileBackend.MakeBucket()
|
||||
require.True(t, errors.Is(err, context.DeadlineExceeded))
|
||||
})
|
||||
|
||||
t.Run("WriteFile", func(t *testing.T) {
|
||||
cfg.AmazonS3RequestTimeoutMilliseconds = 1000
|
||||
|
||||
fileBackend, err := NewS3FileBackend(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = fileBackend.MakeBucket()
|
||||
require.NoError(t, err)
|
||||
|
||||
r, w := io.Pipe()
|
||||
go func() {
|
||||
defer w.Close()
|
||||
for range 10 {
|
||||
_, writeErr := w.Write([]byte("data"))
|
||||
require.NoError(t, writeErr)
|
||||
time.Sleep(time.Millisecond * 200)
|
||||
}
|
||||
}()
|
||||
|
||||
_, err = fileBackend.WriteFile(r, "tests/"+randomString()+".png")
|
||||
require.True(t, errors.Is(err, context.DeadlineExceeded))
|
||||
})
|
||||
}
|
||||
|
||||
func TestInsecureMakeBucket(t *testing.T) {
|
||||
s3Host := os.Getenv("CI_MINIO_HOST")
|
||||
if s3Host == "" {
|
||||
s3Host = "localhost"
|
||||
}
|
||||
|
||||
s3Port := os.Getenv("CI_MINIO_PORT")
|
||||
if s3Port == "" {
|
||||
s3Port = "9000"
|
||||
}
|
||||
|
||||
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
|
||||
|
||||
proxySelfSignedHTTPS := newTLSProxyServer(&url.URL{Scheme: "http", Host: s3Endpoint})
|
||||
defer proxySelfSignedHTTPS.Close()
|
||||
|
||||
enableInsecure, secure := true, false
|
||||
|
||||
testCases := []struct {
|
||||
description string
|
||||
skipVerify bool
|
||||
expectedAllowed bool
|
||||
}{
|
||||
{"allow self-signed HTTPS when insecure enabled", enableInsecure, true},
|
||||
{"reject self-signed HTTPS when secured", secure, false},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.description, func(t *testing.T) {
|
||||
// Generate a random bucket name
|
||||
b := make([]byte, 30)
|
||||
rand.Read(b)
|
||||
bucketName := base64.StdEncoding.EncodeToString(b)
|
||||
bucketName = strings.ToLower(bucketName)
|
||||
bucketName = strings.Replace(bucketName, "+", "", -1)
|
||||
bucketName = strings.Replace(bucketName, "/", "", -1)
|
||||
|
||||
cfg := FileBackendSettings{
|
||||
DriverName: model.ImageDriverS3,
|
||||
AmazonS3AccessKeyId: model.MinioAccessKey,
|
||||
AmazonS3SecretAccessKey: model.MinioSecretKey,
|
||||
AmazonS3Bucket: bucketName,
|
||||
AmazonS3Endpoint: proxySelfSignedHTTPS.URL[8:],
|
||||
AmazonS3Region: "",
|
||||
AmazonS3PathPrefix: "",
|
||||
AmazonS3SSL: true,
|
||||
SkipVerify: testCase.skipVerify,
|
||||
AmazonS3RequestTimeoutMilliseconds: 5000,
|
||||
}
|
||||
|
||||
fileBackend, err := NewS3FileBackend(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = fileBackend.MakeBucket()
|
||||
if testCase.expectedAllowed {
|
||||
require.NoError(t, err)
|
||||
} else {
|
||||
require.Error(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
func newTLSProxyServer(backend *url.URL) *httptest.Server {
|
||||
return httptest.NewTLSServer(httputil.NewSingleHostReverseProxy(backend))
|
||||
}
|
||||
|
||||
func TestS3WithCancel(t *testing.T) {
|
||||
// Some of these tests use time.Sleep to wait for the timeout to expire.
|
||||
// They are run in parallel to reduce wait times.
|
||||
|
||||
t.Run("zero timeout", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
r, ctx := newMockS3WithCancel(0, nil)
|
||||
|
||||
time.Sleep(10 * time.Millisecond) // give the context time to cancel
|
||||
|
||||
require.False(t, r.CancelTimeout())
|
||||
require.Error(t, ctx.Err())
|
||||
})
|
||||
|
||||
t.Run("timeout", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
r, ctx := newMockS3WithCancel(50*time.Millisecond, nil)
|
||||
|
||||
time.Sleep(100 * time.Millisecond) // give the context time to cancel
|
||||
|
||||
require.False(t, r.CancelTimeout())
|
||||
require.Error(t, ctx.Err())
|
||||
})
|
||||
|
||||
t.Run("timeout cancel", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
r, ctx := newMockS3WithCancel(50*time.Millisecond, nil)
|
||||
|
||||
time.Sleep(10 * time.Millisecond) // give the context time to cancel
|
||||
|
||||
require.True(t, r.CancelTimeout())
|
||||
require.NoError(t, ctx.Err())
|
||||
|
||||
time.Sleep(100 * time.Millisecond) // wait for the original (canceled) timeout to expire
|
||||
|
||||
require.False(t, r.CancelTimeout())
|
||||
require.NoError(t, ctx.Err())
|
||||
require.NoError(t, r.Close())
|
||||
})
|
||||
|
||||
t.Run("timeout closed", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
r, ctx := newMockS3WithCancel(50*time.Millisecond, nil)
|
||||
|
||||
time.Sleep(10 * time.Millisecond) // give the context time to cancel
|
||||
|
||||
require.True(t, r.CancelTimeout())
|
||||
require.NoError(t, ctx.Err())
|
||||
require.NoError(t, r.Close())
|
||||
|
||||
time.Sleep(100 * time.Millisecond) // wait for the original (canceled) timeout to expire
|
||||
|
||||
require.False(t, r.CancelTimeout())
|
||||
require.Error(t, ctx.Err())
|
||||
require.NoError(t, r.Close())
|
||||
})
|
||||
|
||||
t.Run("close cancel close", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
r, ctx := newMockS3WithCancel(50*time.Millisecond, nil)
|
||||
|
||||
time.Sleep(10 * time.Millisecond) // give the context time to cancel
|
||||
|
||||
require.True(t, r.CancelTimeout())
|
||||
require.NoError(t, r.Close())
|
||||
require.Error(t, ctx.Err())
|
||||
require.False(t, r.CancelTimeout())
|
||||
require.Error(t, ctx.Err())
|
||||
require.NoError(t, r.Close())
|
||||
})
|
||||
|
||||
t.Run("close error", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
r, ctx := newMockS3WithCancel(50*time.Millisecond, errors.New("test error"))
|
||||
|
||||
time.Sleep(10 * time.Millisecond) // give the context time to cancel
|
||||
|
||||
require.NoError(t, ctx.Err())
|
||||
require.Error(t, r.Close())
|
||||
require.False(t, r.CancelTimeout())
|
||||
require.Error(t, ctx.Err())
|
||||
})
|
||||
}
|
||||
|
||||
func newMockS3WithCancel(timeout time.Duration, closeErr error) (*fauxCloser, context.Context) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
return &fauxCloser{
|
||||
s3WithCancel: &s3WithCancel{
|
||||
Object: &s3.Object{},
|
||||
timer: time.AfterFunc(timeout, cancel),
|
||||
cancel: cancel,
|
||||
},
|
||||
closeErr: closeErr,
|
||||
}, ctx
|
||||
}
|
||||
|
||||
type fauxCloser struct {
|
||||
*s3WithCancel
|
||||
closeErr error
|
||||
}
|
||||
|
||||
func (fc fauxCloser) Close() error {
|
||||
fc.s3WithCancel.timer.Stop()
|
||||
fc.s3WithCancel.cancel()
|
||||
return fc.closeErr
|
||||
}
|
||||
|
||||
func TestListDirectory(t *testing.T) {
|
||||
s3Host := os.Getenv("CI_MINIO_HOST")
|
||||
if s3Host == "" {
|
||||
s3Host = "localhost"
|
||||
}
|
||||
|
||||
s3Port := os.Getenv("CI_MINIO_PORT")
|
||||
if s3Port == "" {
|
||||
s3Port = "9000"
|
||||
}
|
||||
|
||||
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
|
||||
|
||||
cfg := FileBackendSettings{
|
||||
DriverName: driverS3,
|
||||
AmazonS3AccessKeyId: "minioaccesskey",
|
||||
AmazonS3SecretAccessKey: "miniosecretkey",
|
||||
AmazonS3Bucket: "mattermost-test-1",
|
||||
AmazonS3Region: "",
|
||||
AmazonS3Endpoint: s3Endpoint,
|
||||
AmazonS3PathPrefix: "",
|
||||
AmazonS3SSL: false,
|
||||
AmazonS3SSE: false,
|
||||
AmazonS3RequestTimeoutMilliseconds: 5000,
|
||||
}
|
||||
|
||||
fileBackend, err := NewS3FileBackend(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
found, err := fileBackend.client.BucketExists(context.Background(), cfg.AmazonS3Bucket)
|
||||
require.NoError(t, err)
|
||||
|
||||
if !found {
|
||||
err = fileBackend.MakeBucket()
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
fileBackend.pathPrefix = "19700101/"
|
||||
require.NoError(t, err)
|
||||
b := []byte("test")
|
||||
|
||||
path1 := "19700101/" + randomString() + ".txt"
|
||||
_, err = fileBackend.WriteFile(bytes.NewReader(b), path1)
|
||||
require.NoError(t, err, "Failed to write file1 to S3")
|
||||
|
||||
_, err = fileBackend.ListDirectory("")
|
||||
var pErr *fs.PathError
|
||||
assert.True(t, errors.As(err, &pErr), "error is not of type fs.PathError")
|
||||
|
||||
err = fileBackend.RemoveFile(path1)
|
||||
require.NoError(t, err, "Failed to remove file1 from S3")
|
||||
|
||||
err = fileBackend.RemoveDirectory("19700101")
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestWriteFileVideoMimeTypes(t *testing.T) {
|
||||
s3Host := os.Getenv("CI_MINIO_HOST")
|
||||
if s3Host == "" {
|
||||
s3Host = "localhost"
|
||||
}
|
||||
|
||||
s3Port := os.Getenv("CI_MINIO_PORT")
|
||||
if s3Port == "" {
|
||||
s3Port = "9000"
|
||||
}
|
||||
|
||||
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
|
||||
|
||||
// Generate a random bucket name
|
||||
b := make([]byte, 30)
|
||||
rand.Read(b)
|
||||
bucketName := base64.StdEncoding.EncodeToString(b)
|
||||
bucketName = strings.ToLower(bucketName)
|
||||
bucketName = strings.Replace(bucketName, "+", "", -1)
|
||||
bucketName = strings.Replace(bucketName, "/", "", -1)
|
||||
|
||||
cfg := FileBackendSettings{
|
||||
DriverName: model.ImageDriverS3,
|
||||
AmazonS3AccessKeyId: model.MinioAccessKey,
|
||||
AmazonS3SecretAccessKey: model.MinioSecretKey,
|
||||
AmazonS3Bucket: bucketName,
|
||||
AmazonS3Endpoint: s3Endpoint,
|
||||
AmazonS3SSL: false,
|
||||
AmazonS3RequestTimeoutMilliseconds: 5000,
|
||||
}
|
||||
|
||||
fileBackend, err := NewS3FileBackend(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = fileBackend.MakeBucket()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Test video types
|
||||
// Test video types with multiple valid MIME types
|
||||
testContent := []byte("test-video-content")
|
||||
videoTypes := map[string][]string{
|
||||
".avi": {"video/vnd.avi", "video/x-msvideo"},
|
||||
".mpeg": {"video/mpeg"},
|
||||
".mp4": {"video/mp4"},
|
||||
}
|
||||
|
||||
for ext, validMimeTypes := range videoTypes {
|
||||
t.Run(strings.TrimPrefix(ext, "."), func(t *testing.T) {
|
||||
path := "test" + ext
|
||||
reader := bytes.NewReader(testContent)
|
||||
|
||||
written, err := fileBackend.WriteFile(reader, path)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(len(testContent)), written)
|
||||
|
||||
// Verify the file exists with correct mime type
|
||||
props, err := fileBackend.client.StatObject(
|
||||
context.Background(),
|
||||
bucketName,
|
||||
path,
|
||||
s3.StatObjectOptions{},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Ensure the MIME type is one of the expected values
|
||||
assert.Contains(t, validMimeTypes, props.ContentType, "Unexpected MIME type: %s", props.ContentType)
|
||||
|
||||
defer func() {
|
||||
err = fileBackend.RemoveFile(path)
|
||||
require.NoError(t, err)
|
||||
}()
|
||||
})
|
||||
}
|
||||
}
|
||||
183
server/platform/shared/mail/inbucket.go
Normal file
183
server/platform/shared/mail/inbucket.go
Normal file
@@ -0,0 +1,183 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package mail
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
InbucketAPI = "/api/v1/mailbox/"
|
||||
)
|
||||
|
||||
// OutputJSONHeader holds the received Header to test sending emails (inbucket)
|
||||
type JSONMessageHeaderInbucket []struct {
|
||||
Mailbox string
|
||||
ID string `json:"Id"`
|
||||
From, Subject, Date string
|
||||
To []string
|
||||
Size int
|
||||
}
|
||||
|
||||
// OutputJSONMessage holds the received Message fto test sending emails (inbucket)
|
||||
type JSONMessageInbucket struct {
|
||||
Mailbox string
|
||||
ID string `json:"Id"`
|
||||
From, Subject, Date string
|
||||
Size int
|
||||
Header map[string][]string
|
||||
Body struct {
|
||||
Text string
|
||||
HTML string `json:"Html"`
|
||||
}
|
||||
Attachments []struct {
|
||||
Filename string
|
||||
ContentType string `json:"content-type"`
|
||||
DownloadLink string `json:"download-link"`
|
||||
Bytes []byte `json:"-"`
|
||||
}
|
||||
}
|
||||
|
||||
func ParseEmail(email string) string {
|
||||
pos := strings.Index(email, "@")
|
||||
parsedEmail := email[0:pos]
|
||||
return parsedEmail
|
||||
}
|
||||
|
||||
func GetMailBox(email string) (results JSONMessageHeaderInbucket, err error) {
|
||||
parsedEmail := ParseEmail(email)
|
||||
|
||||
url := fmt.Sprintf("%s%s%s", getInbucketHost(), InbucketAPI, parsedEmail)
|
||||
resp, err := http.Get(url)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
}()
|
||||
|
||||
if resp.Body == nil {
|
||||
return nil, fmt.Errorf("no mailbox")
|
||||
}
|
||||
|
||||
var record JSONMessageHeaderInbucket
|
||||
err = json.NewDecoder(resp.Body).Decode(&record)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error: %w", err)
|
||||
}
|
||||
if len(record) == 0 {
|
||||
return nil, fmt.Errorf("no mailbox")
|
||||
}
|
||||
|
||||
return record, nil
|
||||
}
|
||||
|
||||
func GetMessageFromMailbox(email, id string) (JSONMessageInbucket, error) {
|
||||
parsedEmail := ParseEmail(email)
|
||||
|
||||
var record JSONMessageInbucket
|
||||
|
||||
url := fmt.Sprintf("%s%s%s/%s", getInbucketHost(), InbucketAPI, parsedEmail, id)
|
||||
emailResponse, err := http.Get(url)
|
||||
if err != nil {
|
||||
return record, err
|
||||
}
|
||||
defer func() {
|
||||
io.Copy(io.Discard, emailResponse.Body)
|
||||
emailResponse.Body.Close()
|
||||
}()
|
||||
|
||||
if err = json.NewDecoder(emailResponse.Body).Decode(&record); err != nil {
|
||||
return record, err
|
||||
}
|
||||
|
||||
// download attachments
|
||||
if len(record.Attachments) > 0 {
|
||||
for i := range record.Attachments {
|
||||
var bytes []byte
|
||||
bytes, err = downloadAttachment(record.Attachments[i].DownloadLink)
|
||||
if err != nil {
|
||||
return record, err
|
||||
}
|
||||
record.Attachments[i].Bytes = make([]byte, len(bytes))
|
||||
copy(record.Attachments[i].Bytes, bytes)
|
||||
}
|
||||
}
|
||||
|
||||
return record, err
|
||||
}
|
||||
|
||||
func downloadAttachment(url string) ([]byte, error) {
|
||||
attachmentResponse, err := http.Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer attachmentResponse.Body.Close()
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
io.Copy(buf, attachmentResponse.Body)
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
func DeleteMailBox(email string) (err error) {
|
||||
parsedEmail := ParseEmail(email)
|
||||
|
||||
url := fmt.Sprintf("%s%s%s", getInbucketHost(), InbucketAPI, parsedEmail)
|
||||
req, err := http.NewRequest("DELETE", url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
client := &http.Client{}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func RetryInbucket(attempts int, callback func() error) (err error) {
|
||||
for i := 0; ; i++ {
|
||||
err = callback()
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if i >= (attempts - 1) {
|
||||
break
|
||||
}
|
||||
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
fmt.Println("retrying...")
|
||||
}
|
||||
return fmt.Errorf("after %d attempts, last error: %s", attempts, err)
|
||||
}
|
||||
|
||||
func getInbucketHost() (host string) {
|
||||
inbucket_host := os.Getenv("CI_INBUCKET_HOST")
|
||||
if inbucket_host == "" {
|
||||
inbucket_host = "localhost"
|
||||
}
|
||||
|
||||
inbucket_port := os.Getenv("CI_INBUCKET_PORT")
|
||||
if inbucket_port == "" {
|
||||
inbucket_port = "9001"
|
||||
}
|
||||
return "http://" + net.JoinHostPort(inbucket_host, inbucket_port)
|
||||
}
|
||||
402
server/platform/shared/mail/mail.go
Normal file
402
server/platform/shared/mail/mail.go
Normal file
@@ -0,0 +1,402 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package mail
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/mail"
|
||||
"net/smtp"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/jaytaylor/html2text"
|
||||
"github.com/pkg/errors"
|
||||
gomail "github.com/wneessen/go-mail"
|
||||
|
||||
"github.com/mattermost/mattermost/server/public/model"
|
||||
"github.com/mattermost/mattermost/server/public/shared/mlog"
|
||||
)
|
||||
|
||||
const (
|
||||
TLS = "TLS"
|
||||
StartTLS = "STARTTLS"
|
||||
)
|
||||
|
||||
type SMTPConfig struct {
|
||||
ConnectionSecurity string
|
||||
SkipServerCertificateVerification bool
|
||||
Hostname string
|
||||
ServerName string
|
||||
Server string
|
||||
Port string
|
||||
ServerTimeout int
|
||||
Username string
|
||||
Password string
|
||||
EnableSMTPAuth bool
|
||||
SendEmailNotifications bool
|
||||
FeedbackName string
|
||||
FeedbackEmail string
|
||||
ReplyToAddress string
|
||||
}
|
||||
|
||||
type mailData struct {
|
||||
mimeTo string
|
||||
smtpTo string
|
||||
from mail.Address
|
||||
cc string
|
||||
replyTo mail.Address
|
||||
subject string
|
||||
htmlBody string
|
||||
embeddedFiles map[string]io.Reader
|
||||
mimeHeaders map[string]string
|
||||
messageID string
|
||||
inReplyTo string
|
||||
references string
|
||||
category string
|
||||
}
|
||||
|
||||
// smtpClient is implemented by an smtp.Client. See https://golang.org/pkg/net/smtp/#Client.
|
||||
type smtpClient interface {
|
||||
Mail(string) error
|
||||
Rcpt(string) error
|
||||
Data() (io.WriteCloser, error)
|
||||
}
|
||||
|
||||
type authChooser struct {
|
||||
smtp.Auth
|
||||
config *SMTPConfig
|
||||
}
|
||||
|
||||
func (a *authChooser) Start(server *smtp.ServerInfo) (string, []byte, error) {
|
||||
smtpAddress := a.config.ServerName + ":" + a.config.Port
|
||||
a.Auth = LoginAuth(a.config.Username, a.config.Password, smtpAddress)
|
||||
if slices.Contains(server.Auth, "PLAIN") {
|
||||
a.Auth = smtp.PlainAuth("", a.config.Username, a.config.Password, a.config.ServerName+":"+a.config.Port)
|
||||
}
|
||||
return a.Auth.Start(server)
|
||||
}
|
||||
|
||||
type loginAuth struct {
|
||||
username, password, host string
|
||||
}
|
||||
|
||||
func LoginAuth(username, password, host string) smtp.Auth {
|
||||
return &loginAuth{username, password, host}
|
||||
}
|
||||
|
||||
func (a *loginAuth) Start(server *smtp.ServerInfo) (string, []byte, error) {
|
||||
if !server.TLS {
|
||||
return "", nil, errors.New("unencrypted connection")
|
||||
}
|
||||
|
||||
if server.Name != a.host {
|
||||
return "", nil, errors.New("wrong host name")
|
||||
}
|
||||
|
||||
return "LOGIN", []byte{}, nil
|
||||
}
|
||||
|
||||
func (a *loginAuth) Next(fromServer []byte, more bool) ([]byte, error) {
|
||||
if more {
|
||||
switch string(fromServer) {
|
||||
case "Username:":
|
||||
return []byte(a.username), nil
|
||||
case "Password:":
|
||||
return []byte(a.password), nil
|
||||
default:
|
||||
return nil, errors.New("Unknown fromServer")
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func ConnectToSMTPServerAdvanced(config *SMTPConfig) (net.Conn, error) {
|
||||
var conn net.Conn
|
||||
var err error
|
||||
|
||||
smtpAddress := config.Server + ":" + config.Port
|
||||
dialer := &net.Dialer{
|
||||
Timeout: time.Duration(config.ServerTimeout) * time.Second,
|
||||
}
|
||||
|
||||
if config.ConnectionSecurity == TLS {
|
||||
tlsconfig := &tls.Config{
|
||||
InsecureSkipVerify: config.SkipServerCertificateVerification,
|
||||
ServerName: config.ServerName,
|
||||
}
|
||||
|
||||
conn, err = tls.DialWithDialer(dialer, "tcp", smtpAddress, tlsconfig)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "unable to connect to the SMTP server through TLS")
|
||||
}
|
||||
} else {
|
||||
conn, err = dialer.Dial("tcp", smtpAddress)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "unable to connect to the SMTP server")
|
||||
}
|
||||
}
|
||||
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
func ConnectToSMTPServer(config *SMTPConfig) (net.Conn, error) {
|
||||
return ConnectToSMTPServerAdvanced(config)
|
||||
}
|
||||
|
||||
func NewSMTPClientAdvanced(ctx context.Context, conn net.Conn, config *SMTPConfig) (*smtp.Client, error) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
var c *smtp.Client
|
||||
ec := make(chan error)
|
||||
go func() {
|
||||
var err error
|
||||
c, err = smtp.NewClient(conn, config.ServerName+":"+config.Port)
|
||||
if err != nil {
|
||||
ec <- err
|
||||
return
|
||||
}
|
||||
cancel()
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
err := ctx.Err()
|
||||
if err != nil && err.Error() != "context canceled" {
|
||||
return nil, errors.Wrap(err, "unable to connect to the SMTP server")
|
||||
}
|
||||
case err := <-ec:
|
||||
return nil, errors.Wrap(err, "unable to connect to the SMTP server")
|
||||
}
|
||||
|
||||
if config.Hostname != "" {
|
||||
err := c.Hello(config.Hostname)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "unable to send hello message")
|
||||
}
|
||||
}
|
||||
|
||||
if config.ConnectionSecurity == StartTLS {
|
||||
tlsconfig := &tls.Config{
|
||||
InsecureSkipVerify: config.SkipServerCertificateVerification,
|
||||
ServerName: config.ServerName,
|
||||
}
|
||||
c.StartTLS(tlsconfig)
|
||||
}
|
||||
|
||||
if config.EnableSMTPAuth {
|
||||
if err := c.Auth(&authChooser{config: config}); err != nil {
|
||||
return nil, errors.Wrap(err, "authentication failed")
|
||||
}
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func NewSMTPClient(ctx context.Context, conn net.Conn, config *SMTPConfig) (*smtp.Client, error) {
|
||||
return NewSMTPClientAdvanced(
|
||||
ctx,
|
||||
conn,
|
||||
config,
|
||||
)
|
||||
}
|
||||
|
||||
func TestConnection(config *SMTPConfig) error {
|
||||
conn, err := ConnectToSMTPServer(config)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "unable to connect")
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
sec := config.ServerTimeout
|
||||
|
||||
ctx := context.Background()
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Duration(sec)*time.Second)
|
||||
defer cancel()
|
||||
|
||||
c, err := NewSMTPClient(ctx, conn, config)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "unable to connect")
|
||||
}
|
||||
c.Close()
|
||||
c.Quit()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func SendMailWithEmbeddedFilesUsingConfig(to, subject, htmlBody string, embeddedFiles map[string]io.Reader, config *SMTPConfig, enableComplianceFeatures bool, messageID string, inReplyTo string, references string, ccMail string, category string) error {
|
||||
fromMail := mail.Address{Name: config.FeedbackName, Address: config.FeedbackEmail}
|
||||
replyTo := mail.Address{Name: config.FeedbackName, Address: config.ReplyToAddress}
|
||||
|
||||
mail := mailData{
|
||||
mimeTo: to,
|
||||
smtpTo: to,
|
||||
from: fromMail,
|
||||
cc: ccMail,
|
||||
replyTo: replyTo,
|
||||
subject: subject,
|
||||
htmlBody: htmlBody,
|
||||
embeddedFiles: embeddedFiles,
|
||||
messageID: messageID,
|
||||
inReplyTo: inReplyTo,
|
||||
references: references,
|
||||
category: category,
|
||||
}
|
||||
|
||||
return sendMailUsingConfigAdvanced(mail, config)
|
||||
}
|
||||
|
||||
func SendMailUsingConfig(to, subject, htmlBody string, config *SMTPConfig, enableComplianceFeatures bool, messageID string, inReplyTo string, references string, ccMail, category string) error {
|
||||
return SendMailWithEmbeddedFilesUsingConfig(to, subject, htmlBody, nil, config, enableComplianceFeatures, messageID, inReplyTo, references, ccMail, category)
|
||||
}
|
||||
|
||||
// allows for sending an email with differing MIME/SMTP recipients
|
||||
func sendMailUsingConfigAdvanced(mail mailData, config *SMTPConfig) error {
|
||||
if config.Server == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
conn, err := ConnectToSMTPServer(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
sec := config.ServerTimeout
|
||||
|
||||
ctx := context.Background()
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Duration(sec)*time.Second)
|
||||
defer cancel()
|
||||
|
||||
c, err := NewSMTPClient(ctx, conn, config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer c.Quit()
|
||||
defer c.Close()
|
||||
|
||||
return sendMail(c, mail, time.Now(), config)
|
||||
}
|
||||
|
||||
const SendGridXSMTPAPIHeader = "X-SMTPAPI"
|
||||
|
||||
func generateMessageID(hostname string) string {
|
||||
return fmt.Sprintf("<%s-%d@%s>", model.NewRandomString(16), time.Now().Unix(), hostname)
|
||||
}
|
||||
|
||||
// validateSingleAddress rejects a value that resolves to anything other than
|
||||
// exactly one address. This package only ever sends to a single recipient per
|
||||
// header; a multi-address value would otherwise be rejected deep inside the
|
||||
// SMTP library (or, worse, silently mishandled), so we fail early with a clear
|
||||
// error rather than adding multi-recipient support. The caller wraps the error
|
||||
// with the relevant header name.
|
||||
func validateSingleAddress(value string) error {
|
||||
addresses, err := mail.ParseAddressList(value)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to parse address")
|
||||
}
|
||||
if len(addresses) != 1 {
|
||||
return errors.Errorf("must contain exactly one address, got %d", len(addresses))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sendMail(c smtpClient, mail mailData, date time.Time, config *SMTPConfig) error {
|
||||
mlog.Info("sending mail", mlog.String("to", mail.smtpTo))
|
||||
|
||||
htmlMessage := mail.htmlBody
|
||||
text, err := html2text.FromString(htmlMessage)
|
||||
if err != nil {
|
||||
mlog.Warn("Unable to convert html body to text", mlog.Err(err))
|
||||
text = ""
|
||||
}
|
||||
|
||||
m := gomail.NewMsg()
|
||||
|
||||
m.SetAddrHeaderFromMailAddress(gomail.HeaderFrom, &mail.from)
|
||||
if err = validateSingleAddress(mail.mimeTo); err != nil {
|
||||
return errors.Wrap(err, "invalid To header")
|
||||
}
|
||||
if err = m.SetAddrHeader(gomail.HeaderTo, mail.mimeTo); err != nil {
|
||||
return errors.Wrap(err, "failed to set To address")
|
||||
}
|
||||
m.SetGenHeader(gomail.HeaderSubject, mail.subject)
|
||||
m.SetGenHeader(gomail.Header("Content-Transfer-Encoding"), "8bit")
|
||||
m.SetGenHeader(gomail.Header("Auto-Submitted"), "auto-generated")
|
||||
m.SetGenHeader(gomail.Header("Precedence"), "bulk")
|
||||
|
||||
if mail.category != "" {
|
||||
m.SetGenHeader(gomail.Header(SendGridXSMTPAPIHeader), fmt.Sprintf(`{"category": %q}`, mail.category))
|
||||
}
|
||||
|
||||
if mail.replyTo.Address != "" {
|
||||
m.SetAddrHeaderFromMailAddress(gomail.HeaderReplyTo, &mail.replyTo)
|
||||
}
|
||||
|
||||
if mail.cc != "" {
|
||||
if err = validateSingleAddress(mail.cc); err != nil {
|
||||
return errors.Wrap(err, "invalid Cc header")
|
||||
}
|
||||
if err = m.SetAddrHeader(gomail.HeaderCc, mail.cc); err != nil {
|
||||
return errors.Wrap(err, "failed to set Cc address")
|
||||
}
|
||||
}
|
||||
|
||||
msgID := mail.messageID
|
||||
if msgID == "" {
|
||||
msgID = generateMessageID(config.Hostname)
|
||||
}
|
||||
// Must be SetGenHeader, not SetGenHeaderPreformatted: only genHeader suppresses go-mail's
|
||||
// auto-generated Message-ID, otherwise we emit two and strict servers reject with a 554.
|
||||
m.SetGenHeader(gomail.HeaderMessageID, msgID)
|
||||
|
||||
if mail.inReplyTo != "" {
|
||||
m.SetGenHeaderPreformatted(gomail.HeaderInReplyTo, mail.inReplyTo)
|
||||
}
|
||||
|
||||
if mail.references != "" {
|
||||
m.SetGenHeaderPreformatted(gomail.HeaderReferences, mail.references)
|
||||
}
|
||||
|
||||
for k, v := range mail.mimeHeaders {
|
||||
m.SetGenHeader(gomail.Header(k), v)
|
||||
}
|
||||
|
||||
m.SetDateWithValue(date)
|
||||
m.SetBodyString(gomail.TypeTextPlain, text)
|
||||
m.AddAlternativeString(gomail.TypeTextHTML, htmlMessage)
|
||||
|
||||
for name, reader := range mail.embeddedFiles {
|
||||
if err = m.EmbedReader(name, reader); err != nil {
|
||||
return errors.Wrap(err, fmt.Sprintf("failed to embed file %q", name))
|
||||
}
|
||||
}
|
||||
|
||||
if err = c.Mail(mail.from.Address); err != nil {
|
||||
return errors.Wrap(err, "failed to set the from address")
|
||||
}
|
||||
|
||||
if err = c.Rcpt(mail.smtpTo); err != nil {
|
||||
return errors.Wrap(err, "failed to set the to address")
|
||||
}
|
||||
|
||||
w, err := c.Data()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to add email message data")
|
||||
}
|
||||
|
||||
_, err = m.WriteTo(w)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to write the email message")
|
||||
}
|
||||
err = w.Close()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to close connection to the SMTP server")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
895
server/platform/shared/mail/mail_test.go
Normal file
895
server/platform/shared/mail/mail_test.go
Normal file
@@ -0,0 +1,895 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package mail
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"net"
|
||||
"net/mail"
|
||||
"net/smtp"
|
||||
"net/textproto"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const embeddedFileMinSize = 1500
|
||||
|
||||
func getConfig() *SMTPConfig {
|
||||
server := os.Getenv("MM_EMAILSETTINGS_SMTPSERVER")
|
||||
if server == "" {
|
||||
server = "localhost"
|
||||
}
|
||||
port := os.Getenv("MM_EMAILSETTINGS_SMTPPORT")
|
||||
if port == "" {
|
||||
port = "10025"
|
||||
}
|
||||
|
||||
return &SMTPConfig{
|
||||
ConnectionSecurity: "",
|
||||
SkipServerCertificateVerification: false,
|
||||
Hostname: "localhost",
|
||||
ServerName: server,
|
||||
Server: server,
|
||||
Port: port,
|
||||
ServerTimeout: 10,
|
||||
Username: "",
|
||||
Password: "",
|
||||
EnableSMTPAuth: false,
|
||||
SendEmailNotifications: true,
|
||||
FeedbackName: "",
|
||||
FeedbackEmail: "test@example.com",
|
||||
ReplyToAddress: "test@example.com",
|
||||
}
|
||||
}
|
||||
|
||||
func TestMailConnectionFromConfig(t *testing.T) {
|
||||
cfg := getConfig()
|
||||
|
||||
conn, err := ConnectToSMTPServer(cfg)
|
||||
require.NoError(t, err, "Should connect to the SMTP Server %v", err)
|
||||
|
||||
_, err = NewSMTPClient(context.Background(), conn, cfg)
|
||||
|
||||
require.NoError(t, err, "Should get new SMTP client")
|
||||
|
||||
cfg.Server = "wrongServer"
|
||||
cfg.Port = "553"
|
||||
|
||||
_, err = ConnectToSMTPServer(cfg)
|
||||
|
||||
require.Error(t, err, "Should not connect to the SMTP Server")
|
||||
}
|
||||
|
||||
func TestMailConnectionAdvanced(t *testing.T) {
|
||||
cfg := getConfig()
|
||||
|
||||
conn, err := ConnectToSMTPServerAdvanced(cfg)
|
||||
require.NoError(t, err, "Should connect to the SMTP Server")
|
||||
defer conn.Close()
|
||||
|
||||
_, err2 := NewSMTPClientAdvanced(context.Background(), conn, cfg)
|
||||
require.NoError(t, err2, "Should get new SMTP client")
|
||||
|
||||
l, err3 := net.Listen("tcp", "localhost:") // emulate nc -l <random-port>
|
||||
require.NoError(t, err3, "Should've open a network socket and listen")
|
||||
defer l.Close()
|
||||
|
||||
cfg = getConfig()
|
||||
cfg.Server = strings.Split(l.Addr().String(), ":")[0]
|
||||
cfg.Port = strings.Split(l.Addr().String(), ":")[1]
|
||||
cfg.ServerTimeout = 1
|
||||
|
||||
conn2, err := ConnectToSMTPServerAdvanced(cfg)
|
||||
require.NoError(t, err, "Should connect to the SMTP Server")
|
||||
defer conn2.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Second)
|
||||
defer cancel()
|
||||
|
||||
cfg = getConfig()
|
||||
cfg.Server = strings.Split(l.Addr().String(), ":")[0]
|
||||
cfg.Port = strings.Split(l.Addr().String(), ":")[1]
|
||||
cfg.ServerTimeout = 1
|
||||
_, err4 := NewSMTPClientAdvanced(
|
||||
ctx,
|
||||
conn2,
|
||||
cfg,
|
||||
)
|
||||
require.Error(t, err4, "Should get a timeout get while creating a new SMTP client")
|
||||
assert.Contains(t, err4.Error(), "unable to connect to the SMTP server")
|
||||
|
||||
cfg = getConfig()
|
||||
cfg.Server = "wrongServer"
|
||||
cfg.Port = "553"
|
||||
cfg.ServerTimeout = 1
|
||||
|
||||
_, err5 := ConnectToSMTPServerAdvanced(cfg)
|
||||
require.Error(t, err5, "Should not connect to the SMTP Server")
|
||||
}
|
||||
|
||||
func TestSendMailUsingConfig(t *testing.T) {
|
||||
cfg := getConfig()
|
||||
|
||||
emailTo := "test@example.com"
|
||||
emailSubject := "Testing this email"
|
||||
emailBody := "This is a test from autobot"
|
||||
emailCC := "test@example.com"
|
||||
|
||||
// Delete all the messages before check the sample email
|
||||
DeleteMailBox(emailTo)
|
||||
|
||||
err2 := SendMailUsingConfig(emailTo, emailSubject, emailBody, cfg, true, "", "", "", emailCC, "")
|
||||
require.NoError(t, err2, "Should connect to the SMTP Server")
|
||||
|
||||
// Check if the email was send to the right email address
|
||||
var resultsMailbox JSONMessageHeaderInbucket
|
||||
err3 := RetryInbucket(5, func() error {
|
||||
var err error
|
||||
resultsMailbox, err = GetMailBox(emailTo)
|
||||
return err
|
||||
})
|
||||
if err3 != nil {
|
||||
t.Log(err3)
|
||||
t.Log("No email was received, maybe due load on the server. Skipping this verification")
|
||||
} else {
|
||||
if len(resultsMailbox) > 0 {
|
||||
require.Contains(t, resultsMailbox[0].To[0], emailTo, "Wrong To: recipient")
|
||||
resultsEmail, err := GetMessageFromMailbox(emailTo, resultsMailbox[0].ID)
|
||||
require.NoError(t, err, "Could not get message from mailbox")
|
||||
require.Contains(t, emailBody, resultsEmail.Body.Text, "Wrong received message %s", resultsEmail.Body.Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendMailPlainText(t *testing.T) {
|
||||
cfg := getConfig()
|
||||
emailSubject := "Testing this email"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
emailBodyHTML string
|
||||
expectedBodyText string
|
||||
}{
|
||||
{
|
||||
name: "Heading",
|
||||
emailBodyHTML: "<h1>This is a test from autobot</h1><h2>This is a subheading</h2>",
|
||||
expectedBodyText: "***************************\nThis is a test from autobot\n***************************\n\n--------------------\nThis is a subheading\n--------------------",
|
||||
},
|
||||
{
|
||||
name: "List",
|
||||
emailBodyHTML: "<ul><li>Item 1</li><li>Item 2</li></ul>",
|
||||
expectedBodyText: "* Item 1\n* Item 2",
|
||||
},
|
||||
{
|
||||
name: "Inline formatting",
|
||||
emailBodyHTML: "<p><strong>Strong</strong> and <a href='https://example.com'>link</a>",
|
||||
expectedBodyText: "*Strong* and link ( https://example.com )",
|
||||
},
|
||||
{
|
||||
// Regression for MM-69721: a link-only body previously yielded an
|
||||
// empty text/plain part (docconv readability returned "MAIN: P is nil").
|
||||
name: "Link-only body",
|
||||
emailBodyHTML: "<a href='https://example.com/reset'>Reset your password</a>",
|
||||
expectedBodyText: "Reset your password ( https://example.com/reset )",
|
||||
},
|
||||
{
|
||||
// Regression for MM-69721: templated transactional emails previously
|
||||
// yielded an empty text/plain part (docconv readability extracted no paragraphs).
|
||||
name: "Templated email",
|
||||
emailBodyHTML: "<h1>Reset Your Password</h1><p>Click the link below to reset your password.</p><p><a href='https://example.com/reset?token=abc'>Reset Password</a></p>",
|
||||
expectedBodyText: "Reset Password ( https://example.com/reset?token=abc )",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
emailTo := fmt.Sprintf("plain-text-%d@example.com", time.Now().UnixNano())
|
||||
emailCC := emailTo
|
||||
DeleteMailBox(emailTo)
|
||||
|
||||
err := SendMailUsingConfig(emailTo, emailSubject, test.emailBodyHTML, cfg, true, "", "", "", emailCC, "")
|
||||
require.NoError(t, err, "Should connect to the SMTP Server")
|
||||
|
||||
var resultsMailbox JSONMessageHeaderInbucket
|
||||
err = RetryInbucket(5, func() error {
|
||||
var err2 error
|
||||
resultsMailbox, err2 = GetMailBox(emailTo)
|
||||
return err2
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Log("No email was received, maybe due load on the server. Failing this test")
|
||||
t.Error(err)
|
||||
} else {
|
||||
require.NotEmpty(t, resultsMailbox, "Mailbox should contain at least one message")
|
||||
require.Contains(t, resultsMailbox[0].To[0], emailTo, "Wrong To: recipient")
|
||||
resultsEmail, err := GetMessageFromMailbox(emailTo, resultsMailbox[0].ID)
|
||||
require.NoError(t, err, "Could not get message from mailbox")
|
||||
require.Contains(t, test.emailBodyHTML, resultsEmail.Body.HTML, "Wrong received message %s", resultsEmail.Body.Text)
|
||||
require.NotEmpty(t, resultsEmail.Body.Text, "text/plain part should not be empty (MM-69721)")
|
||||
require.Contains(t, resultsEmail.Body.Text, test.expectedBodyText, "Wrong message plain text conversion %s", resultsEmail.Body.Text)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendMailWithEmbeddedFilesUsingConfig(t *testing.T) {
|
||||
cfg := getConfig()
|
||||
|
||||
emailTo := fmt.Sprintf("embedded-files-%d@example.com", time.Now().UnixNano())
|
||||
emailSubject := "Testing this email"
|
||||
emailBody := "This is a test from autobot"
|
||||
emailCC := emailTo
|
||||
|
||||
// Delete all the messages before check the sample email
|
||||
DeleteMailBox(emailTo)
|
||||
|
||||
embeddedFiles := map[string]io.Reader{
|
||||
"test1.png": bytes.NewReader([]byte("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")),
|
||||
"test2.png": bytes.NewReader([]byte("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")),
|
||||
}
|
||||
err2 := SendMailWithEmbeddedFilesUsingConfig(emailTo, emailSubject, emailBody, embeddedFiles, cfg, true, "", "", "", emailCC, "")
|
||||
require.NoError(t, err2, "Should connect to the SMTP Server")
|
||||
|
||||
// Check if the email was send to the right email address
|
||||
var resultsMailbox JSONMessageHeaderInbucket
|
||||
var resultsEmail JSONMessageInbucket
|
||||
err3 := RetryInbucket(10, func() error {
|
||||
var err error
|
||||
resultsMailbox, err = GetMailBox(emailTo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(resultsMailbox) == 0 {
|
||||
return fmt.Errorf("no messages in mailbox")
|
||||
}
|
||||
|
||||
resultsEmail, err = GetMessageFromMailbox(emailTo, resultsMailbox[0].ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resultsEmail.Size <= embeddedFileMinSize {
|
||||
return fmt.Errorf("message size %d does not yet reflect embedded attachments", resultsEmail.Size)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err3 != nil {
|
||||
t.Log(err3)
|
||||
t.Log("No email was received, maybe due load on the server. Skipping this verification")
|
||||
} else {
|
||||
if len(resultsMailbox) > 0 {
|
||||
require.Contains(t, resultsMailbox[0].To[0], emailTo, "Wrong To: recipient")
|
||||
require.Contains(t, emailBody, resultsEmail.Body.Text, "Wrong received message %s", resultsEmail.Body.Text)
|
||||
// Usign the message size because the inbucket API doesn't return embedded attachments through the API
|
||||
require.Greater(t, resultsEmail.Size, embeddedFileMinSize, "the file size should be more because the embedded attachments")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendMailUsingConfigAdvanced(t *testing.T) {
|
||||
cfg := getConfig()
|
||||
|
||||
// Delete all the messages before check the sample email
|
||||
DeleteMailBox("test2@example.com")
|
||||
|
||||
// create two files with the same name that will both be attached to the email
|
||||
file1, err := os.CreateTemp("", "*")
|
||||
require.NoError(t, err)
|
||||
defer os.Remove(file1.Name())
|
||||
file1.WriteString("hello world")
|
||||
file1.Close()
|
||||
file2, err := os.CreateTemp("", "*")
|
||||
|
||||
require.NoError(t, err)
|
||||
defer os.Remove(file2.Name())
|
||||
file2.WriteString("foo bar")
|
||||
file2.Close()
|
||||
|
||||
embeddedFiles := map[string]io.Reader{
|
||||
"test": bytes.NewReader([]byte("test data")),
|
||||
}
|
||||
|
||||
headers := make(map[string]string)
|
||||
headers["TestHeader"] = "TestValue"
|
||||
|
||||
md := mailData{
|
||||
mimeTo: "test@example.com",
|
||||
smtpTo: "test2@example.com",
|
||||
from: mail.Address{Name: "Nobody", Address: "nobody@mattermost.com"},
|
||||
replyTo: mail.Address{Name: "ReplyTo", Address: "reply_to@mattermost.com"},
|
||||
subject: "Testing this email",
|
||||
htmlBody: "This is a test from autobot",
|
||||
embeddedFiles: embeddedFiles,
|
||||
mimeHeaders: headers,
|
||||
}
|
||||
|
||||
err = sendMailUsingConfigAdvanced(md, cfg)
|
||||
require.NoError(t, err, "Should connect to the SMTP Server: %v", err)
|
||||
|
||||
// Check if the email was send to the right email address
|
||||
var resultsMailbox JSONMessageHeaderInbucket
|
||||
err = RetryInbucket(5, func() error {
|
||||
var mailErr error
|
||||
resultsMailbox, mailErr = GetMailBox(md.smtpTo)
|
||||
return mailErr
|
||||
})
|
||||
require.NoError(t, err, "No emails found for address %s. error: %v", md.smtpTo, err)
|
||||
require.NotEqual(t, len(resultsMailbox), 0)
|
||||
|
||||
require.Contains(t, resultsMailbox[0].To[0], md.mimeTo, "Wrong To recipient")
|
||||
|
||||
resultsEmail, err := GetMessageFromMailbox(md.smtpTo, resultsMailbox[0].ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Contains(t, md.htmlBody, resultsEmail.Body.Text, "Wrong received message")
|
||||
|
||||
// verify that the To header of the email message is set to the MIME recipient, even though we got it out of the SMTP recipient's email inbox
|
||||
require.NotEmpty(t, resultsEmail.Header["To"], "missing To header")
|
||||
resultEmail, err := mail.ParseAddress(resultsEmail.Header["To"][0])
|
||||
require.NoError(t, err, "failed to parse To header")
|
||||
assert.Equal(t, md.mimeTo, resultEmail.Address)
|
||||
|
||||
// verify that the MIME from address is correct - unfortunately, we can't verify the SMTP from address
|
||||
assert.Equal(t, md.from.String(), resultsEmail.Header["From"][0])
|
||||
|
||||
// check that the custom mime headers came through - header case seems to get mutated
|
||||
assert.Equal(t, "TestValue", resultsEmail.Header["Testheader"][0])
|
||||
}
|
||||
|
||||
func TestAuthMethods(t *testing.T) {
|
||||
auth := &authChooser{
|
||||
config: &SMTPConfig{
|
||||
Username: "test",
|
||||
Password: "fakepass",
|
||||
ServerName: "fakeserver",
|
||||
Server: "fakeserver",
|
||||
Port: "25",
|
||||
},
|
||||
}
|
||||
tests := []struct {
|
||||
desc string
|
||||
server *smtp.ServerInfo
|
||||
err string
|
||||
}{
|
||||
{
|
||||
desc: "auth PLAIN success",
|
||||
server: &smtp.ServerInfo{Name: "fakeserver:25", Auth: []string{"PLAIN"}, TLS: true},
|
||||
},
|
||||
{
|
||||
desc: "auth PLAIN unencrypted connection fail",
|
||||
server: &smtp.ServerInfo{Name: "fakeserver:25", Auth: []string{"PLAIN"}, TLS: false},
|
||||
err: "unencrypted connection",
|
||||
},
|
||||
{
|
||||
desc: "auth PLAIN wrong host name",
|
||||
server: &smtp.ServerInfo{Name: "wrongServer:999", Auth: []string{"PLAIN"}, TLS: true},
|
||||
err: "wrong host name",
|
||||
},
|
||||
{
|
||||
desc: "auth LOGIN success",
|
||||
server: &smtp.ServerInfo{Name: "fakeserver:25", Auth: []string{"LOGIN"}, TLS: true},
|
||||
},
|
||||
{
|
||||
desc: "auth LOGIN unencrypted connection fail",
|
||||
server: &smtp.ServerInfo{Name: "wrongServer:999", Auth: []string{"LOGIN"}, TLS: true},
|
||||
err: "wrong host name",
|
||||
},
|
||||
{
|
||||
desc: "auth LOGIN wrong host name",
|
||||
server: &smtp.ServerInfo{Name: "fakeserver:25", Auth: []string{"LOGIN"}, TLS: false},
|
||||
err: "unencrypted connection",
|
||||
},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
_, _, err := auth.Start(test.server)
|
||||
got := ""
|
||||
if err != nil {
|
||||
got = err.Error()
|
||||
}
|
||||
assert.True(t, got == test.err, "%d. got error = %q; want %q", i, got, test.err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginAuthNext(t *testing.T) {
|
||||
auth := &loginAuth{username: "user", password: "pass", host: "host:25"}
|
||||
|
||||
resp, err := auth.Next([]byte("Username:"), true)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []byte("user"), resp)
|
||||
|
||||
resp, err = auth.Next([]byte("Password:"), true)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []byte("pass"), resp)
|
||||
|
||||
_, err = auth.Next([]byte("Unknown:"), true)
|
||||
require.Error(t, err)
|
||||
|
||||
resp, err = auth.Next(nil, false)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, resp)
|
||||
}
|
||||
|
||||
type errReader struct {
|
||||
err error
|
||||
payload []byte
|
||||
}
|
||||
|
||||
func (r *errReader) Read(p []byte) (int, error) {
|
||||
if len(r.payload) > 0 {
|
||||
n := copy(p, r.payload)
|
||||
r.payload = r.payload[n:]
|
||||
return n, nil
|
||||
}
|
||||
return 0, r.err
|
||||
}
|
||||
|
||||
func TestSendMailEmbedReaderError(t *testing.T) {
|
||||
mocm := &mockMailer{}
|
||||
m := mailData{
|
||||
mimeTo: "test@example.com",
|
||||
smtpTo: "test@example.com",
|
||||
from: mail.Address{Address: "from@example.com"},
|
||||
embeddedFiles: map[string]io.Reader{
|
||||
"attachment.txt": &errReader{payload: []byte("partial data"), err: errors.New("read failure")},
|
||||
},
|
||||
}
|
||||
err := sendMail(mocm, m, time.Now(), getConfig())
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "failed to embed file")
|
||||
}
|
||||
|
||||
func TestSendMailRejectsMultipleAddresses(t *testing.T) {
|
||||
multiAddress := []string{
|
||||
"a@example.com, b@example.com",
|
||||
"a@example.com,b@example.com",
|
||||
`"A" <a@example.com>, "B" <b@example.com>`,
|
||||
}
|
||||
|
||||
t.Run("rejects multiple To addresses", func(t *testing.T) {
|
||||
for _, to := range multiAddress {
|
||||
mocm := &mockMailer{}
|
||||
m := mailData{
|
||||
mimeTo: to,
|
||||
smtpTo: to,
|
||||
from: mail.Address{Address: "from@example.com"},
|
||||
}
|
||||
err := sendMail(mocm, m, time.Now(), getConfig())
|
||||
require.Error(t, err, "expected %q to be rejected", to)
|
||||
assert.Contains(t, err.Error(), "invalid To header: must contain exactly one address")
|
||||
assert.Empty(t, mocm.data, "no message should be written when the To address is rejected")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejects multiple Cc addresses", func(t *testing.T) {
|
||||
for _, cc := range multiAddress {
|
||||
mocm := &mockMailer{}
|
||||
m := mailData{
|
||||
mimeTo: "test@example.com",
|
||||
smtpTo: "test@example.com",
|
||||
from: mail.Address{Address: "from@example.com"},
|
||||
cc: cc,
|
||||
}
|
||||
err := sendMail(mocm, m, time.Now(), getConfig())
|
||||
require.Error(t, err, "expected %q to be rejected", cc)
|
||||
assert.Contains(t, err.Error(), "invalid Cc header: must contain exactly one address")
|
||||
assert.Empty(t, mocm.data, "no message should be written when the Cc address is rejected")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("accepts a single To and Cc address", func(t *testing.T) {
|
||||
mocm := &mockMailer{}
|
||||
m := mailData{
|
||||
mimeTo: "test@example.com",
|
||||
smtpTo: "test@example.com",
|
||||
from: mail.Address{Address: "from@example.com"},
|
||||
cc: "cc@example.com",
|
||||
htmlBody: "<p>hello</p>",
|
||||
}
|
||||
err := sendMail(mocm, m, time.Now(), getConfig())
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, mocm.data)
|
||||
})
|
||||
}
|
||||
|
||||
type mockMailer struct {
|
||||
data []byte
|
||||
}
|
||||
|
||||
func (m *mockMailer) Mail(string) error { return nil }
|
||||
func (m *mockMailer) Rcpt(string) error { return nil }
|
||||
func (m *mockMailer) Data() (io.WriteCloser, error) { return m, nil }
|
||||
func (m *mockMailer) Write(p []byte) (int, error) {
|
||||
m.data = append(m.data, p...)
|
||||
return len(p), nil
|
||||
}
|
||||
func (m *mockMailer) Close() error { return nil }
|
||||
|
||||
func TestSendMail(t *testing.T) {
|
||||
dir, err := os.MkdirTemp(".", "mail-test-")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(dir)
|
||||
mocm := &mockMailer{}
|
||||
|
||||
testCases := map[string]struct {
|
||||
replyTo mail.Address
|
||||
messageID string
|
||||
inReplyTo string
|
||||
references string
|
||||
contains string
|
||||
notContains string
|
||||
}{
|
||||
"adds reply-to header": {
|
||||
mail.Address{Address: "foo@test.com"},
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"\r\nReply-To: <foo@test.com>\r\n",
|
||||
"",
|
||||
},
|
||||
"doesn't add reply-to header": {
|
||||
mail.Address{},
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"\r\nReply-To:",
|
||||
},
|
||||
|
||||
"adds message-id header": {
|
||||
mail.Address{},
|
||||
"<abc123@mattermost.com>",
|
||||
"",
|
||||
"",
|
||||
"\r\nMessage-ID: <abc123@mattermost.com>\r\n",
|
||||
"",
|
||||
},
|
||||
"always adds message-id header": {
|
||||
mail.Address{},
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"\r\nMessage-ID: <",
|
||||
"",
|
||||
},
|
||||
"adds in-reply-to header": {
|
||||
mail.Address{},
|
||||
"",
|
||||
"<defg456@mattermost.com>",
|
||||
"",
|
||||
"\r\nIn-Reply-To: <defg456@mattermost.com>\r\n",
|
||||
"",
|
||||
},
|
||||
"doesn't add in-reply-to header": {
|
||||
mail.Address{},
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"\r\nIn-Reply-To:",
|
||||
},
|
||||
"adds references header": {
|
||||
mail.Address{},
|
||||
"",
|
||||
"",
|
||||
"<ghi789@mattermost.com>",
|
||||
"\r\nReferences: <ghi789@mattermost.com>\r\n",
|
||||
"",
|
||||
},
|
||||
"doesn't add references header": {
|
||||
mail.Address{},
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"\r\nReferences:",
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("sets exactly one message-id header when provided", func(t *testing.T) {
|
||||
m := mailData{
|
||||
mimeTo: "test@example.com",
|
||||
smtpTo: "test@example.com",
|
||||
from: mail.Address{Address: "from@example.com"},
|
||||
messageID: "<abc123@mattermost.com>",
|
||||
}
|
||||
err = sendMail(mocm, m, time.Now(), getConfig())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, strings.Count(string(mocm.data), "Message-ID:"), "expected exactly one Message-ID header, got:\n%s", string(mocm.data))
|
||||
assert.Contains(t, string(mocm.data), "\r\nMessage-ID: <abc123@mattermost.com>\r\n")
|
||||
mocm.data = []byte{}
|
||||
})
|
||||
|
||||
t.Run("sets exactly one message-id header when generated", func(t *testing.T) {
|
||||
m := mailData{
|
||||
mimeTo: "test@example.com",
|
||||
smtpTo: "test@example.com",
|
||||
from: mail.Address{Address: "from@example.com"},
|
||||
}
|
||||
err = sendMail(mocm, m, time.Now(), getConfig())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, strings.Count(string(mocm.data), "Message-ID:"), "expected exactly one Message-ID header, got:\n%s", string(mocm.data))
|
||||
mocm.data = []byte{}
|
||||
})
|
||||
|
||||
t.Run("adds cc header", func(t *testing.T) {
|
||||
m := mailData{
|
||||
mimeTo: "test@example.com",
|
||||
smtpTo: "test@example.com",
|
||||
from: mail.Address{Address: "from@example.com"},
|
||||
cc: "cc@example.com",
|
||||
}
|
||||
err = sendMail(mocm, m, time.Now(), getConfig())
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, string(mocm.data), "\r\nCc: <cc@example.com>\r\n")
|
||||
mocm.data = []byte{}
|
||||
})
|
||||
|
||||
t.Run("adds sendgrid category header", func(t *testing.T) {
|
||||
m := mailData{
|
||||
mimeTo: "test@example.com",
|
||||
smtpTo: "test@example.com",
|
||||
from: mail.Address{Address: "from@example.com"},
|
||||
category: "transactional",
|
||||
}
|
||||
err = sendMail(mocm, m, time.Now(), getConfig())
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, string(mocm.data), SendGridXSMTPAPIHeader)
|
||||
require.Contains(t, string(mocm.data), `"transactional"`)
|
||||
mocm.data = []byte{}
|
||||
})
|
||||
|
||||
for testName, tc := range testCases {
|
||||
t.Run(testName, func(t *testing.T) {
|
||||
md := mailData{"test@example.com", "test@example.com", mail.Address{Address: "from@example.com"}, "", tc.replyTo, "", "", nil, nil, tc.messageID, tc.inReplyTo, tc.references, ""}
|
||||
cfg := getConfig()
|
||||
err = sendMail(mocm, md, time.Now(), cfg)
|
||||
require.NoError(t, err)
|
||||
if tc.contains != "" {
|
||||
require.Contains(t, string(mocm.data), tc.contains)
|
||||
}
|
||||
if tc.notContains != "" {
|
||||
require.NotContains(t, string(mocm.data), tc.notContains)
|
||||
}
|
||||
mocm.data = []byte{}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// mailHeaderValue returns the value of the named header (case-insensitive) from
|
||||
// the raw RFC 5322 message, unfolding any continuation lines. It fails the test
|
||||
// if the message is not parseable, and returns "" and false if the header is
|
||||
// absent.
|
||||
func mailHeaderValue(t *testing.T, raw, name string) (string, bool) {
|
||||
t.Helper()
|
||||
msg, err := mail.ReadMessage(strings.NewReader(raw))
|
||||
require.NoError(t, err, "message must be RFC 5322 parseable")
|
||||
if _, ok := msg.Header[textproto.CanonicalMIMEHeaderKey(name)]; !ok {
|
||||
return "", false
|
||||
}
|
||||
return msg.Header.Get(name), true
|
||||
}
|
||||
|
||||
// TestSendMailSubjectEncoding verifies that a non-ASCII subject is emitted as
|
||||
// an RFC 2047 encoded-word, never as raw UTF-8 bytes in the header. A library
|
||||
// swap that stopped encoding the subject would corrupt it on the wire.
|
||||
func TestSendMailSubjectEncoding(t *testing.T) {
|
||||
mocm := &mockMailer{}
|
||||
m := mailData{
|
||||
mimeTo: "test@example.com",
|
||||
smtpTo: "test@example.com",
|
||||
from: mail.Address{Address: "from@example.com"},
|
||||
subject: "Café ☕ 你好",
|
||||
htmlBody: "hi",
|
||||
}
|
||||
err := sendMail(mocm, m, time.Now(), getConfig())
|
||||
require.NoError(t, err)
|
||||
|
||||
subject, ok := mailHeaderValue(t, string(mocm.data), "Subject")
|
||||
require.True(t, ok, "Subject header must be present")
|
||||
|
||||
// The value must be an RFC 2047 encoded-word, not the raw UTF-8 string.
|
||||
assert.Contains(t, strings.ToLower(subject), "=?utf-8?", "Subject must be RFC 2047 encoded")
|
||||
assert.NotContains(t, subject, "Café", "raw UTF-8 subject must not leak into the header")
|
||||
assert.NotContains(t, subject, "你好", "raw UTF-8 subject must not leak into the header")
|
||||
|
||||
// Decoding the encoded-word(s) must round-trip back to the original subject,
|
||||
// without locking in the exact encoding go-mail happens to emit.
|
||||
decoded, err := new(mime.WordDecoder).DecodeHeader(subject)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, m.subject, decoded)
|
||||
}
|
||||
|
||||
// TestSendMailMIMEStructure verifies the multipart container layout: with text
|
||||
// and HTML bodies a multipart/alternative is produced, and with an embedded
|
||||
// file the whole thing is wrapped in a multipart/related whose embedded part
|
||||
// carries a Content-ID and inline disposition.
|
||||
func TestSendMailMIMEStructure(t *testing.T) {
|
||||
t.Run("text and html without embed produce multipart/alternative", func(t *testing.T) {
|
||||
mocm := &mockMailer{}
|
||||
m := mailData{
|
||||
mimeTo: "test@example.com",
|
||||
smtpTo: "test@example.com",
|
||||
from: mail.Address{Address: "from@example.com"},
|
||||
htmlBody: "<p>Hello <b>world</b></p>",
|
||||
}
|
||||
err := sendMail(mocm, m, time.Now(), getConfig())
|
||||
require.NoError(t, err)
|
||||
|
||||
data := string(mocm.data)
|
||||
ct, ok := mailHeaderValue(t, data, "Content-Type")
|
||||
require.True(t, ok, "Content-Type header must be present")
|
||||
assert.Contains(t, ct, "multipart/alternative", "top-level container should be multipart/alternative")
|
||||
assert.NotContains(t, ct, "multipart/related", "no embedded files means no multipart/related wrapper")
|
||||
assert.Contains(t, data, "text/plain", "plain text part must be present")
|
||||
assert.Contains(t, data, "text/html", "html part must be present")
|
||||
})
|
||||
|
||||
t.Run("embedded file produces multipart/related with inline Content-ID part", func(t *testing.T) {
|
||||
mocm := &mockMailer{}
|
||||
m := mailData{
|
||||
mimeTo: "test@example.com",
|
||||
smtpTo: "test@example.com",
|
||||
from: mail.Address{Address: "from@example.com"},
|
||||
htmlBody: "<p>Hello <b>world</b></p>",
|
||||
embeddedFiles: map[string]io.Reader{
|
||||
"logo.png": bytes.NewReader([]byte("PNGDATA")),
|
||||
},
|
||||
}
|
||||
err := sendMail(mocm, m, time.Now(), getConfig())
|
||||
require.NoError(t, err)
|
||||
|
||||
data := string(mocm.data)
|
||||
ct, ok := mailHeaderValue(t, data, "Content-Type")
|
||||
require.True(t, ok, "Content-Type header must be present")
|
||||
assert.Contains(t, ct, "multipart/related", "an embedded file must be wrapped in multipart/related")
|
||||
|
||||
// Both alternatives still exist inside the related container.
|
||||
assert.Contains(t, data, "multipart/alternative", "multipart/alternative must remain nested inside multipart/related")
|
||||
assert.Contains(t, data, "text/plain")
|
||||
assert.Contains(t, data, "text/html")
|
||||
|
||||
lower := strings.ToLower(data)
|
||||
assert.Contains(t, lower, "content-id: <logo.png>", "embedded part must carry a Content-ID")
|
||||
assert.Contains(t, lower, "content-disposition: inline", "embedded part must be inline")
|
||||
assert.Contains(t, data, `filename="logo.png"`, "embedded part must reference its filename")
|
||||
})
|
||||
}
|
||||
|
||||
// TestSendMailDateHeader verifies a Date header is present and RFC 5322 shaped
|
||||
// (parseable by net/mail). SetDateWithValue regressions would break this.
|
||||
func TestSendMailDateHeader(t *testing.T) {
|
||||
mocm := &mockMailer{}
|
||||
date := time.Date(2026, 7, 6, 12, 0, 0, 0, time.UTC)
|
||||
m := mailData{
|
||||
mimeTo: "test@example.com",
|
||||
smtpTo: "test@example.com",
|
||||
from: mail.Address{Address: "from@example.com"},
|
||||
htmlBody: "hi",
|
||||
}
|
||||
err := sendMail(mocm, m, date, getConfig())
|
||||
require.NoError(t, err)
|
||||
|
||||
value, ok := mailHeaderValue(t, string(mocm.data), "Date")
|
||||
require.True(t, ok, "Date header must be present")
|
||||
|
||||
parsed, err := mail.ParseDate(value)
|
||||
require.NoError(t, err, "Date header %q must be RFC 5322 parseable", value)
|
||||
assert.True(t, parsed.Equal(date), "parsed Date %s should match the supplied time %s", parsed, date)
|
||||
}
|
||||
|
||||
// TestSendMailComplianceHeaders verifies the bulk/auto-generated headers used to
|
||||
// signal automated mail are present.
|
||||
func TestSendMailComplianceHeaders(t *testing.T) {
|
||||
mocm := &mockMailer{}
|
||||
m := mailData{
|
||||
mimeTo: "test@example.com",
|
||||
smtpTo: "test@example.com",
|
||||
from: mail.Address{Address: "from@example.com"},
|
||||
htmlBody: "hi",
|
||||
}
|
||||
err := sendMail(mocm, m, time.Now(), getConfig())
|
||||
require.NoError(t, err)
|
||||
|
||||
autoSubmitted, ok := mailHeaderValue(t, string(mocm.data), "Auto-Submitted")
|
||||
require.True(t, ok, "Auto-Submitted header must be present")
|
||||
assert.Equal(t, "auto-generated", autoSubmitted)
|
||||
|
||||
precedence, ok := mailHeaderValue(t, string(mocm.data), "Precedence")
|
||||
require.True(t, ok, "Precedence header must be present")
|
||||
assert.Equal(t, "bulk", precedence)
|
||||
}
|
||||
|
||||
// TestSendMailBodyCharset verifies the body parts declare a UTF-8 charset,
|
||||
// tolerant of casing and quoting differences between libraries.
|
||||
func TestSendMailBodyCharset(t *testing.T) {
|
||||
mocm := &mockMailer{}
|
||||
m := mailData{
|
||||
mimeTo: "test@example.com",
|
||||
smtpTo: "test@example.com",
|
||||
from: mail.Address{Address: "from@example.com"},
|
||||
htmlBody: "<p>hi</p>",
|
||||
}
|
||||
err := sendMail(mocm, m, time.Now(), getConfig())
|
||||
require.NoError(t, err)
|
||||
|
||||
lower := strings.ToLower(string(mocm.data))
|
||||
// Be tolerant of charset=UTF-8 vs charset="UTF-8".
|
||||
hasCharset := strings.Contains(lower, "charset=utf-8") || strings.Contains(lower, `charset="utf-8"`)
|
||||
assert.True(t, hasCharset, "body parts must declare a UTF-8 charset")
|
||||
}
|
||||
|
||||
// TestSendMailFromDisplayName verifies that a From display name renders and that
|
||||
// a non-ASCII display name is RFC 2047 encoded rather than emitted raw.
|
||||
func TestSendMailFromDisplayName(t *testing.T) {
|
||||
t.Run("ASCII display name renders literally", func(t *testing.T) {
|
||||
mocm := &mockMailer{}
|
||||
m := mailData{
|
||||
mimeTo: "test@example.com",
|
||||
smtpTo: "test@example.com",
|
||||
from: mail.Address{Name: "Plain Name", Address: "from@example.com"},
|
||||
htmlBody: "hi",
|
||||
}
|
||||
err := sendMail(mocm, m, time.Now(), getConfig())
|
||||
require.NoError(t, err)
|
||||
|
||||
from, ok := mailHeaderValue(t, string(mocm.data), "From")
|
||||
require.True(t, ok, "From header must be present")
|
||||
assert.Contains(t, from, "Plain Name")
|
||||
assert.Contains(t, from, "<from@example.com>")
|
||||
assert.NotContains(t, from, "=?", "an ASCII display name must not be RFC 2047 encoded")
|
||||
})
|
||||
|
||||
t.Run("non-ASCII display name is RFC 2047 encoded", func(t *testing.T) {
|
||||
mocm := &mockMailer{}
|
||||
m := mailData{
|
||||
mimeTo: "test@example.com",
|
||||
smtpTo: "test@example.com",
|
||||
from: mail.Address{Name: "Café Admin ☕", Address: "from@example.com"},
|
||||
htmlBody: "hi",
|
||||
}
|
||||
err := sendMail(mocm, m, time.Now(), getConfig())
|
||||
require.NoError(t, err)
|
||||
|
||||
from, ok := mailHeaderValue(t, string(mocm.data), "From")
|
||||
require.True(t, ok, "From header must be present")
|
||||
assert.Contains(t, strings.ToLower(from), "=?utf-8?", "non-ASCII display name must be RFC 2047 encoded")
|
||||
assert.NotContains(t, from, "Café", "raw UTF-8 display name must not leak into the header")
|
||||
assert.Contains(t, from, "<from@example.com>")
|
||||
})
|
||||
}
|
||||
|
||||
// TestSendMailCcDisplayName verifies a Cc address with a display name renders
|
||||
// correctly through the address parser.
|
||||
func TestSendMailCcDisplayName(t *testing.T) {
|
||||
mocm := &mockMailer{}
|
||||
m := mailData{
|
||||
mimeTo: "test@example.com",
|
||||
smtpTo: "test@example.com",
|
||||
from: mail.Address{Address: "from@example.com"},
|
||||
cc: `"Cc Person" <cc@example.com>`,
|
||||
htmlBody: "hi",
|
||||
}
|
||||
err := sendMail(mocm, m, time.Now(), getConfig())
|
||||
require.NoError(t, err)
|
||||
|
||||
cc, ok := mailHeaderValue(t, string(mocm.data), "Cc")
|
||||
require.True(t, ok, "Cc header must be present")
|
||||
assert.Contains(t, cc, "Cc Person")
|
||||
assert.Contains(t, cc, "<cc@example.com>")
|
||||
}
|
||||
166
server/platform/shared/mfa/mfa.go
Normal file
166
server/platform/shared/mfa/mfa.go
Normal file
@@ -0,0 +1,166 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package mfa
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/base32"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/dgryski/dgoogauth"
|
||||
"github.com/mattermost/mattermost/server/public/model"
|
||||
"github.com/mattermost/rsc/qr"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// InvalidToken indicates the case where the token validation has failed.
|
||||
var InvalidToken = errors.New("invalid mfa token")
|
||||
|
||||
const (
|
||||
// This will result in 160 bits of entropy (base32 encoded), as recommended by rfc4226.
|
||||
mfaSecretSize = 20
|
||||
)
|
||||
|
||||
type Store interface {
|
||||
UpdateMfaActive(userId string, active bool) error
|
||||
UpdateMfaSecret(userId, secret string) error
|
||||
StoreMfaUsedTimestamps(userId string, ts []int) error
|
||||
GetMfaUsedTimestamps(userId string) ([]int, error)
|
||||
}
|
||||
|
||||
type MFA struct {
|
||||
store Store
|
||||
}
|
||||
|
||||
func New(store Store) *MFA {
|
||||
return &MFA{store}
|
||||
}
|
||||
|
||||
// newRandomBase32String returns a base32 encoded string of a random slice
|
||||
// of bytes of the given size. The resulting entropy will be (8 * size) bits.
|
||||
func newRandomBase32String(size int) string {
|
||||
data := make([]byte, size)
|
||||
rand.Read(data)
|
||||
return base32.StdEncoding.EncodeToString(data)
|
||||
}
|
||||
|
||||
func getIssuerFromURL(uri string) string {
|
||||
issuer := "Mattermost"
|
||||
siteURL := strings.TrimSpace(uri)
|
||||
|
||||
if siteURL != "" {
|
||||
siteURL = strings.TrimPrefix(siteURL, "https://")
|
||||
siteURL = strings.TrimPrefix(siteURL, "http://")
|
||||
issuer = strings.TrimPrefix(siteURL, "www.")
|
||||
}
|
||||
|
||||
return url.QueryEscape(issuer)
|
||||
}
|
||||
|
||||
// GenerateSecret generates a new user mfa secret and store it with the StoreSecret function provided
|
||||
func (m *MFA) GenerateSecret(siteURL, userEmail, userID string) (string, []byte, error) {
|
||||
issuer := getIssuerFromURL(siteURL)
|
||||
|
||||
secret := newRandomBase32String(mfaSecretSize)
|
||||
|
||||
authLink := fmt.Sprintf("otpauth://totp/%s:%s?secret=%s&issuer=%s", issuer, userEmail, secret, issuer)
|
||||
|
||||
code, err := qr.Encode(authLink, qr.H)
|
||||
|
||||
if err != nil {
|
||||
return "", nil, errors.Wrap(err, "unable to generate qr code")
|
||||
}
|
||||
|
||||
img := code.PNG()
|
||||
|
||||
if err := m.store.UpdateMfaSecret(userID, secret); err != nil {
|
||||
return "", nil, errors.Wrap(err, "unable to store mfa secret")
|
||||
}
|
||||
|
||||
return secret, img, nil
|
||||
}
|
||||
|
||||
// Activate set the mfa as active and store it with the StoreActive function provided
|
||||
func (m *MFA) Activate(userMfaSecret, userID string, token string) error {
|
||||
usedTs, err := m.store.GetMfaUsedTimestamps(userID)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "unable to retrieve the DisallowReuse slice")
|
||||
}
|
||||
|
||||
otpConfig, err := m.authenticate(userMfaSecret, usedTs, token)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "unable to authenticate the token")
|
||||
}
|
||||
|
||||
if err = m.store.UpdateMfaActive(userID, true); err != nil {
|
||||
return errors.Wrap(err, "unable to store mfa active")
|
||||
}
|
||||
|
||||
err = m.store.StoreMfaUsedTimestamps(userID, otpConfig.DisallowReuse)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "unable to store the DisallowReuse slice")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Deactivate set the mfa as deactivated, remove the mfa secret, store it with the StoreActive and StoreSecret functions provided
|
||||
func (m *MFA) Deactivate(userId string) error {
|
||||
if err := m.store.UpdateMfaActive(userId, false); err != nil {
|
||||
return errors.Wrap(err, "unable to store mfa active")
|
||||
}
|
||||
|
||||
if err := m.store.UpdateMfaSecret(userId, ""); err != nil {
|
||||
return errors.Wrap(err, "unable to store mfa secret")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate the provide token using the secret provided
|
||||
func (m *MFA) ValidateToken(user *model.User, token string) (bool, error) {
|
||||
usedTs, err := m.store.GetMfaUsedTimestamps(user.Id)
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, "unable to retrieve the DisallowReuse slice")
|
||||
}
|
||||
|
||||
otpConfig, err := m.authenticate(user.MfaSecret, usedTs, token)
|
||||
if err != nil {
|
||||
if err == InvalidToken {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return false, errors.Wrap(err, "unable to parse the token")
|
||||
}
|
||||
|
||||
err = m.store.StoreMfaUsedTimestamps(user.Id, otpConfig.DisallowReuse)
|
||||
if err != nil {
|
||||
return true, errors.Wrap(err, "unable to store the DisallowReuse slice")
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (*MFA) authenticate(userMfaSecret string, usedTs []int, token string) (*dgoogauth.OTPConfig, error) {
|
||||
trimmedToken := strings.TrimSpace(token)
|
||||
|
||||
otpConfig := &dgoogauth.OTPConfig{
|
||||
Secret: userMfaSecret,
|
||||
WindowSize: 3,
|
||||
HotpCounter: 0,
|
||||
DisallowReuse: usedTs,
|
||||
}
|
||||
|
||||
ok, err := otpConfig.Authenticate(trimmedToken)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "unable to parse the token")
|
||||
}
|
||||
if !ok {
|
||||
return nil, InvalidToken
|
||||
}
|
||||
|
||||
return otpConfig, nil
|
||||
}
|
||||
237
server/platform/shared/mfa/mfa_test.go
Normal file
237
server/platform/shared/mfa/mfa_test.go
Normal file
@@ -0,0 +1,237 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package mfa
|
||||
|
||||
import (
|
||||
"encoding/base32"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dgryski/dgoogauth"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/mattermost/mattermost/server/public/model"
|
||||
"github.com/mattermost/mattermost/server/public/plugin/plugintest/mock"
|
||||
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
|
||||
)
|
||||
|
||||
func TestGenerateSecret(t *testing.T) {
|
||||
userID := "user-id"
|
||||
userEmail := "sample@sample.com"
|
||||
siteURL := "http://localhost:8065"
|
||||
|
||||
t.Run("fail on store action fail", func(t *testing.T) {
|
||||
storeMock := mocks.UserStore{}
|
||||
storeMock.On("UpdateMfaSecret", userID, mock.AnythingOfType("string")).Return(func(userId string, secret string) error {
|
||||
return errors.New("failed to update mfa secret")
|
||||
})
|
||||
|
||||
_, _, err := New(&storeMock).GenerateSecret(siteURL, userEmail, userID)
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "unable to store mfa secret")
|
||||
})
|
||||
|
||||
t.Run("Successful generate secret", func(t *testing.T) {
|
||||
storeMock := mocks.UserStore{}
|
||||
storeMock.On("UpdateMfaSecret", userID, mock.AnythingOfType("string")).Return(func(userId string, secret string) error {
|
||||
return nil
|
||||
})
|
||||
|
||||
secret, img, err := New(&storeMock).GenerateSecret(siteURL, userEmail, userID)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, secret, 32)
|
||||
require.NotEmpty(t, img, "no image set")
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetIssuerFromURL(t *testing.T) {
|
||||
cases := []struct {
|
||||
Input string
|
||||
Expected string
|
||||
}{
|
||||
{"http://somewebsite.com", url.QueryEscape("somewebsite.com")},
|
||||
{"https://somewebsite.com", url.QueryEscape("somewebsite.com")},
|
||||
{"https://some.website.com", url.QueryEscape("some.website.com")},
|
||||
{" https://www.somewebsite.com", url.QueryEscape("somewebsite.com")},
|
||||
{"http://somewebsite.com/chat", url.QueryEscape("somewebsite.com/chat")},
|
||||
{"somewebsite.com ", url.QueryEscape("somewebsite.com")},
|
||||
{"http://localhost:8065", url.QueryEscape("localhost:8065")},
|
||||
{"", "Mattermost"},
|
||||
{" ", "Mattermost"},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
assert.Equal(t, c.Expected, getIssuerFromURL(c.Input))
|
||||
}
|
||||
}
|
||||
|
||||
func TestActivate(t *testing.T) {
|
||||
userID := "user-id"
|
||||
userMfaSecret := newRandomBase32String(mfaSecretSize)
|
||||
|
||||
token := dgoogauth.ComputeCode(userMfaSecret, time.Now().UTC().Unix()/30)
|
||||
|
||||
t.Run("fail on wrongly formatted token", func(t *testing.T) {
|
||||
storeMock := mocks.UserStore{}
|
||||
storeMock.On("GetMfaUsedTimestamps", userID).Return([]int{}, nil).Once()
|
||||
|
||||
err := New(&storeMock).Activate(userMfaSecret, userID, "invalid-token")
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "unable to parse the token")
|
||||
})
|
||||
|
||||
t.Run("fail on invalid token", func(t *testing.T) {
|
||||
storeMock := mocks.UserStore{}
|
||||
storeMock.On("GetMfaUsedTimestamps", userID).Return([]int{}, nil).Once()
|
||||
|
||||
err := New(&storeMock).Activate(userMfaSecret, userID, "000000")
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "invalid mfa token")
|
||||
})
|
||||
|
||||
t.Run("fail on store action fail", func(t *testing.T) {
|
||||
storeMock := mocks.UserStore{}
|
||||
storeMock.On("GetMfaUsedTimestamps", userID).Return([]int{}, nil).Once()
|
||||
storeMock.On("UpdateMfaActive", userID, true).Return(func(userId string, active bool) error {
|
||||
return errors.New("failed to update mfa active")
|
||||
})
|
||||
|
||||
err := New(&storeMock).Activate(userMfaSecret, userID, fmt.Sprintf("%06d", token))
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "unable to store mfa active")
|
||||
})
|
||||
|
||||
t.Run("Successful activate", func(t *testing.T) {
|
||||
userID := model.NewId()
|
||||
secret := newRandomBase32String(mfaSecretSize)
|
||||
|
||||
t0 := time.Now().UTC().Unix() / 30
|
||||
code := fmt.Sprintf("%06d", dgoogauth.ComputeCode(secret, t0))
|
||||
|
||||
usMock := mocks.UserStore{}
|
||||
usMock.On("GetMfaUsedTimestamps", userID).Return([]int{}, nil).Once()
|
||||
usMock.On("UpdateMfaActive", userID, true).Return(nil).Once()
|
||||
usMock.On("StoreMfaUsedTimestamps", userID, mock.AnythingOfType("[]int")).Return(nil).Once()
|
||||
|
||||
err := New(&usMock).Activate(secret, userID, code)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("disallow reuse of totp", func(t *testing.T) {
|
||||
userID := model.NewId()
|
||||
secret := newRandomBase32String(mfaSecretSize)
|
||||
|
||||
t0 := time.Now().UTC().Unix() / 30
|
||||
code := fmt.Sprintf("%06d", dgoogauth.ComputeCode(secret, t0))
|
||||
|
||||
usMock := mocks.UserStore{}
|
||||
usMock.On("GetMfaUsedTimestamps", userID).Return([]int{int(t0)}, nil).Once()
|
||||
|
||||
err := New(&usMock).Activate(secret, userID, code)
|
||||
require.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestDeactivate(t *testing.T) {
|
||||
userID := "user-id"
|
||||
|
||||
t.Run("fail on store UpdateMfaActive action fail", func(t *testing.T) {
|
||||
storeMock := mocks.UserStore{}
|
||||
storeMock.On("UpdateMfaActive", userID, false).Return(func(userId string, active bool) error {
|
||||
return errors.New("failed to update mfa active")
|
||||
})
|
||||
storeMock.On("UpdateMfaSecret", userID, "").Return(func(userId string, secret string) error {
|
||||
return errors.New("failed to update mfa secret")
|
||||
})
|
||||
|
||||
err := New(&storeMock).Deactivate(userID)
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "unable to store mfa active")
|
||||
})
|
||||
|
||||
t.Run("fail on store UpdateMfaSecret action fail", func(t *testing.T) {
|
||||
storeMock := mocks.UserStore{}
|
||||
storeMock.On("UpdateMfaActive", userID, false).Return(func(userId string, active bool) error {
|
||||
return nil
|
||||
})
|
||||
storeMock.On("UpdateMfaSecret", userID, "").Return(func(userId string, secret string) error {
|
||||
return errors.New("failed to update mfa secret")
|
||||
})
|
||||
|
||||
err := New(&storeMock).Deactivate(userID)
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "unable to store mfa secret")
|
||||
})
|
||||
|
||||
t.Run("Successful deactivate", func(t *testing.T) {
|
||||
storeMock := mocks.UserStore{}
|
||||
storeMock.On("UpdateMfaActive", userID, false).Return(func(userId string, active bool) error {
|
||||
return nil
|
||||
})
|
||||
storeMock.On("UpdateMfaSecret", userID, "").Return(func(userId string, secret string) error {
|
||||
return nil
|
||||
})
|
||||
|
||||
err := New(&storeMock).Deactivate(userID)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestValidateToken(t *testing.T) {
|
||||
t.Run("fail on wrongly formatted token", func(t *testing.T) {
|
||||
id := model.NewId()
|
||||
secret := newRandomBase32String(mfaSecretSize)
|
||||
u := &model.User{Id: id, MfaSecret: secret}
|
||||
|
||||
usMock := mocks.UserStore{}
|
||||
usMock.On("GetMfaUsedTimestamps", u.Id).Return([]int{}, nil).Once()
|
||||
ok, err := New(&usMock).ValidateToken(u, "invalid-token")
|
||||
require.Error(t, err)
|
||||
require.False(t, ok)
|
||||
require.Contains(t, err.Error(), "unable to parse the token")
|
||||
})
|
||||
|
||||
t.Run("successful validation", func(t *testing.T) {
|
||||
id := model.NewId()
|
||||
secret := newRandomBase32String(mfaSecretSize)
|
||||
u := &model.User{Id: id, MfaSecret: secret}
|
||||
|
||||
code := fmt.Sprintf("%06d", dgoogauth.ComputeCode(secret, time.Now().UTC().Unix()/30))
|
||||
|
||||
usMock := mocks.UserStore{}
|
||||
usMock.On("GetMfaUsedTimestamps", u.Id).Return([]int{}, nil).Once()
|
||||
usMock.On("StoreMfaUsedTimestamps", u.Id, mock.AnythingOfType("[]int")).Return(nil).Once()
|
||||
|
||||
ok, err := New(&usMock).ValidateToken(u, code)
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
})
|
||||
|
||||
t.Run("disallow reuse of totp", func(t *testing.T) {
|
||||
id := model.NewId()
|
||||
secret := newRandomBase32String(mfaSecretSize)
|
||||
u := &model.User{Id: id, MfaSecret: secret}
|
||||
|
||||
t0 := time.Now().UTC().Unix() / 30
|
||||
code := fmt.Sprintf("%06d", dgoogauth.ComputeCode(secret, t0))
|
||||
|
||||
usMock := mocks.UserStore{}
|
||||
usMock.On("GetMfaUsedTimestamps", u.Id).Return([]int{int(t0)}, nil).Once()
|
||||
|
||||
ok, err := New(&usMock).ValidateToken(u, code)
|
||||
require.False(t, ok)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestRandomBase32String(t *testing.T) {
|
||||
for i := range 1000 {
|
||||
str := newRandomBase32String(i)
|
||||
require.Len(t, str, base32.StdEncoding.EncodedLen(i))
|
||||
}
|
||||
}
|
||||
74
server/platform/shared/templates/templates.go
Normal file
74
server/platform/shared/templates/templates.go
Normal file
@@ -0,0 +1,74 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package templates
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"html/template"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/mattermost/mattermost/server/v8/channels/utils/fileutils"
|
||||
)
|
||||
|
||||
// Container represents a set of templates that can be render
|
||||
type Container struct {
|
||||
templates *template.Template
|
||||
mutex sync.RWMutex
|
||||
}
|
||||
|
||||
// Data contains the data used to populate the template variables, it has Props
|
||||
// that can be of any type and HTML that only can be `template.HTML` types.
|
||||
type Data struct {
|
||||
Props map[string]any
|
||||
HTML map[string]template.HTML
|
||||
}
|
||||
|
||||
func GetTemplateDirectory() (string, bool) {
|
||||
return fileutils.FindDir("templates")
|
||||
}
|
||||
|
||||
// NewFromTemplates creates a new templates container using a
|
||||
// `template.Template` object
|
||||
func NewFromTemplate(templates *template.Template) *Container {
|
||||
return &Container{templates: templates}
|
||||
}
|
||||
|
||||
// New creates a new templates container scanning a directory.
|
||||
func New(directory string) (*Container, error) {
|
||||
c := &Container{}
|
||||
|
||||
htmlTemplates, err := template.ParseGlob(filepath.Join(directory, "*.html"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.templates = htmlTemplates
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// RenderToString renders the template referenced with the template name using
|
||||
// the data provided and return a string with the result
|
||||
func (c *Container) RenderToString(templateName string, data Data) (string, error) {
|
||||
var text bytes.Buffer
|
||||
if err := c.Render(&text, templateName, data); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return text.String(), nil
|
||||
}
|
||||
|
||||
// RenderToString renders the template referenced with the template name using
|
||||
// the data provided and write it to the writer provided
|
||||
func (c *Container) Render(w io.Writer, templateName string, data Data) error {
|
||||
c.mutex.RLock()
|
||||
htmlTemplates := c.templates
|
||||
c.mutex.RUnlock()
|
||||
|
||||
if err := htmlTemplates.ExecuteTemplate(w, templateName, data); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
67
server/platform/shared/templates/templates_test.go
Normal file
67
server/platform/shared/templates/templates_test.go
Normal file
@@ -0,0 +1,67 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package templates
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"html/template"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNew_BadDirectory(t *testing.T) {
|
||||
container, err := New("notarealdirectory")
|
||||
assert.Nil(t, container)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestRender(t *testing.T) {
|
||||
tpl := template.New("test")
|
||||
_, err := tpl.Parse(`{{ define "foo" }}foo{{ .Props.Bar }}{{ end }}`)
|
||||
require.NoError(t, err)
|
||||
mt := NewFromTemplate(tpl)
|
||||
|
||||
data := Data{
|
||||
Props: map[string]any{
|
||||
"Bar": "bar",
|
||||
},
|
||||
}
|
||||
text, err := mt.RenderToString("foo", data)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "foobar", text)
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
require.NoError(t, mt.Render(buf, "foo", data))
|
||||
assert.Equal(t, "foobar", buf.String())
|
||||
}
|
||||
|
||||
func TestRenderError(t *testing.T) {
|
||||
tpl := template.New("test")
|
||||
_, err := tpl.Parse(`{{ define "foo" }}foo{{ .Foo.Bar }}bar{{ end }}`)
|
||||
require.NoError(t, err)
|
||||
mt := NewFromTemplate(tpl)
|
||||
|
||||
text, err := mt.RenderToString("foo", Data{})
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, "", text)
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
assert.Error(t, mt.Render(buf, "foo", Data{}))
|
||||
assert.Equal(t, "foo", buf.String())
|
||||
}
|
||||
|
||||
func TestRenderUnknownTemplate(t *testing.T) {
|
||||
tpl := template.New("")
|
||||
mt := NewFromTemplate(tpl)
|
||||
|
||||
text, err := mt.RenderToString("foo", Data{})
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, "", text)
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
assert.Error(t, mt.Render(buf, "foo", Data{}))
|
||||
assert.Equal(t, "", buf.String())
|
||||
}
|
||||
114
server/platform/shared/web/files.go
Normal file
114
server/platform/shared/web/files.go
Normal file
@@ -0,0 +1,114 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package web
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
var UnsafeContentTypes = [...]string{
|
||||
"application/javascript",
|
||||
"application/ecmascript",
|
||||
"text/javascript",
|
||||
"text/ecmascript",
|
||||
"application/x-javascript",
|
||||
"text/html",
|
||||
}
|
||||
|
||||
var MediaContentTypes = [...]string{
|
||||
"image/jpeg",
|
||||
"image/png",
|
||||
"image/bmp",
|
||||
"image/gif",
|
||||
"image/tiff",
|
||||
"image/webp",
|
||||
"video/avi",
|
||||
"video/mpeg",
|
||||
"video/mp4",
|
||||
"audio/mpeg",
|
||||
"audio/wav",
|
||||
}
|
||||
|
||||
// WriteFileResponse copies the io.ReadSeeker `fileReader` to the ResponseWriter `w`. Use this when you have a
|
||||
// ReadSeeker.
|
||||
func WriteFileResponse(filename string, contentType string, contentSize int64, lastModification time.Time, webserverMode string, fileReader io.ReadSeeker, forceDownload bool, w http.ResponseWriter, r *http.Request) {
|
||||
setHeaders(w, contentType, forceDownload, filename)
|
||||
|
||||
if contentSize > 0 {
|
||||
contentSizeStr := strconv.Itoa(int(contentSize))
|
||||
if webserverMode == "gzip" {
|
||||
w.Header().Set("X-Uncompressed-Content-Length", contentSizeStr)
|
||||
} else {
|
||||
w.Header().Set("Content-Length", contentSizeStr)
|
||||
}
|
||||
}
|
||||
|
||||
http.ServeContent(w, r, filename, lastModification, fileReader)
|
||||
}
|
||||
|
||||
// WriteStreamResponse copies the Reader `r` to the ResponseWriter `w`. Use this when you need to stream a response
|
||||
// to the client that will appear as a file `filename` of type `contentType`.
|
||||
func WriteStreamResponse(w http.ResponseWriter, r io.Reader, filename string, contentType string, forceDownload bool) error {
|
||||
setHeaders(w, contentType, forceDownload, filename)
|
||||
|
||||
if _, err := io.Copy(w, r); err != nil {
|
||||
return errors.Wrap(err, "error streaming Reader")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func setHeaders(w http.ResponseWriter, contentType string, forceDownload bool, filename string) {
|
||||
// Only set Cache-Control if it hasn't been set already
|
||||
if w.Header().Get("Cache-Control") == "" {
|
||||
w.Header().Set("Cache-Control", "private, max-age=86400")
|
||||
}
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
} else {
|
||||
for _, unsafeContentType := range UnsafeContentTypes {
|
||||
if strings.HasPrefix(contentType, unsafeContentType) {
|
||||
contentType = "text/plain"
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
|
||||
var toDownload bool
|
||||
if forceDownload {
|
||||
toDownload = true
|
||||
} else {
|
||||
isMediaType := false
|
||||
for _, mediaContentType := range MediaContentTypes {
|
||||
if strings.HasPrefix(contentType, mediaContentType) {
|
||||
isMediaType = true
|
||||
break
|
||||
}
|
||||
}
|
||||
toDownload = !isMediaType
|
||||
}
|
||||
|
||||
filename = url.PathEscape(filename)
|
||||
|
||||
if toDownload {
|
||||
w.Header().Set("Content-Disposition", "attachment;filename=\""+filename+"\"; filename*=UTF-8''"+filename)
|
||||
} else {
|
||||
w.Header().Set("Content-Disposition", "inline;filename=\""+filename+"\"; filename*=UTF-8''"+filename)
|
||||
}
|
||||
|
||||
// prevent file links from being embedded in iframes
|
||||
w.Header().Set("X-Frame-Options", "DENY")
|
||||
w.Header().Set("Content-Security-Policy", "Frame-ancestors 'none'")
|
||||
}
|
||||
Reference in New Issue
Block a user