snapshot of https://github.com/mattermost/mattermost.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
Server CI / Postgres (shard 0) (push) Has been skipped
Server CI / Postgres (shard 3) (push) Has been skipped
BuildEnv Docker Image / build-image (push) Failing after 4s
BuildEnv Docker Image / build-image-fips (push) Failing after 4s
Migration Review & Release Notes / Generate migration review & release notes (push) Failing after 3s
Server CI / Build Environment (push) Failing after 3s
Server CI / Postgres (shard 1) (push) Has been skipped
Tools CI / check-style (mattermost-govet) (push) Failing after 4s
Server CI / Postgres (shard 2) (push) Has been skipped
Tools CI / check-style (mmgotool) (push) Failing after 3s
Server CI / Elasticsearch v8 Compatibility (push) Has been skipped
Server CI / OpenSearch v2 Compatibility (push) Has been skipped
Server CI / Postgres FIPS (shard 0) (push) Has been skipped
Tools CI / Test (mattermost-govet) (push) Failing after 3s
Server CI / Postgres FIPS (shard 1) (push) Has been skipped
Server CI / Postgres FIPS (shard 2) (push) Has been skipped
Tools CI / Test (mmgotool) (push) Failing after 3s
Server CI / Postgres FIPS (shard 3) (push) Has been skipped
Server CI / Run mmctl tests (push) Has been skipped
Server CI / Run mmctl tests (FIPS) (push) Has been skipped
Server CI / Merge Postgres Test Results (push) Has been skipped
Server CI / Merge Postgres FIPS Test Results (push) Has been skipped
Server CI / Check backport migrations (push) Has been skipped
Server CI / Check generated files (push) Has been skipped
Server CI / check-style (push) Has been skipped
Server CI / Vet API (push) Has been skipped
Server CI / Build mattermost server app (push) Has been skipped
Server CI / Server CI Complete (push) Failing after 2s
Server CI / Server CI Artifacts (push) Has been skipped
Server CI / Sentry Upload (push) Has been skipped
Server CI / Server CI Report (push) Failing after 5m1s
Server CI Nightly Race / Race Detector (push) Has been skipped
CodeQL / Analyze (go) (push) Has been cancelled
CodeQL / Analyze (javascript) (push) Has been cancelled
Webapp CI Cache Warm / Warm Webapp node_modules Cache (push) Has been cancelled
E2E CI Cache Warm / Warm E2E npm Registry Cache (push) Has been cancelled
API / build (push) Has been cancelled
Web App CI / check-types (push) Has been cancelled
Web App CI / check-lint (push) Has been cancelled
Web App CI / check-i18n (push) Has been cancelled
Web App CI / check-external-links (push) Has been cancelled
Web App CI / test (platform) (push) Has been cancelled
Web App CI / test (mattermost-redux) (push) Has been cancelled
Web App CI / test (channels shard 1/4) (push) Has been cancelled
Web App CI / test (channels shard 2/4) (push) Has been cancelled
Web App CI / test (channels shard 3/4) (push) Has been cancelled
Web App CI / test (channels shard 4/4) (push) Has been cancelled
Web App CI / upload-coverage (push) Has been cancelled
Web App CI / build (push) Has been cancelled
YAML Lint / yamllint (push) Has been cancelled
Push mirrored docker images / cd/Push mirrored docker images (push) Has been cancelled
docs-cd / Build and deploy docs (push) Has been cancelled
Docs CI / build (push) Has been cancelled
E2E Cache Warm: Cypress deps / Warm Cypress deps cache (push) Has been cancelled
Scorecards supply-chain security / Scorecard analysis (push) Has been cancelled
Server CI Weekly / Postgres FIPS (push) Has been skipped
Server CI Weekly / Run mmctl tests (FIPS) (push) Has been skipped
Server CI Weekly / Postgres with binary parameters (push) Has been skipped
Server CI Weekly / Build Environment (push) Failing after 1s
Server CI Cache Warm / Warm Go Module and Build Cache (push) Failing after 2s
Server CI Nightly Race / Build Environment (push) Failing after 1s
Some checks failed
Server CI / Postgres (shard 0) (push) Has been skipped
Server CI / Postgres (shard 3) (push) Has been skipped
BuildEnv Docker Image / build-image (push) Failing after 4s
BuildEnv Docker Image / build-image-fips (push) Failing after 4s
Migration Review & Release Notes / Generate migration review & release notes (push) Failing after 3s
Server CI / Build Environment (push) Failing after 3s
Server CI / Postgres (shard 1) (push) Has been skipped
Tools CI / check-style (mattermost-govet) (push) Failing after 4s
Server CI / Postgres (shard 2) (push) Has been skipped
Tools CI / check-style (mmgotool) (push) Failing after 3s
Server CI / Elasticsearch v8 Compatibility (push) Has been skipped
Server CI / OpenSearch v2 Compatibility (push) Has been skipped
Server CI / Postgres FIPS (shard 0) (push) Has been skipped
Tools CI / Test (mattermost-govet) (push) Failing after 3s
Server CI / Postgres FIPS (shard 1) (push) Has been skipped
Server CI / Postgres FIPS (shard 2) (push) Has been skipped
Tools CI / Test (mmgotool) (push) Failing after 3s
Server CI / Postgres FIPS (shard 3) (push) Has been skipped
Server CI / Run mmctl tests (push) Has been skipped
Server CI / Run mmctl tests (FIPS) (push) Has been skipped
Server CI / Merge Postgres Test Results (push) Has been skipped
Server CI / Merge Postgres FIPS Test Results (push) Has been skipped
Server CI / Check backport migrations (push) Has been skipped
Server CI / Check generated files (push) Has been skipped
Server CI / check-style (push) Has been skipped
Server CI / Vet API (push) Has been skipped
Server CI / Build mattermost server app (push) Has been skipped
Server CI / Server CI Complete (push) Failing after 2s
Server CI / Server CI Artifacts (push) Has been skipped
Server CI / Sentry Upload (push) Has been skipped
Server CI / Server CI Report (push) Failing after 5m1s
Server CI Nightly Race / Race Detector (push) Has been skipped
CodeQL / Analyze (go) (push) Has been cancelled
CodeQL / Analyze (javascript) (push) Has been cancelled
Webapp CI Cache Warm / Warm Webapp node_modules Cache (push) Has been cancelled
E2E CI Cache Warm / Warm E2E npm Registry Cache (push) Has been cancelled
API / build (push) Has been cancelled
Web App CI / check-types (push) Has been cancelled
Web App CI / check-lint (push) Has been cancelled
Web App CI / check-i18n (push) Has been cancelled
Web App CI / check-external-links (push) Has been cancelled
Web App CI / test (platform) (push) Has been cancelled
Web App CI / test (mattermost-redux) (push) Has been cancelled
Web App CI / test (channels shard 1/4) (push) Has been cancelled
Web App CI / test (channels shard 2/4) (push) Has been cancelled
Web App CI / test (channels shard 3/4) (push) Has been cancelled
Web App CI / test (channels shard 4/4) (push) Has been cancelled
Web App CI / upload-coverage (push) Has been cancelled
Web App CI / build (push) Has been cancelled
YAML Lint / yamllint (push) Has been cancelled
Push mirrored docker images / cd/Push mirrored docker images (push) Has been cancelled
docs-cd / Build and deploy docs (push) Has been cancelled
Docs CI / build (push) Has been cancelled
E2E Cache Warm: Cypress deps / Warm Cypress deps cache (push) Has been cancelled
Scorecards supply-chain security / Scorecard analysis (push) Has been cancelled
Server CI Weekly / Postgres FIPS (push) Has been skipped
Server CI Weekly / Run mmctl tests (FIPS) (push) Has been skipped
Server CI Weekly / Postgres with binary parameters (push) Has been skipped
Server CI Weekly / Build Environment (push) Failing after 1s
Server CI Cache Warm / Warm Go Module and Build Cache (push) Failing after 2s
Server CI Nightly Race / Build Environment (push) Failing after 1s
This commit is contained in:
120
server/platform/services/docextractor/archive.go
Normal file
120
server/platform/services/docextractor/archive.go
Normal file
@@ -0,0 +1,120 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package docextractor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/mholt/archives"
|
||||
|
||||
"github.com/mattermost/mattermost/server/v8/channels/utils"
|
||||
)
|
||||
|
||||
type archiveExtractor struct {
|
||||
SubExtractor Extractor
|
||||
}
|
||||
|
||||
func (ae *archiveExtractor) Name() string {
|
||||
return "archiveExtractor"
|
||||
}
|
||||
|
||||
func (ae *archiveExtractor) Match(filename string) bool {
|
||||
_, _, err := archives.Identify(context.Background(), filename, nil)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// getExtAlsoTarGz returns the extension of the given file name, special casing .tar.gz.
|
||||
func getExtAlsoTarGz(name string) string {
|
||||
if strings.HasSuffix(name, ".tar.gz") {
|
||||
return ".tar.gz"
|
||||
}
|
||||
|
||||
return filepath.Ext(name)
|
||||
}
|
||||
|
||||
func (ae *archiveExtractor) Extract(ctx context.Context, name string, r io.ReadSeeker, maxFileSize int64) (string, error) {
|
||||
// MM-65701: Skip 7zip files due to OOM vulnerability in bodgit/sevenzip library
|
||||
match, _ := (archives.SevenZip{}).Match(context.Background(), name, r)
|
||||
_, _ = r.Seek(0, io.SeekStart) // Reset reader position after Match reads from stream
|
||||
if match.ByName || match.ByStream {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
ext := getExtAlsoTarGz(name)
|
||||
|
||||
// Create a temporary file, using `*` control the random component while preserving the extension.
|
||||
f, err := os.CreateTemp("", "archiver-*"+ext)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error creating temporary file: %v", err)
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
|
||||
_, err = io.Copy(f, r)
|
||||
f.Close()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error copying data into temporary file: %v", err)
|
||||
}
|
||||
|
||||
var text strings.Builder
|
||||
fsys, err := archives.FileSystem(context.Background(), f.Name(), nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error creating file system: %w", err)
|
||||
}
|
||||
|
||||
err = fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
text.WriteString(path + " ")
|
||||
if ae.SubExtractor != nil {
|
||||
filename := filepath.Base(path)
|
||||
filename = strings.ReplaceAll(filename, "-", " ")
|
||||
filename = strings.ReplaceAll(filename, ".", " ")
|
||||
filename = strings.ReplaceAll(filename, ",", " ")
|
||||
|
||||
file, err := fsys.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// Limit the size of decompressed archive entries to prevent
|
||||
// memory exhaustion from zip bombs or other malicious archives.
|
||||
var reader io.Reader = file
|
||||
if maxFileSize > 0 {
|
||||
reader = utils.NewLimitedReaderWithError(file, maxFileSize)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading archive entry: %w", err)
|
||||
}
|
||||
|
||||
subtext, extractErr := ae.SubExtractor.Extract(ctx, filename, bytes.NewReader(data), maxFileSize)
|
||||
if extractErr == nil {
|
||||
text.WriteString(subtext + " ")
|
||||
} else if errors.Is(extractErr, context.Canceled) || errors.Is(extractErr, context.DeadlineExceeded) {
|
||||
return fmt.Errorf("error extracting archive entry: %w", extractErr)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return text.String(), nil
|
||||
}
|
||||
117
server/platform/services/docextractor/archive_test.go
Normal file
117
server/platform/services/docextractor/archive_test.go
Normal file
@@ -0,0 +1,117 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package docextractor
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestArchiveExtractorSkips7zip(t *testing.T) {
|
||||
ae := &archiveExtractor{}
|
||||
|
||||
t.Run("7zip file with .7z extension returns empty string", func(t *testing.T) {
|
||||
// Valid 7zip header (minimal)
|
||||
sevenZipData := []byte{0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c, 0x00, 0x00}
|
||||
result, err := ae.Extract(context.Background(), "test.7z", bytes.NewReader(sevenZipData), 0)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, result)
|
||||
})
|
||||
|
||||
t.Run("7zip content with wrong extension is still blocked", func(t *testing.T) {
|
||||
// 7zip content disguised with .zip extension - should still be blocked via stream detection
|
||||
sevenZipData := []byte{0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c, 0x00, 0x00}
|
||||
result, err := ae.Extract(context.Background(), "malicious.zip", bytes.NewReader(sevenZipData), 0)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, result)
|
||||
})
|
||||
|
||||
t.Run("7zip at offset with .7z extension is blocked via filename", func(t *testing.T) {
|
||||
junkPrefix := []byte{0x00, 0x00, 0x00, 0x00}
|
||||
sevenZipSig := []byte{0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c, 0x00, 0x00}
|
||||
dataWithOffset := append(junkPrefix, sevenZipSig...)
|
||||
result, err := ae.Extract(context.Background(), "test.7z", bytes.NewReader(dataWithOffset), 0)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, result)
|
||||
})
|
||||
|
||||
t.Run("7zip at offset with wrong extension fails safely", func(t *testing.T) {
|
||||
// Edge case: 7zip content at offset with non-.7z extension.
|
||||
// Our check won't catch it (ByName=false, ByStream=false), but archives.FileSystem
|
||||
// also won't identify it as 7zip, so it fails to extract rather than triggering OOM.
|
||||
junkPrefix := []byte{0x00, 0x00, 0x00, 0x00}
|
||||
sevenZipSig := []byte{0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c, 0x00, 0x00}
|
||||
dataWithOffset := append(junkPrefix, sevenZipSig...)
|
||||
_, err := ae.Extract(context.Background(), "malicious.zip", bytes.NewReader(dataWithOffset), 0)
|
||||
assert.Error(t, err) // fails to extract as any valid archive format
|
||||
})
|
||||
}
|
||||
|
||||
// contextErrorExtractor fails every extraction with the given context error.
|
||||
type contextErrorExtractor struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (ce *contextErrorExtractor) Name() string {
|
||||
return "contextErrorExtractor"
|
||||
}
|
||||
|
||||
func (ce *contextErrorExtractor) Match(filename string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (ce *contextErrorExtractor) Extract(_ context.Context, _ string, _ io.ReadSeeker, _ int64) (string, error) {
|
||||
return "", ce.err
|
||||
}
|
||||
|
||||
func TestArchiveExtractorErrorOmitsEntryName(t *testing.T) {
|
||||
// The entry name is transformed before reaching the nested extraction error
|
||||
// (separators become spaces), so assert on the stem token as well: it
|
||||
// survives that transformation and would appear in either leaky error.
|
||||
const entryName = "confidential-customer-list.txt"
|
||||
const entryStem = "confidential"
|
||||
|
||||
var archive bytes.Buffer
|
||||
zw := zip.NewWriter(&archive)
|
||||
entry, err := zw.Create(entryName)
|
||||
require.NoError(t, err)
|
||||
_, err = entry.Write([]byte(strings.Repeat("a", 1024)))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
|
||||
requireNoEntryName := func(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
require.Error(t, err)
|
||||
assert.NotContains(t, err.Error(), entryName)
|
||||
assert.NotContains(t, err.Error(), entryStem)
|
||||
}
|
||||
|
||||
t.Run("entry read failure", func(t *testing.T) {
|
||||
ae := &archiveExtractor{SubExtractor: &plainExtractor{}}
|
||||
|
||||
// A maxFileSize below the entry size fails the entry read.
|
||||
_, err := ae.Extract(context.Background(), "archive.zip", bytes.NewReader(archive.Bytes()), 8)
|
||||
requireNoEntryName(t, err)
|
||||
})
|
||||
|
||||
for name, contextErr := range map[string]error{
|
||||
"cancelled": context.Canceled,
|
||||
"deadline exceeded": context.DeadlineExceeded,
|
||||
} {
|
||||
t.Run("nested extraction "+name, func(t *testing.T) {
|
||||
ae := &archiveExtractor{SubExtractor: &contextErrorExtractor{err: contextErr}}
|
||||
|
||||
_, err := ae.Extract(context.Background(), "archive.zip", bytes.NewReader(archive.Bytes()), 0)
|
||||
requireNoEntryName(t, err)
|
||||
assert.ErrorIs(t, err, contextErr)
|
||||
})
|
||||
}
|
||||
}
|
||||
55
server/platform/services/docextractor/combine.go
Normal file
55
server/platform/services/docextractor/combine.go
Normal file
@@ -0,0 +1,55 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package docextractor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/mattermost/mattermost/server/public/shared/mlog"
|
||||
)
|
||||
|
||||
type combineExtractor struct {
|
||||
logger mlog.LoggerIFace
|
||||
SubExtractors []Extractor
|
||||
}
|
||||
|
||||
func (ce *combineExtractor) Name() string {
|
||||
return "combineExtractor"
|
||||
}
|
||||
|
||||
func (ce *combineExtractor) Add(extractor Extractor) {
|
||||
ce.SubExtractors = append(ce.SubExtractors, extractor)
|
||||
}
|
||||
|
||||
func (ce *combineExtractor) Match(filename string) bool {
|
||||
for _, extractor := range ce.SubExtractors {
|
||||
if extractor.Match(filename) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (ce *combineExtractor) Extract(ctx context.Context, filename string, r io.ReadSeeker, maxFileSize int64) (string, error) {
|
||||
for _, extractor := range ce.SubExtractors {
|
||||
if extractor.Match(filename) {
|
||||
r.Seek(0, io.SeekStart)
|
||||
text, err := extractor.Extract(ctx, filename, r, maxFileSize)
|
||||
if err != nil {
|
||||
// Context errors must not be retried: the caller explicitly
|
||||
// cancelled or timed out, so stop immediately.
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return "", err
|
||||
}
|
||||
|
||||
ce.logger.Warn("Unable to extract file content", mlog.String("extractor", extractor.Name()), mlog.Err(err))
|
||||
continue
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
40
server/platform/services/docextractor/concurrency.go
Normal file
40
server/platform/services/docextractor/concurrency.go
Normal file
@@ -0,0 +1,40 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package docextractor
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// maxConcurrentExtractions caps how many docconv extractions may run at once,
|
||||
// including detached goroutines that continue after a per-extraction timeout.
|
||||
var maxConcurrentExtractions = runtime.NumCPU()
|
||||
|
||||
var (
|
||||
extractionSlotsMu sync.Mutex
|
||||
extractionSlots = make(chan struct{}, maxConcurrentExtractions)
|
||||
)
|
||||
|
||||
func tryAcquireExtractionSlot() bool {
|
||||
select {
|
||||
case extractionSlots <- struct{}{}:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func releaseExtractionSlot() {
|
||||
<-extractionSlots
|
||||
}
|
||||
|
||||
// resetExtractionConcurrencyForTest reinitializes the extraction slot semaphore.
|
||||
// It is only intended for use from tests in this package.
|
||||
func resetExtractionConcurrencyForTest(limit int) {
|
||||
extractionSlotsMu.Lock()
|
||||
defer extractionSlotsMu.Unlock()
|
||||
maxConcurrentExtractions = limit
|
||||
extractionSlots = make(chan struct{}, limit)
|
||||
}
|
||||
149
server/platform/services/docextractor/docextractor.go
Normal file
149
server/platform/services/docextractor/docextractor.go
Normal file
@@ -0,0 +1,149 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package docextractor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/mattermost/mattermost/server/public/shared/mlog"
|
||||
)
|
||||
|
||||
// ExtractSettings defines the features enabled/disable during the document text extraction.
|
||||
type ExtractSettings struct {
|
||||
// Ctx is the parent context for the extraction. Cancellation propagates
|
||||
// to context-aware extractors (e.g. the Go-native PDF extractor stops at
|
||||
// the next page boundary). If nil, context.Background() is used.
|
||||
Ctx context.Context
|
||||
ArchiveRecursion bool
|
||||
MaxFileSize int64
|
||||
MMPreviewURL string
|
||||
MMPreviewSecret string
|
||||
// Timeout bounds how long a caller waits for a single extraction. When
|
||||
// set, a child context with this deadline is derived from Ctx and passed
|
||||
// to the extractor. Context-aware extractors (pdfExtractor) honour it
|
||||
// and stop at the next page boundary; others run to completion on a
|
||||
// detached goroutine but no longer hold the caller's worker slot. A
|
||||
// global cap on concurrent extractions (including those detached
|
||||
// goroutines) prevents sustained uploads from accumulating unbounded work.
|
||||
Timeout time.Duration
|
||||
// ReaderCloser, when set, transfers ownership of closing the input reader
|
||||
// to this package. It is closed only after extraction has actually
|
||||
// finished reading. This matters with Timeout set: on timeout the caller
|
||||
// returns while the converter may still be reading on a detached
|
||||
// goroutine, so the caller must NOT close the reader itself or it would
|
||||
// race with (and close the file out from under) that goroutine.
|
||||
ReaderCloser io.Closer
|
||||
}
|
||||
|
||||
// Extract extract the text from a document using the system default extractors
|
||||
func Extract(logger mlog.LoggerIFace, filename string, r io.ReadSeeker, settings ExtractSettings) (string, error) {
|
||||
return ExtractWithExtraExtractors(logger, filename, r, settings, []Extractor{})
|
||||
}
|
||||
|
||||
// ExtractWithExtraExtractors extract the text from a document using the provided extractors beside the system default extractors.
|
||||
func ExtractWithExtraExtractors(logger mlog.LoggerIFace, filename string, r io.ReadSeeker, settings ExtractSettings, extraExtractors []Extractor) (string, error) {
|
||||
enabledExtractors := &combineExtractor{
|
||||
logger: logger,
|
||||
}
|
||||
for _, extraExtractor := range extraExtractors {
|
||||
enabledExtractors.Add(extraExtractor)
|
||||
}
|
||||
enabledExtractors.Add(&documentExtractor{})
|
||||
enabledExtractors.Add(&pdfExtractor{})
|
||||
|
||||
if settings.ArchiveRecursion {
|
||||
enabledExtractors.Add(&archiveExtractor{SubExtractor: enabledExtractors})
|
||||
} else {
|
||||
enabledExtractors.Add(&archiveExtractor{})
|
||||
}
|
||||
|
||||
if settings.MMPreviewURL != "" {
|
||||
enabledExtractors.Add(newMMPreviewExtractor(settings.MMPreviewURL, settings.MMPreviewSecret, pdfExtractor{}))
|
||||
}
|
||||
enabledExtractors.Add(&plainExtractor{})
|
||||
|
||||
if enabledExtractors.Match(filename) {
|
||||
return extractWithTimeout(enabledExtractors, filename, r, settings)
|
||||
}
|
||||
|
||||
// No extractor matched, so nothing will read r; close it here since
|
||||
// extractWithTimeout (which otherwise owns the close) is never reached.
|
||||
if settings.ReaderCloser != nil {
|
||||
settings.ReaderCloser.Close()
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// extractWithTimeout runs the extraction under a context derived from
|
||||
// settings.Ctx. When settings.Timeout > 0 a deadline is added; on expiry the
|
||||
// caller is released and the extraction runs on a detached goroutine.
|
||||
// Context-aware extractors (pdfExtractor) honour cancellation at page
|
||||
// boundaries and stop promptly; others run to completion in the background.
|
||||
// A global concurrency cap applies to every in-flight extraction, including
|
||||
// detached goroutines, so timed-out work cannot accumulate beyond the limit.
|
||||
func extractWithTimeout(e Extractor, filename string, r io.ReadSeeker, settings ExtractSettings) (string, error) {
|
||||
if !tryAcquireExtractionSlot() {
|
||||
if settings.ReaderCloser != nil {
|
||||
settings.ReaderCloser.Close()
|
||||
}
|
||||
return "", fmt.Errorf("document text extraction capacity exhausted (%d concurrent extractions)", maxConcurrentExtractions)
|
||||
}
|
||||
|
||||
ctx := settings.Ctx
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
|
||||
if settings.Timeout <= 0 {
|
||||
defer releaseExtractionSlot()
|
||||
if settings.ReaderCloser != nil {
|
||||
defer settings.ReaderCloser.Close()
|
||||
}
|
||||
return e.Extract(ctx, filename, r, settings.MaxFileSize)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, settings.Timeout)
|
||||
defer cancel()
|
||||
|
||||
type extractResult struct {
|
||||
text string
|
||||
err error
|
||||
}
|
||||
resultCh := make(chan extractResult, 1)
|
||||
go func() {
|
||||
defer releaseExtractionSlot()
|
||||
// This goroutine owns the reader for the lifetime of the extraction.
|
||||
// After the timeout fires the caller returns, but the converter may
|
||||
// still be reading r here, so the reader is closed only once this
|
||||
// goroutine is done with it - never by the caller.
|
||||
if settings.ReaderCloser != nil {
|
||||
defer settings.ReaderCloser.Close()
|
||||
}
|
||||
// This goroutine is detached, so an unrecovered panic in an extractor
|
||||
// would crash the whole server. Convert it into an error instead.
|
||||
// resultCh is buffered (cap 1), so this send never blocks even if the
|
||||
// caller already timed out and stopped receiving.
|
||||
defer func() {
|
||||
if rec := recover(); rec != nil {
|
||||
resultCh <- extractResult{err: fmt.Errorf("panic during document text extraction: %v", rec)}
|
||||
}
|
||||
}()
|
||||
text, err := e.Extract(ctx, filename, r, settings.MaxFileSize)
|
||||
resultCh <- extractResult{text: text, err: err}
|
||||
}()
|
||||
|
||||
select {
|
||||
case res := <-resultCh:
|
||||
return res.text, res.err
|
||||
case <-ctx.Done():
|
||||
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
||||
return "", fmt.Errorf("document text extraction timed out after %s", settings.Timeout)
|
||||
}
|
||||
return "", fmt.Errorf("document text extraction cancelled: %w", ctx.Err())
|
||||
}
|
||||
}
|
||||
608
server/platform/services/docextractor/docextractor_test.go
Normal file
608
server/platform/services/docextractor/docextractor_test.go
Normal file
@@ -0,0 +1,608 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package docextractor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/mattermost/mattermost/server/public/shared/mlog"
|
||||
"github.com/mattermost/mattermost/server/v8/channels/utils/testutils"
|
||||
)
|
||||
|
||||
func TestExtract(t *testing.T) {
|
||||
logger := mlog.CreateConsoleTestLogger(t)
|
||||
|
||||
testCases := []struct {
|
||||
Name string
|
||||
TestFileName string
|
||||
Settings ExtractSettings
|
||||
Contains []string
|
||||
NotContains []string
|
||||
ExpectError bool
|
||||
}{
|
||||
{
|
||||
"Plain text file",
|
||||
"test-markdown-basics.md",
|
||||
ExtractSettings{},
|
||||
[]string{"followed", "separated", "Basic"},
|
||||
[]string{},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"Plain small text file",
|
||||
"test-hashtags.md",
|
||||
ExtractSettings{},
|
||||
[]string{"should", "render", "strings"},
|
||||
[]string{},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"Zip file without recursion",
|
||||
"Fake_Team_Import.zip",
|
||||
ExtractSettings{},
|
||||
[]string{"users", "channels", "general"},
|
||||
[]string{"purpose", "announcements"},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"Zip file with recursion",
|
||||
"Fake_Team_Import.zip",
|
||||
ExtractSettings{ArchiveRecursion: true},
|
||||
[]string{"users", "channels", "general", "purpose", "announcements"},
|
||||
[]string{},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"Rar file without recursion",
|
||||
"Fake_Team_Import.rar",
|
||||
ExtractSettings{},
|
||||
[]string{"users", "channels", "general"},
|
||||
[]string{"purpose", "announcements"},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"Rar file with recursion",
|
||||
"Fake_Team_Import.rar",
|
||||
ExtractSettings{ArchiveRecursion: true},
|
||||
[]string{"users", "channels", "general", "purpose", "announcements"},
|
||||
[]string{},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"Tar.gz file without recursion",
|
||||
"Fake_Team_Import.tar.gz",
|
||||
ExtractSettings{},
|
||||
[]string{"users", "channels", "general"},
|
||||
[]string{"purpose", "announcements"},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"Tar.gz file with recursion",
|
||||
"Fake_Team_Import.tar.gz",
|
||||
ExtractSettings{ArchiveRecursion: true},
|
||||
[]string{"users", "channels", "general", "purpose", "announcements"},
|
||||
[]string{},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"Pdf file",
|
||||
"sample-doc.pdf",
|
||||
ExtractSettings{},
|
||||
[]string{"simple", "document", "contains"},
|
||||
[]string{},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"Docx file",
|
||||
"sample-doc.docx",
|
||||
ExtractSettings{},
|
||||
[]string{"simple", "document", "contains"},
|
||||
[]string{},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"Odt file",
|
||||
"sample-doc.odt",
|
||||
ExtractSettings{},
|
||||
[]string{"simple", "document", "contains"},
|
||||
[]string{},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"Pptx file",
|
||||
"sample-doc.pptx",
|
||||
ExtractSettings{},
|
||||
[]string{"simple", "document", "contains"},
|
||||
[]string{},
|
||||
false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.Name, func(t *testing.T) {
|
||||
data, err := testutils.ReadTestFile(tc.TestFileName)
|
||||
require.NoError(t, err)
|
||||
text, err := Extract(logger, tc.TestFileName, bytes.NewReader(data), tc.Settings)
|
||||
if tc.ExpectError {
|
||||
require.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
for _, expectedString := range tc.Contains {
|
||||
assert.Contains(t, text, expectedString)
|
||||
}
|
||||
for _, notExpectedString := range tc.NotContains {
|
||||
assert.NotContains(t, text, notExpectedString)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("Unsupported binary file", func(t *testing.T) {
|
||||
data, err := testutils.ReadTestFile("testjpg.jpg")
|
||||
require.NoError(t, err)
|
||||
text, err := Extract(logger, "testjpg.jpg", bytes.NewReader(data), ExtractSettings{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "", text)
|
||||
})
|
||||
|
||||
t.Run("Wrong docx extension", func(t *testing.T) {
|
||||
data, err := testutils.ReadTestFile("sample-doc.pdf")
|
||||
require.NoError(t, err)
|
||||
text, err := Extract(logger, "sample-doc.docx", bytes.NewReader(data), ExtractSettings{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "", text)
|
||||
})
|
||||
}
|
||||
|
||||
type customTestPdfExtractor struct{}
|
||||
|
||||
func (te *customTestPdfExtractor) Name() string {
|
||||
return "customTestPdfExtractor"
|
||||
}
|
||||
|
||||
func (te *customTestPdfExtractor) Match(filename string) bool {
|
||||
return strings.HasSuffix(filename, ".pdf")
|
||||
}
|
||||
|
||||
func (te *customTestPdfExtractor) Extract(_ context.Context, filename string, r io.ReadSeeker, _ int64) (string, error) {
|
||||
return "this is a text generated content", nil
|
||||
}
|
||||
|
||||
type failingExtractor struct{}
|
||||
|
||||
func (te *failingExtractor) Name() string {
|
||||
return "failingExtractor"
|
||||
}
|
||||
|
||||
func (te *failingExtractor) Match(filename string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (te *failingExtractor) Extract(_ context.Context, filename string, r io.ReadSeeker, _ int64) (string, error) {
|
||||
return "", errors.New("this always fail")
|
||||
}
|
||||
|
||||
func TestExtractWithExtraExtractors(t *testing.T) {
|
||||
logger := mlog.CreateConsoleTestLogger(t)
|
||||
t.Run("override existing extractor", func(t *testing.T) {
|
||||
data, err := testutils.ReadTestFile("sample-doc.pdf")
|
||||
require.NoError(t, err)
|
||||
|
||||
text, err := ExtractWithExtraExtractors(logger, "sample-doc.pdf", bytes.NewReader(data), ExtractSettings{}, []Extractor{&customTestPdfExtractor{}})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, text, "this is a text generated content")
|
||||
})
|
||||
|
||||
t.Run("failing extractor", func(t *testing.T) {
|
||||
data, err := testutils.ReadTestFile("sample-doc.pdf")
|
||||
require.NoError(t, err)
|
||||
|
||||
text, err := ExtractWithExtraExtractors(logger, "sample-doc.pdf", bytes.NewReader(data), ExtractSettings{}, []Extractor{&failingExtractor{}})
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, text, "simple")
|
||||
assert.Contains(t, text, "document")
|
||||
assert.Contains(t, text, "contains")
|
||||
})
|
||||
|
||||
t.Run("cancelled context aborts extraction", func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
text, err := ExtractWithExtraExtractors(
|
||||
logger,
|
||||
"file.txt",
|
||||
bytes.NewReader([]byte("hello world")),
|
||||
ExtractSettings{
|
||||
Ctx: ctx,
|
||||
Timeout: time.Second,
|
||||
},
|
||||
[]Extractor{&slowExtractor{delay: 10 * time.Second}})
|
||||
require.Error(t, err)
|
||||
require.Empty(t, text)
|
||||
require.Contains(t, err.Error(), "cancelled")
|
||||
})
|
||||
|
||||
// Without context propagation a cancelled context would reach documentExtractor
|
||||
// (which uses docconv and ignores the context), extract successfully, and return
|
||||
// no error, silently swallowing the cancellation.
|
||||
t.Run("cancelled context propagates through combineExtractor", func(t *testing.T) {
|
||||
data, err := testutils.ReadTestFile("sample-doc.pdf")
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
// Inject pdfExtractor first so it runs before the system documentExtractor.
|
||||
text, err := ExtractWithExtraExtractors(
|
||||
logger,
|
||||
"sample-doc.pdf",
|
||||
bytes.NewReader(data),
|
||||
ExtractSettings{Ctx: ctx}, []Extractor{&pdfExtractor{}})
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
require.Empty(t, text)
|
||||
})
|
||||
}
|
||||
|
||||
type slowExtractor struct {
|
||||
delay time.Duration
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func (se *slowExtractor) Name() string { return "slowExtractor" }
|
||||
|
||||
func (se *slowExtractor) Match(filename string) bool { return true }
|
||||
|
||||
func (se *slowExtractor) Extract(_ context.Context, filename string, r io.ReadSeeker, _ int64) (string, error) {
|
||||
defer func() {
|
||||
if se.done != nil {
|
||||
close(se.done)
|
||||
}
|
||||
}()
|
||||
time.Sleep(se.delay)
|
||||
return "done", nil
|
||||
}
|
||||
|
||||
func TestExtractTimeout(t *testing.T) {
|
||||
logger := mlog.CreateConsoleTestLogger(t)
|
||||
data := []byte("hello world")
|
||||
|
||||
t.Run("aborts a slow extraction once the timeout elapses", func(t *testing.T) {
|
||||
extractDone := make(chan struct{})
|
||||
start := time.Now()
|
||||
text, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), ExtractSettings{Timeout: 50 * time.Millisecond}, []Extractor{&slowExtractor{delay: 500 * time.Millisecond, done: extractDone}})
|
||||
elapsed := time.Since(start)
|
||||
|
||||
require.Error(t, err)
|
||||
require.Empty(t, text)
|
||||
assert.Contains(t, err.Error(), "timed out")
|
||||
assert.Less(t, elapsed, 200*time.Millisecond, "should return shortly after the timeout, not wait for the extraction")
|
||||
|
||||
select {
|
||||
case <-extractDone:
|
||||
case <-time.After(2 * time.Second):
|
||||
require.FailNow(t, "detached extraction did not finish within the deadline")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("returns the result when extraction finishes within the timeout", func(t *testing.T) {
|
||||
text, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), ExtractSettings{Timeout: 5 * time.Second}, []Extractor{&slowExtractor{delay: 10 * time.Millisecond}})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "done", text)
|
||||
})
|
||||
|
||||
t.Run("a zero timeout disables the bound", func(t *testing.T) {
|
||||
text, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), ExtractSettings{Timeout: 0}, []Extractor{&slowExtractor{delay: 10 * time.Millisecond}})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "done", text)
|
||||
})
|
||||
|
||||
t.Run("a panic in the detached extraction is converted to an error", func(t *testing.T) {
|
||||
text, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), ExtractSettings{Timeout: time.Second}, []Extractor{&panickingExtractor{}})
|
||||
require.Error(t, err)
|
||||
require.Empty(t, text)
|
||||
require.Contains(t, err.Error(), "panic")
|
||||
})
|
||||
}
|
||||
|
||||
type panickingExtractor struct{}
|
||||
|
||||
func (pe *panickingExtractor) Name() string { return "panickingExtractor" }
|
||||
|
||||
func (pe *panickingExtractor) Match(filename string) bool { return true }
|
||||
|
||||
func (pe *panickingExtractor) Extract(_ context.Context, filename string, r io.ReadSeeker, _ int64) (string, error) {
|
||||
panic("boom")
|
||||
}
|
||||
|
||||
type recordingCloser struct {
|
||||
closed atomic.Bool
|
||||
}
|
||||
|
||||
func (c *recordingCloser) Close() error {
|
||||
c.closed.Store(true)
|
||||
return nil
|
||||
}
|
||||
|
||||
// blockingExtractor blocks inside Extract until release is closed, simulating a
|
||||
// converter that is still using the reader after an extraction timeout fires.
|
||||
type blockingExtractor struct {
|
||||
started chan struct{}
|
||||
release chan struct{}
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func (be *blockingExtractor) Name() string { return "blockingExtractor" }
|
||||
|
||||
func (be *blockingExtractor) Match(filename string) bool { return true }
|
||||
|
||||
func (be *blockingExtractor) Extract(_ context.Context, filename string, r io.ReadSeeker, _ int64) (string, error) {
|
||||
close(be.started)
|
||||
<-be.release
|
||||
if be.done != nil {
|
||||
close(be.done)
|
||||
}
|
||||
return "done", nil
|
||||
}
|
||||
|
||||
// waitForExtractionSlotsIdle polls until every extraction slot has been
|
||||
// released. be.done only signals that the extractor finished; the detached
|
||||
// goroutine may still be running defers that release its slot.
|
||||
func waitForExtractionSlotsIdle(t *testing.T, limit int) {
|
||||
t.Helper()
|
||||
require.Eventually(t, func() bool {
|
||||
acquired := 0
|
||||
ok := true
|
||||
for range limit {
|
||||
if !tryAcquireExtractionSlot() {
|
||||
ok = false
|
||||
break
|
||||
}
|
||||
acquired++
|
||||
}
|
||||
for range acquired {
|
||||
releaseExtractionSlot()
|
||||
}
|
||||
return ok
|
||||
}, 2*time.Second, 10*time.Millisecond, "extraction slots did not become idle")
|
||||
}
|
||||
|
||||
func TestExtractReaderCloserOwnership(t *testing.T) {
|
||||
logger := mlog.CreateConsoleTestLogger(t)
|
||||
|
||||
t.Run("reader is closed only after the detached extraction finishes on timeout", func(t *testing.T) {
|
||||
closer := &recordingCloser{}
|
||||
be := &blockingExtractor{started: make(chan struct{}), release: make(chan struct{})}
|
||||
settings := ExtractSettings{Timeout: 50 * time.Millisecond, ReaderCloser: closer}
|
||||
|
||||
_, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader([]byte("hi")), settings, []Extractor{be})
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "timed out")
|
||||
|
||||
// Wait (with a deadline) for the detached extraction to start so the
|
||||
// test fails fast instead of hanging if it never runs.
|
||||
select {
|
||||
case <-be.started:
|
||||
case <-time.After(2 * time.Second):
|
||||
require.FailNow(t, "extraction did not start within the deadline")
|
||||
}
|
||||
// The extraction goroutine is still running, so closing the reader now
|
||||
// would race with it; it must stay open.
|
||||
require.False(t, closer.closed.Load(), "reader must not be closed while the extraction goroutine is still running")
|
||||
|
||||
close(be.release)
|
||||
require.Eventually(t, closer.closed.Load, 2*time.Second, 5*time.Millisecond, "reader should be closed once the extraction goroutine finishes")
|
||||
})
|
||||
|
||||
t.Run("reader is closed on the synchronous path", func(t *testing.T) {
|
||||
closer := &recordingCloser{}
|
||||
_, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader([]byte("hi")), ExtractSettings{ReaderCloser: closer}, []Extractor{&slowExtractor{delay: 0}})
|
||||
require.NoError(t, err)
|
||||
require.True(t, closer.closed.Load(), "reader should be closed after synchronous extraction")
|
||||
})
|
||||
}
|
||||
|
||||
func TestExtractWithTimeout(t *testing.T) {
|
||||
content, err := testutils.ReadTestFile("sample-doc.pdf")
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("no-timeout path propagates cancelled context", func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
text, err := extractWithTimeout(
|
||||
&pdfExtractor{},
|
||||
"sample-doc.pdf",
|
||||
bytes.NewReader(content),
|
||||
ExtractSettings{Ctx: ctx})
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
require.Empty(t, text)
|
||||
})
|
||||
|
||||
// context.WithTimeout is derived from the (already cancelled) ExtractSettings.Ctx,
|
||||
// so ctx.Done() fires immediately and the select returns a cancellation error
|
||||
// before extraction can complete.
|
||||
t.Run("goroutine/select path returns cancellation error", func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
text, err := extractWithTimeout(&pdfExtractor{},
|
||||
"sample-doc.pdf",
|
||||
bytes.NewReader(content),
|
||||
ExtractSettings{
|
||||
Ctx: ctx,
|
||||
Timeout: time.Second,
|
||||
})
|
||||
require.Error(t, err)
|
||||
require.Empty(t, text)
|
||||
// Error comes from either the select's ctx.Done() branch ("cancelled") or
|
||||
// directly from pdfExtractor propagating the cancelled context ("canceled").
|
||||
require.Contains(t, strings.ToLower(err.Error()), "cancel")
|
||||
})
|
||||
}
|
||||
|
||||
func TestDocumentMaxFileSize(t *testing.T) {
|
||||
logger := mlog.CreateConsoleTestLogger(t)
|
||||
|
||||
data, err := testutils.ReadTestFile("sample-doc.docx")
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("a generous limit extracts the document content", func(t *testing.T) {
|
||||
text, err := Extract(logger, "sample-doc.docx", bytes.NewReader(data), ExtractSettings{MaxFileSize: 10 * 1024 * 1024})
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, text, "simple")
|
||||
})
|
||||
|
||||
t.Run("a tiny limit prevents the document content from being extracted", func(t *testing.T) {
|
||||
text, err := Extract(logger, "sample-doc.docx", bytes.NewReader(data), ExtractSettings{MaxFileSize: 16})
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, text, "simple")
|
||||
})
|
||||
}
|
||||
|
||||
func TestArchiveMaxFileSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
file string
|
||||
recursion bool
|
||||
limit int64
|
||||
expectContains []string
|
||||
expectMissing []string
|
||||
}{
|
||||
{
|
||||
name: "Zip with recursion and large limit extracts fully",
|
||||
file: "Fake_Team_Import.zip",
|
||||
recursion: true,
|
||||
limit: 10 * 1024 * 1024,
|
||||
expectContains: []string{"purpose", "announcements"},
|
||||
},
|
||||
{
|
||||
name: "Zip with recursion and tiny limit rejects oversized entries",
|
||||
file: "Fake_Team_Import.zip",
|
||||
recursion: true,
|
||||
limit: 1,
|
||||
expectMissing: []string{"purpose", "announcements"},
|
||||
},
|
||||
{
|
||||
name: "Zip with recursion and zero limit means unlimited",
|
||||
file: "Fake_Team_Import.zip",
|
||||
recursion: true,
|
||||
limit: 0,
|
||||
expectContains: []string{"purpose", "announcements"},
|
||||
},
|
||||
{
|
||||
name: "Zip without recursion lists paths regardless of limit",
|
||||
file: "Fake_Team_Import.zip",
|
||||
recursion: false,
|
||||
limit: 1,
|
||||
expectContains: []string{"channels"},
|
||||
},
|
||||
{
|
||||
name: "Tar.gz with recursion and tiny limit rejects oversized entries",
|
||||
file: "Fake_Team_Import.tar.gz",
|
||||
recursion: true,
|
||||
limit: 1,
|
||||
expectMissing: []string{"purpose", "announcements"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
data, err := testutils.ReadTestFile(tc.file)
|
||||
require.NoError(t, err)
|
||||
|
||||
settings := ExtractSettings{ArchiveRecursion: tc.recursion, MaxFileSize: tc.limit}
|
||||
text, err := Extract(mlog.CreateConsoleTestLogger(t), tc.file, bytes.NewReader(data), settings)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, s := range tc.expectContains {
|
||||
assert.Contains(t, text, s)
|
||||
}
|
||||
for _, s := range tc.expectMissing {
|
||||
assert.NotContains(t, text, s)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractConcurrency(t *testing.T) {
|
||||
t.Skip("Skipped due to flakiness — tracked in https://mattermost.atlassian.net/browse/MM-70109")
|
||||
logger := mlog.CreateConsoleTestLogger(t)
|
||||
resetExtractionConcurrencyForTest(1)
|
||||
t.Cleanup(func() {
|
||||
waitForExtractionSlotsIdle(t, 1)
|
||||
resetExtractionConcurrencyForTest(runtime.NumCPU())
|
||||
})
|
||||
|
||||
data := []byte("hello world")
|
||||
|
||||
t.Run("rejects new extractions while the concurrency limit is reached", func(t *testing.T) {
|
||||
be := &blockingExtractor{started: make(chan struct{}), release: make(chan struct{}), done: make(chan struct{})}
|
||||
settings := ExtractSettings{Timeout: 50 * time.Millisecond, ReaderCloser: &recordingCloser{}}
|
||||
|
||||
go func() {
|
||||
_, _ = ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), settings, []Extractor{be})
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-be.started:
|
||||
case <-time.After(2 * time.Second):
|
||||
require.FailNow(t, "first extraction did not start within the deadline")
|
||||
}
|
||||
|
||||
text, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), ExtractSettings{Timeout: time.Second}, []Extractor{&slowExtractor{delay: 0}})
|
||||
require.Error(t, err)
|
||||
require.Empty(t, text)
|
||||
require.Contains(t, err.Error(), "capacity exhausted")
|
||||
|
||||
close(be.release)
|
||||
select {
|
||||
case <-be.done:
|
||||
case <-time.After(2 * time.Second):
|
||||
require.FailNow(t, "detached extraction did not finish within the deadline")
|
||||
}
|
||||
waitForExtractionSlotsIdle(t, 1)
|
||||
})
|
||||
|
||||
t.Run("a timed-out extraction keeps its slot until the detached goroutine finishes", func(t *testing.T) {
|
||||
be := &blockingExtractor{started: make(chan struct{}), release: make(chan struct{}), done: make(chan struct{})}
|
||||
settings := ExtractSettings{Timeout: 50 * time.Millisecond, ReaderCloser: &recordingCloser{}}
|
||||
|
||||
_, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), settings, []Extractor{be})
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "timed out")
|
||||
|
||||
select {
|
||||
case <-be.started:
|
||||
case <-time.After(2 * time.Second):
|
||||
require.FailNow(t, "detached extraction did not start within the deadline")
|
||||
}
|
||||
|
||||
_, err = ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), ExtractSettings{Timeout: time.Second}, []Extractor{&slowExtractor{delay: 0}})
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "capacity exhausted")
|
||||
|
||||
close(be.release)
|
||||
select {
|
||||
case <-be.done:
|
||||
case <-time.After(2 * time.Second):
|
||||
require.FailNow(t, "detached extraction did not finish within the deadline")
|
||||
}
|
||||
waitForExtractionSlotsIdle(t, 1)
|
||||
})
|
||||
}
|
||||
69
server/platform/services/docextractor/documents.go
Normal file
69
server/platform/services/docextractor/documents.go
Normal file
@@ -0,0 +1,69 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package docextractor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"code.sajari.com/docconv/v2"
|
||||
|
||||
"github.com/mattermost/mattermost/server/v8/channels/utils"
|
||||
)
|
||||
|
||||
type documentExtractor struct{}
|
||||
|
||||
var doconvConverterByExtensions = map[string]func(io.Reader) (string, map[string]string, error){
|
||||
"doc": docconv.ConvertDoc,
|
||||
"docx": docconv.ConvertDocx,
|
||||
"pptx": docconv.ConvertPptx,
|
||||
"odt": docconv.ConvertODT,
|
||||
"html": func(r io.Reader) (string, map[string]string, error) { return docconv.ConvertHTML(r, true) },
|
||||
// Temporarily disabled to avoid crashes on malicious .pages files
|
||||
// "pages": docconv.ConvertPages,
|
||||
"rtf": docconv.ConvertRTF,
|
||||
"pdf": docconv.ConvertPDF,
|
||||
}
|
||||
|
||||
func (de *documentExtractor) Name() string {
|
||||
return "documentExtractor"
|
||||
}
|
||||
|
||||
func (de *documentExtractor) Match(filename string) bool {
|
||||
extension := strings.TrimPrefix(path.Ext(filename), ".")
|
||||
_, ok := doconvConverterByExtensions[extension]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (de *documentExtractor) Extract(_ context.Context, filename string, r io.ReadSeeker, maxFileSize int64) (out string, outErr error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
out = ""
|
||||
outErr = errors.New("error extracting document text")
|
||||
}
|
||||
}()
|
||||
|
||||
extension := strings.TrimPrefix(path.Ext(filename), ".")
|
||||
converter, ok := doconvConverterByExtensions[extension]
|
||||
if !ok {
|
||||
return "", errors.New("unknown converter")
|
||||
}
|
||||
|
||||
// Bound how much data the converter is allowed to read so a small upload
|
||||
// cannot expand into an unbounded amount of in-memory work.
|
||||
var reader io.Reader = r
|
||||
if maxFileSize > 0 {
|
||||
reader = utils.NewLimitedReaderWithError(r, maxFileSize)
|
||||
}
|
||||
|
||||
text, _, err := converter(reader)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return text, nil
|
||||
}
|
||||
16
server/platform/services/docextractor/interface.go
Normal file
16
server/platform/services/docextractor/interface.go
Normal file
@@ -0,0 +1,16 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package docextractor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
)
|
||||
|
||||
// Extractors define the interface needed to extract file content
|
||||
type Extractor interface {
|
||||
Match(filename string) bool
|
||||
Extract(ctx context.Context, filename string, file io.ReadSeeker, maxFileSize int64) (string, error)
|
||||
Name() string
|
||||
}
|
||||
90
server/platform/services/docextractor/mmpreview.go
Normal file
90
server/platform/services/docextractor/mmpreview.go
Normal file
@@ -0,0 +1,90 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package docextractor
|
||||
|
||||
// MMPreview is a micro-service to convert from any libreoffice supported
|
||||
// format into a PDF file, and then we use the regular pdf extractor to convert
|
||||
// it into plain text.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type mmPreviewExtractor struct {
|
||||
url string
|
||||
secret string
|
||||
pdfExtractor pdfExtractor
|
||||
}
|
||||
|
||||
var mmpreviewSupportedExtensions = map[string]bool{
|
||||
"ppt": true,
|
||||
"odp": true,
|
||||
"xls": true,
|
||||
"xlsx": true,
|
||||
"ods": true,
|
||||
}
|
||||
|
||||
func newMMPreviewExtractor(url string, secret string, pdfExtractor pdfExtractor) *mmPreviewExtractor {
|
||||
return &mmPreviewExtractor{url: url, secret: secret, pdfExtractor: pdfExtractor}
|
||||
}
|
||||
|
||||
func (mpe *mmPreviewExtractor) Name() string {
|
||||
return "mmPreviewExtractor"
|
||||
}
|
||||
|
||||
func (mpe *mmPreviewExtractor) Match(filename string) bool {
|
||||
extension := strings.TrimPrefix(path.Ext(filename), ".")
|
||||
return mmpreviewSupportedExtensions[extension]
|
||||
}
|
||||
|
||||
func (mpe *mmPreviewExtractor) Extract(ctx context.Context, filename string, file io.ReadSeeker, maxFileSize int64) (string, error) {
|
||||
b, w, err := createMultipartFormData("file", filename, file)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", mpe.url+"/toPDF", &b)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
|
||||
}
|
||||
req.Header.Set("Content-Type", w.FormDataContentType())
|
||||
if mpe.secret != "" {
|
||||
req.Header.Add("Authentication", mpe.secret)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
return "", errors.New("Unable to generate file preview using mmpreview (The server has replied with an error)")
|
||||
}
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "unable to read the response from mmpreview")
|
||||
}
|
||||
return mpe.pdfExtractor.Extract(ctx, filename, bytes.NewReader(data), maxFileSize)
|
||||
}
|
||||
|
||||
func createMultipartFormData(fieldName, fileName string, fileData io.ReadSeeker) (bytes.Buffer, *multipart.Writer, error) {
|
||||
var b bytes.Buffer
|
||||
var err error
|
||||
w := multipart.NewWriter(&b)
|
||||
var fw io.Writer
|
||||
if fw, err = w.CreateFormFile(fieldName, fileName); err != nil {
|
||||
return b, nil, err
|
||||
}
|
||||
if _, err = io.Copy(fw, fileData); err != nil {
|
||||
return b, nil, err
|
||||
}
|
||||
w.Close()
|
||||
return b, w, nil
|
||||
}
|
||||
72
server/platform/services/docextractor/pdf.go
Normal file
72
server/platform/services/docextractor/pdf.go
Normal file
@@ -0,0 +1,72 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package docextractor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/mattermost/pdf"
|
||||
|
||||
"github.com/mattermost/mattermost/server/v8/channels/utils"
|
||||
)
|
||||
|
||||
type pdfExtractor struct{}
|
||||
|
||||
func (pe *pdfExtractor) Name() string {
|
||||
return "pdfExtractor"
|
||||
}
|
||||
|
||||
func (pe *pdfExtractor) Match(filename string) bool {
|
||||
supportedExtensions := map[string]bool{
|
||||
"pdf": true,
|
||||
}
|
||||
extension := strings.TrimPrefix(path.Ext(filename), ".")
|
||||
return supportedExtensions[extension]
|
||||
}
|
||||
|
||||
func (pe *pdfExtractor) Extract(ctx context.Context, filename string, r io.ReadSeeker, maxFileSize int64) (out string, outErr error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
out = ""
|
||||
outErr = errors.New("error extracting pdf text")
|
||||
}
|
||||
}()
|
||||
f, err := os.CreateTemp(os.TempDir(), "pdflib")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error creating temporary file: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
defer os.Remove(f.Name())
|
||||
|
||||
// Bound how much data is copied to disk so a small upload cannot expand
|
||||
// into an unbounded amount of temporary storage.
|
||||
var src io.Reader = r
|
||||
if maxFileSize > 0 {
|
||||
src = utils.NewLimitedReaderWithError(r, maxFileSize)
|
||||
}
|
||||
size, err := io.Copy(f, src)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error copying data into temporary file: %v", err)
|
||||
}
|
||||
|
||||
reader, err := pdf.NewReader(f, size)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
b, err := reader.GetPlainText(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
buf.ReadFrom(b)
|
||||
return buf.String(), nil
|
||||
}
|
||||
92
server/platform/services/docextractor/pdf_test.go
Normal file
92
server/platform/services/docextractor/pdf_test.go
Normal file
@@ -0,0 +1,92 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package docextractor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/mattermost/mattermost/server/v8/channels/utils/testutils"
|
||||
)
|
||||
|
||||
func TestPdfEmptyFile(t *testing.T) {
|
||||
extractor := pdfExtractor{}
|
||||
_, err := extractor.Extract(context.Background(), "test.pdf", bytes.NewReader([]byte{}), 0)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestPdfFile(t *testing.T) {
|
||||
extractor := pdfExtractor{}
|
||||
contentText := "\nThis is a simple document that contains some text."
|
||||
content, err := testutils.ReadTestFile("sample-doc.pdf")
|
||||
require.NoError(t, err)
|
||||
extractedText, err := extractor.Extract(context.Background(), "sample-doc.pdf", bytes.NewReader(content), 0)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, contentText, extractedText)
|
||||
}
|
||||
|
||||
func TestPdfDeeplyNestedObjects(t *testing.T) {
|
||||
// Test for MM-63434
|
||||
var buf bytes.Buffer
|
||||
buf.WriteString("%PDF-1.0\n")
|
||||
for range 10_000 {
|
||||
buf.WriteString("0\n0\nobj\n")
|
||||
}
|
||||
buf.WriteString("startxref\n0\n%%EOF\n")
|
||||
|
||||
extractor := pdfExtractor{}
|
||||
text, err := extractor.Extract(context.Background(), "excessive-nests.pdf", bytes.NewReader(buf.Bytes()), 0)
|
||||
require.Error(t, err)
|
||||
require.Empty(t, text)
|
||||
}
|
||||
|
||||
func TestWrongPdfFile(t *testing.T) {
|
||||
extractor := pdfExtractor{}
|
||||
content, err := testutils.ReadTestFile("sample-doc.docx")
|
||||
require.NoError(t, err)
|
||||
_, err = extractor.Extract(context.Background(), "sample-doc.pdf", bytes.NewReader(content), 0)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestPdfCancelledContext(t *testing.T) {
|
||||
extractor := pdfExtractor{}
|
||||
content, err := testutils.ReadTestFile("sample-doc.pdf")
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
_, err = extractor.Extract(ctx, "sample-doc.pdf", bytes.NewReader(content), 0)
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
}
|
||||
|
||||
func TestPdfMaxFileSize(t *testing.T) {
|
||||
extractor := pdfExtractor{}
|
||||
content, err := testutils.ReadTestFile("sample-doc.pdf")
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, len(content), 16, "fixture must be larger than the tight limit under test")
|
||||
|
||||
t.Run("a zero limit means unlimited and extracts the content", func(t *testing.T) {
|
||||
text, err := extractor.Extract(context.Background(), "sample-doc.pdf", bytes.NewReader(content), 0)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, text, "simple")
|
||||
})
|
||||
|
||||
t.Run("a generous limit extracts the content", func(t *testing.T) {
|
||||
text, err := extractor.Extract(context.Background(), "sample-doc.pdf", bytes.NewReader(content), 10*1024*1024)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, text, "simple")
|
||||
})
|
||||
|
||||
t.Run("a tight limit prevents extraction", func(t *testing.T) {
|
||||
// The reader errors once it reads past the limit, so io.Copy to the
|
||||
// temp file fails and no text is extracted.
|
||||
text, err := extractor.Extract(context.Background(), "sample-doc.pdf", bytes.NewReader(content), 16)
|
||||
require.Error(t, err)
|
||||
require.Empty(t, text)
|
||||
})
|
||||
}
|
||||
56
server/platform/services/docextractor/plain.go
Normal file
56
server/platform/services/docextractor/plain.go
Normal file
@@ -0,0 +1,56 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package docextractor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
type plainExtractor struct{}
|
||||
|
||||
func (pe *plainExtractor) Name() string {
|
||||
return "plainExtractor"
|
||||
}
|
||||
|
||||
func (pe *plainExtractor) Match(filename string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (pe *plainExtractor) Extract(_ context.Context, filename string, r io.ReadSeeker, _ int64) (string, error) {
|
||||
// This detects any visible character plus any whitespace
|
||||
validRanges := append(unicode.GraphicRanges, unicode.White_Space)
|
||||
|
||||
runes := make([]byte, 1024)
|
||||
total, err := r.Read(runes)
|
||||
if err != nil && err != io.EOF {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if total == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
count := 0
|
||||
for {
|
||||
c, size := utf8.DecodeRune(runes[count:])
|
||||
if !unicode.In(c, validRanges...) {
|
||||
return "", nil
|
||||
}
|
||||
if size == 0 {
|
||||
break
|
||||
}
|
||||
count += size
|
||||
|
||||
// subtract the max rune size to prevent accidentally splitted runes at the end of first 1024 bytes
|
||||
if count > total-utf8.UTFMax {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
text, _ := io.ReadAll(r)
|
||||
return string(runes[0:total]) + string(text), nil
|
||||
}
|
||||
54
server/platform/services/docextractor/plain_test.go
Normal file
54
server/platform/services/docextractor/plain_test.go
Normal file
@@ -0,0 +1,54 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package docextractor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPlainEmptyFile(t *testing.T) {
|
||||
extractor := plainExtractor{}
|
||||
extractedText, err := extractor.Extract(context.Background(), "test.txt", bytes.NewReader([]byte{}), 0)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "", extractedText)
|
||||
}
|
||||
|
||||
func TestPlainTextSmallFile(t *testing.T) {
|
||||
extractor := plainExtractor{}
|
||||
content := strings.Repeat("test \n", 5)
|
||||
extractedText, err := extractor.Extract(context.Background(), "test.txt", bytes.NewReader([]byte(content)), 0)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, content, extractedText)
|
||||
}
|
||||
|
||||
func TestPlainBigFile(t *testing.T) {
|
||||
extractor := plainExtractor{}
|
||||
content := strings.Repeat("test \n", 1000)
|
||||
extractedText, err := extractor.Extract(context.Background(), "test.txt", bytes.NewReader([]byte(content)), 0)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, content, extractedText)
|
||||
}
|
||||
|
||||
func TestSmallBinaryFile(t *testing.T) {
|
||||
extractor := plainExtractor{}
|
||||
notUTF8Char := byte(0x7)
|
||||
content := bytes.Repeat([]byte{notUTF8Char}, 1000)
|
||||
extractedText, err := extractor.Extract(context.Background(), "test.bin", bytes.NewReader(content), 0)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "", extractedText)
|
||||
}
|
||||
|
||||
func TestBigBinaryFile(t *testing.T) {
|
||||
extractor := plainExtractor{}
|
||||
notUTF8Char := byte(0x7)
|
||||
content := bytes.Repeat([]byte{notUTF8Char}, 10000)
|
||||
extractedText, err := extractor.Extract(context.Background(), "test.bin", bytes.NewReader(content), 0)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "", extractedText)
|
||||
}
|
||||
Reference in New Issue
Block a user