snapshot of https://github.com/mattermost/mattermost.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
Server CI / Postgres (shard 0) (push) Has been skipped
Server CI / Postgres (shard 3) (push) Has been skipped
BuildEnv Docker Image / build-image (push) Failing after 4s
BuildEnv Docker Image / build-image-fips (push) Failing after 4s
Migration Review & Release Notes / Generate migration review & release notes (push) Failing after 3s
Server CI / Build Environment (push) Failing after 3s
Server CI / Postgres (shard 1) (push) Has been skipped
Tools CI / check-style (mattermost-govet) (push) Failing after 4s
Server CI / Postgres (shard 2) (push) Has been skipped
Tools CI / check-style (mmgotool) (push) Failing after 3s
Server CI / Elasticsearch v8 Compatibility (push) Has been skipped
Server CI / OpenSearch v2 Compatibility (push) Has been skipped
Server CI / Postgres FIPS (shard 0) (push) Has been skipped
Tools CI / Test (mattermost-govet) (push) Failing after 3s
Server CI / Postgres FIPS (shard 1) (push) Has been skipped
Server CI / Postgres FIPS (shard 2) (push) Has been skipped
Tools CI / Test (mmgotool) (push) Failing after 3s
Server CI / Postgres FIPS (shard 3) (push) Has been skipped
Server CI / Run mmctl tests (push) Has been skipped
Server CI / Run mmctl tests (FIPS) (push) Has been skipped
Server CI / Merge Postgres Test Results (push) Has been skipped
Server CI / Merge Postgres FIPS Test Results (push) Has been skipped
Server CI / Check backport migrations (push) Has been skipped
Server CI / Check generated files (push) Has been skipped
Server CI / check-style (push) Has been skipped
Server CI / Vet API (push) Has been skipped
Server CI / Build mattermost server app (push) Has been skipped
Server CI / Server CI Complete (push) Failing after 2s
Server CI / Server CI Artifacts (push) Has been skipped
Server CI / Sentry Upload (push) Has been skipped
Server CI / Server CI Report (push) Failing after 5m1s
Server CI Nightly Race / Race Detector (push) Has been skipped
CodeQL / Analyze (go) (push) Has been cancelled
CodeQL / Analyze (javascript) (push) Has been cancelled
Webapp CI Cache Warm / Warm Webapp node_modules Cache (push) Has been cancelled
E2E CI Cache Warm / Warm E2E npm Registry Cache (push) Has been cancelled
API / build (push) Has been cancelled
Web App CI / check-types (push) Has been cancelled
Web App CI / check-lint (push) Has been cancelled
Web App CI / check-i18n (push) Has been cancelled
Web App CI / check-external-links (push) Has been cancelled
Web App CI / test (platform) (push) Has been cancelled
Web App CI / test (mattermost-redux) (push) Has been cancelled
Web App CI / test (channels shard 1/4) (push) Has been cancelled
Web App CI / test (channels shard 2/4) (push) Has been cancelled
Web App CI / test (channels shard 3/4) (push) Has been cancelled
Web App CI / test (channels shard 4/4) (push) Has been cancelled
Web App CI / upload-coverage (push) Has been cancelled
Web App CI / build (push) Has been cancelled
YAML Lint / yamllint (push) Has been cancelled
Push mirrored docker images / cd/Push mirrored docker images (push) Has been cancelled
docs-cd / Build and deploy docs (push) Has been cancelled
Docs CI / build (push) Has been cancelled
E2E Cache Warm: Cypress deps / Warm Cypress deps cache (push) Has been cancelled
Scorecards supply-chain security / Scorecard analysis (push) Has been cancelled
Server CI Weekly / Postgres FIPS (push) Has been skipped
Server CI Weekly / Run mmctl tests (FIPS) (push) Has been skipped
Server CI Weekly / Postgres with binary parameters (push) Has been skipped
Server CI Weekly / Build Environment (push) Failing after 1s
Server CI Cache Warm / Warm Go Module and Build Cache (push) Failing after 2s
Server CI Nightly Race / Build Environment (push) Failing after 1s

This commit is contained in:
upstream-mirror
2026-09-16 00:29:55 +05:00
commit 240db4e2a7
15309 changed files with 2173406 additions and 0 deletions

View File

@@ -0,0 +1,120 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package docextractor
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"github.com/mholt/archives"
"github.com/mattermost/mattermost/server/v8/channels/utils"
)
type archiveExtractor struct {
SubExtractor Extractor
}
func (ae *archiveExtractor) Name() string {
return "archiveExtractor"
}
func (ae *archiveExtractor) Match(filename string) bool {
_, _, err := archives.Identify(context.Background(), filename, nil)
return err == nil
}
// getExtAlsoTarGz returns the extension of the given file name, special casing .tar.gz.
func getExtAlsoTarGz(name string) string {
if strings.HasSuffix(name, ".tar.gz") {
return ".tar.gz"
}
return filepath.Ext(name)
}
func (ae *archiveExtractor) Extract(ctx context.Context, name string, r io.ReadSeeker, maxFileSize int64) (string, error) {
// MM-65701: Skip 7zip files due to OOM vulnerability in bodgit/sevenzip library
match, _ := (archives.SevenZip{}).Match(context.Background(), name, r)
_, _ = r.Seek(0, io.SeekStart) // Reset reader position after Match reads from stream
if match.ByName || match.ByStream {
return "", nil
}
ext := getExtAlsoTarGz(name)
// Create a temporary file, using `*` control the random component while preserving the extension.
f, err := os.CreateTemp("", "archiver-*"+ext)
if err != nil {
return "", fmt.Errorf("error creating temporary file: %v", err)
}
defer os.Remove(f.Name())
_, err = io.Copy(f, r)
f.Close()
if err != nil {
return "", fmt.Errorf("error copying data into temporary file: %v", err)
}
var text strings.Builder
fsys, err := archives.FileSystem(context.Background(), f.Name(), nil)
if err != nil {
return "", fmt.Errorf("error creating file system: %w", err)
}
err = fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
text.WriteString(path + " ")
if ae.SubExtractor != nil {
filename := filepath.Base(path)
filename = strings.ReplaceAll(filename, "-", " ")
filename = strings.ReplaceAll(filename, ".", " ")
filename = strings.ReplaceAll(filename, ",", " ")
file, err := fsys.Open(path)
if err != nil {
return err
}
defer file.Close()
// Limit the size of decompressed archive entries to prevent
// memory exhaustion from zip bombs or other malicious archives.
var reader io.Reader = file
if maxFileSize > 0 {
reader = utils.NewLimitedReaderWithError(file, maxFileSize)
}
data, err := io.ReadAll(reader)
if err != nil {
return fmt.Errorf("error reading archive entry: %w", err)
}
subtext, extractErr := ae.SubExtractor.Extract(ctx, filename, bytes.NewReader(data), maxFileSize)
if extractErr == nil {
text.WriteString(subtext + " ")
} else if errors.Is(extractErr, context.Canceled) || errors.Is(extractErr, context.DeadlineExceeded) {
return fmt.Errorf("error extracting archive entry: %w", extractErr)
}
}
return nil
})
if err != nil {
return "", err
}
return text.String(), nil
}

View File

@@ -0,0 +1,117 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package docextractor
import (
"archive/zip"
"bytes"
"context"
"io"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestArchiveExtractorSkips7zip(t *testing.T) {
ae := &archiveExtractor{}
t.Run("7zip file with .7z extension returns empty string", func(t *testing.T) {
// Valid 7zip header (minimal)
sevenZipData := []byte{0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c, 0x00, 0x00}
result, err := ae.Extract(context.Background(), "test.7z", bytes.NewReader(sevenZipData), 0)
require.NoError(t, err)
assert.Empty(t, result)
})
t.Run("7zip content with wrong extension is still blocked", func(t *testing.T) {
// 7zip content disguised with .zip extension - should still be blocked via stream detection
sevenZipData := []byte{0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c, 0x00, 0x00}
result, err := ae.Extract(context.Background(), "malicious.zip", bytes.NewReader(sevenZipData), 0)
require.NoError(t, err)
assert.Empty(t, result)
})
t.Run("7zip at offset with .7z extension is blocked via filename", func(t *testing.T) {
junkPrefix := []byte{0x00, 0x00, 0x00, 0x00}
sevenZipSig := []byte{0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c, 0x00, 0x00}
dataWithOffset := append(junkPrefix, sevenZipSig...)
result, err := ae.Extract(context.Background(), "test.7z", bytes.NewReader(dataWithOffset), 0)
require.NoError(t, err)
assert.Empty(t, result)
})
t.Run("7zip at offset with wrong extension fails safely", func(t *testing.T) {
// Edge case: 7zip content at offset with non-.7z extension.
// Our check won't catch it (ByName=false, ByStream=false), but archives.FileSystem
// also won't identify it as 7zip, so it fails to extract rather than triggering OOM.
junkPrefix := []byte{0x00, 0x00, 0x00, 0x00}
sevenZipSig := []byte{0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c, 0x00, 0x00}
dataWithOffset := append(junkPrefix, sevenZipSig...)
_, err := ae.Extract(context.Background(), "malicious.zip", bytes.NewReader(dataWithOffset), 0)
assert.Error(t, err) // fails to extract as any valid archive format
})
}
// contextErrorExtractor fails every extraction with the given context error.
type contextErrorExtractor struct {
err error
}
func (ce *contextErrorExtractor) Name() string {
return "contextErrorExtractor"
}
func (ce *contextErrorExtractor) Match(filename string) bool {
return true
}
func (ce *contextErrorExtractor) Extract(_ context.Context, _ string, _ io.ReadSeeker, _ int64) (string, error) {
return "", ce.err
}
func TestArchiveExtractorErrorOmitsEntryName(t *testing.T) {
// The entry name is transformed before reaching the nested extraction error
// (separators become spaces), so assert on the stem token as well: it
// survives that transformation and would appear in either leaky error.
const entryName = "confidential-customer-list.txt"
const entryStem = "confidential"
var archive bytes.Buffer
zw := zip.NewWriter(&archive)
entry, err := zw.Create(entryName)
require.NoError(t, err)
_, err = entry.Write([]byte(strings.Repeat("a", 1024)))
require.NoError(t, err)
require.NoError(t, zw.Close())
requireNoEntryName := func(t *testing.T, err error) {
t.Helper()
require.Error(t, err)
assert.NotContains(t, err.Error(), entryName)
assert.NotContains(t, err.Error(), entryStem)
}
t.Run("entry read failure", func(t *testing.T) {
ae := &archiveExtractor{SubExtractor: &plainExtractor{}}
// A maxFileSize below the entry size fails the entry read.
_, err := ae.Extract(context.Background(), "archive.zip", bytes.NewReader(archive.Bytes()), 8)
requireNoEntryName(t, err)
})
for name, contextErr := range map[string]error{
"cancelled": context.Canceled,
"deadline exceeded": context.DeadlineExceeded,
} {
t.Run("nested extraction "+name, func(t *testing.T) {
ae := &archiveExtractor{SubExtractor: &contextErrorExtractor{err: contextErr}}
_, err := ae.Extract(context.Background(), "archive.zip", bytes.NewReader(archive.Bytes()), 0)
requireNoEntryName(t, err)
assert.ErrorIs(t, err, contextErr)
})
}
}

View File

@@ -0,0 +1,55 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package docextractor
import (
"context"
"errors"
"io"
"github.com/mattermost/mattermost/server/public/shared/mlog"
)
type combineExtractor struct {
logger mlog.LoggerIFace
SubExtractors []Extractor
}
func (ce *combineExtractor) Name() string {
return "combineExtractor"
}
func (ce *combineExtractor) Add(extractor Extractor) {
ce.SubExtractors = append(ce.SubExtractors, extractor)
}
func (ce *combineExtractor) Match(filename string) bool {
for _, extractor := range ce.SubExtractors {
if extractor.Match(filename) {
return true
}
}
return false
}
func (ce *combineExtractor) Extract(ctx context.Context, filename string, r io.ReadSeeker, maxFileSize int64) (string, error) {
for _, extractor := range ce.SubExtractors {
if extractor.Match(filename) {
r.Seek(0, io.SeekStart)
text, err := extractor.Extract(ctx, filename, r, maxFileSize)
if err != nil {
// Context errors must not be retried: the caller explicitly
// cancelled or timed out, so stop immediately.
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return "", err
}
ce.logger.Warn("Unable to extract file content", mlog.String("extractor", extractor.Name()), mlog.Err(err))
continue
}
return text, nil
}
}
return "", nil
}

View File

@@ -0,0 +1,40 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package docextractor
import (
"runtime"
"sync"
)
// maxConcurrentExtractions caps how many docconv extractions may run at once,
// including detached goroutines that continue after a per-extraction timeout.
var maxConcurrentExtractions = runtime.NumCPU()
var (
extractionSlotsMu sync.Mutex
extractionSlots = make(chan struct{}, maxConcurrentExtractions)
)
func tryAcquireExtractionSlot() bool {
select {
case extractionSlots <- struct{}{}:
return true
default:
return false
}
}
func releaseExtractionSlot() {
<-extractionSlots
}
// resetExtractionConcurrencyForTest reinitializes the extraction slot semaphore.
// It is only intended for use from tests in this package.
func resetExtractionConcurrencyForTest(limit int) {
extractionSlotsMu.Lock()
defer extractionSlotsMu.Unlock()
maxConcurrentExtractions = limit
extractionSlots = make(chan struct{}, limit)
}

View File

@@ -0,0 +1,149 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package docextractor
import (
"context"
"errors"
"fmt"
"io"
"time"
"github.com/mattermost/mattermost/server/public/shared/mlog"
)
// ExtractSettings defines the features enabled/disable during the document text extraction.
type ExtractSettings struct {
// Ctx is the parent context for the extraction. Cancellation propagates
// to context-aware extractors (e.g. the Go-native PDF extractor stops at
// the next page boundary). If nil, context.Background() is used.
Ctx context.Context
ArchiveRecursion bool
MaxFileSize int64
MMPreviewURL string
MMPreviewSecret string
// Timeout bounds how long a caller waits for a single extraction. When
// set, a child context with this deadline is derived from Ctx and passed
// to the extractor. Context-aware extractors (pdfExtractor) honour it
// and stop at the next page boundary; others run to completion on a
// detached goroutine but no longer hold the caller's worker slot. A
// global cap on concurrent extractions (including those detached
// goroutines) prevents sustained uploads from accumulating unbounded work.
Timeout time.Duration
// ReaderCloser, when set, transfers ownership of closing the input reader
// to this package. It is closed only after extraction has actually
// finished reading. This matters with Timeout set: on timeout the caller
// returns while the converter may still be reading on a detached
// goroutine, so the caller must NOT close the reader itself or it would
// race with (and close the file out from under) that goroutine.
ReaderCloser io.Closer
}
// Extract extract the text from a document using the system default extractors
func Extract(logger mlog.LoggerIFace, filename string, r io.ReadSeeker, settings ExtractSettings) (string, error) {
return ExtractWithExtraExtractors(logger, filename, r, settings, []Extractor{})
}
// ExtractWithExtraExtractors extract the text from a document using the provided extractors beside the system default extractors.
func ExtractWithExtraExtractors(logger mlog.LoggerIFace, filename string, r io.ReadSeeker, settings ExtractSettings, extraExtractors []Extractor) (string, error) {
enabledExtractors := &combineExtractor{
logger: logger,
}
for _, extraExtractor := range extraExtractors {
enabledExtractors.Add(extraExtractor)
}
enabledExtractors.Add(&documentExtractor{})
enabledExtractors.Add(&pdfExtractor{})
if settings.ArchiveRecursion {
enabledExtractors.Add(&archiveExtractor{SubExtractor: enabledExtractors})
} else {
enabledExtractors.Add(&archiveExtractor{})
}
if settings.MMPreviewURL != "" {
enabledExtractors.Add(newMMPreviewExtractor(settings.MMPreviewURL, settings.MMPreviewSecret, pdfExtractor{}))
}
enabledExtractors.Add(&plainExtractor{})
if enabledExtractors.Match(filename) {
return extractWithTimeout(enabledExtractors, filename, r, settings)
}
// No extractor matched, so nothing will read r; close it here since
// extractWithTimeout (which otherwise owns the close) is never reached.
if settings.ReaderCloser != nil {
settings.ReaderCloser.Close()
}
return "", nil
}
// extractWithTimeout runs the extraction under a context derived from
// settings.Ctx. When settings.Timeout > 0 a deadline is added; on expiry the
// caller is released and the extraction runs on a detached goroutine.
// Context-aware extractors (pdfExtractor) honour cancellation at page
// boundaries and stop promptly; others run to completion in the background.
// A global concurrency cap applies to every in-flight extraction, including
// detached goroutines, so timed-out work cannot accumulate beyond the limit.
func extractWithTimeout(e Extractor, filename string, r io.ReadSeeker, settings ExtractSettings) (string, error) {
if !tryAcquireExtractionSlot() {
if settings.ReaderCloser != nil {
settings.ReaderCloser.Close()
}
return "", fmt.Errorf("document text extraction capacity exhausted (%d concurrent extractions)", maxConcurrentExtractions)
}
ctx := settings.Ctx
if ctx == nil {
ctx = context.Background()
}
if settings.Timeout <= 0 {
defer releaseExtractionSlot()
if settings.ReaderCloser != nil {
defer settings.ReaderCloser.Close()
}
return e.Extract(ctx, filename, r, settings.MaxFileSize)
}
ctx, cancel := context.WithTimeout(ctx, settings.Timeout)
defer cancel()
type extractResult struct {
text string
err error
}
resultCh := make(chan extractResult, 1)
go func() {
defer releaseExtractionSlot()
// This goroutine owns the reader for the lifetime of the extraction.
// After the timeout fires the caller returns, but the converter may
// still be reading r here, so the reader is closed only once this
// goroutine is done with it - never by the caller.
if settings.ReaderCloser != nil {
defer settings.ReaderCloser.Close()
}
// This goroutine is detached, so an unrecovered panic in an extractor
// would crash the whole server. Convert it into an error instead.
// resultCh is buffered (cap 1), so this send never blocks even if the
// caller already timed out and stopped receiving.
defer func() {
if rec := recover(); rec != nil {
resultCh <- extractResult{err: fmt.Errorf("panic during document text extraction: %v", rec)}
}
}()
text, err := e.Extract(ctx, filename, r, settings.MaxFileSize)
resultCh <- extractResult{text: text, err: err}
}()
select {
case res := <-resultCh:
return res.text, res.err
case <-ctx.Done():
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
return "", fmt.Errorf("document text extraction timed out after %s", settings.Timeout)
}
return "", fmt.Errorf("document text extraction cancelled: %w", ctx.Err())
}
}

View File

@@ -0,0 +1,608 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package docextractor
import (
"bytes"
"context"
"errors"
"io"
"runtime"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/v8/channels/utils/testutils"
)
func TestExtract(t *testing.T) {
logger := mlog.CreateConsoleTestLogger(t)
testCases := []struct {
Name string
TestFileName string
Settings ExtractSettings
Contains []string
NotContains []string
ExpectError bool
}{
{
"Plain text file",
"test-markdown-basics.md",
ExtractSettings{},
[]string{"followed", "separated", "Basic"},
[]string{},
false,
},
{
"Plain small text file",
"test-hashtags.md",
ExtractSettings{},
[]string{"should", "render", "strings"},
[]string{},
false,
},
{
"Zip file without recursion",
"Fake_Team_Import.zip",
ExtractSettings{},
[]string{"users", "channels", "general"},
[]string{"purpose", "announcements"},
false,
},
{
"Zip file with recursion",
"Fake_Team_Import.zip",
ExtractSettings{ArchiveRecursion: true},
[]string{"users", "channels", "general", "purpose", "announcements"},
[]string{},
false,
},
{
"Rar file without recursion",
"Fake_Team_Import.rar",
ExtractSettings{},
[]string{"users", "channels", "general"},
[]string{"purpose", "announcements"},
false,
},
{
"Rar file with recursion",
"Fake_Team_Import.rar",
ExtractSettings{ArchiveRecursion: true},
[]string{"users", "channels", "general", "purpose", "announcements"},
[]string{},
false,
},
{
"Tar.gz file without recursion",
"Fake_Team_Import.tar.gz",
ExtractSettings{},
[]string{"users", "channels", "general"},
[]string{"purpose", "announcements"},
false,
},
{
"Tar.gz file with recursion",
"Fake_Team_Import.tar.gz",
ExtractSettings{ArchiveRecursion: true},
[]string{"users", "channels", "general", "purpose", "announcements"},
[]string{},
false,
},
{
"Pdf file",
"sample-doc.pdf",
ExtractSettings{},
[]string{"simple", "document", "contains"},
[]string{},
false,
},
{
"Docx file",
"sample-doc.docx",
ExtractSettings{},
[]string{"simple", "document", "contains"},
[]string{},
false,
},
{
"Odt file",
"sample-doc.odt",
ExtractSettings{},
[]string{"simple", "document", "contains"},
[]string{},
false,
},
{
"Pptx file",
"sample-doc.pptx",
ExtractSettings{},
[]string{"simple", "document", "contains"},
[]string{},
false,
},
}
for _, tc := range testCases {
t.Run(tc.Name, func(t *testing.T) {
data, err := testutils.ReadTestFile(tc.TestFileName)
require.NoError(t, err)
text, err := Extract(logger, tc.TestFileName, bytes.NewReader(data), tc.Settings)
if tc.ExpectError {
require.Error(t, err)
} else {
require.NoError(t, err)
for _, expectedString := range tc.Contains {
assert.Contains(t, text, expectedString)
}
for _, notExpectedString := range tc.NotContains {
assert.NotContains(t, text, notExpectedString)
}
}
})
}
t.Run("Unsupported binary file", func(t *testing.T) {
data, err := testutils.ReadTestFile("testjpg.jpg")
require.NoError(t, err)
text, err := Extract(logger, "testjpg.jpg", bytes.NewReader(data), ExtractSettings{})
require.NoError(t, err)
require.Equal(t, "", text)
})
t.Run("Wrong docx extension", func(t *testing.T) {
data, err := testutils.ReadTestFile("sample-doc.pdf")
require.NoError(t, err)
text, err := Extract(logger, "sample-doc.docx", bytes.NewReader(data), ExtractSettings{})
require.NoError(t, err)
require.Equal(t, "", text)
})
}
type customTestPdfExtractor struct{}
func (te *customTestPdfExtractor) Name() string {
return "customTestPdfExtractor"
}
func (te *customTestPdfExtractor) Match(filename string) bool {
return strings.HasSuffix(filename, ".pdf")
}
func (te *customTestPdfExtractor) Extract(_ context.Context, filename string, r io.ReadSeeker, _ int64) (string, error) {
return "this is a text generated content", nil
}
type failingExtractor struct{}
func (te *failingExtractor) Name() string {
return "failingExtractor"
}
func (te *failingExtractor) Match(filename string) bool {
return true
}
func (te *failingExtractor) Extract(_ context.Context, filename string, r io.ReadSeeker, _ int64) (string, error) {
return "", errors.New("this always fail")
}
func TestExtractWithExtraExtractors(t *testing.T) {
logger := mlog.CreateConsoleTestLogger(t)
t.Run("override existing extractor", func(t *testing.T) {
data, err := testutils.ReadTestFile("sample-doc.pdf")
require.NoError(t, err)
text, err := ExtractWithExtraExtractors(logger, "sample-doc.pdf", bytes.NewReader(data), ExtractSettings{}, []Extractor{&customTestPdfExtractor{}})
require.NoError(t, err)
require.Equal(t, text, "this is a text generated content")
})
t.Run("failing extractor", func(t *testing.T) {
data, err := testutils.ReadTestFile("sample-doc.pdf")
require.NoError(t, err)
text, err := ExtractWithExtraExtractors(logger, "sample-doc.pdf", bytes.NewReader(data), ExtractSettings{}, []Extractor{&failingExtractor{}})
require.NoError(t, err)
assert.Contains(t, text, "simple")
assert.Contains(t, text, "document")
assert.Contains(t, text, "contains")
})
t.Run("cancelled context aborts extraction", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
text, err := ExtractWithExtraExtractors(
logger,
"file.txt",
bytes.NewReader([]byte("hello world")),
ExtractSettings{
Ctx: ctx,
Timeout: time.Second,
},
[]Extractor{&slowExtractor{delay: 10 * time.Second}})
require.Error(t, err)
require.Empty(t, text)
require.Contains(t, err.Error(), "cancelled")
})
// Without context propagation a cancelled context would reach documentExtractor
// (which uses docconv and ignores the context), extract successfully, and return
// no error, silently swallowing the cancellation.
t.Run("cancelled context propagates through combineExtractor", func(t *testing.T) {
data, err := testutils.ReadTestFile("sample-doc.pdf")
require.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background())
cancel()
// Inject pdfExtractor first so it runs before the system documentExtractor.
text, err := ExtractWithExtraExtractors(
logger,
"sample-doc.pdf",
bytes.NewReader(data),
ExtractSettings{Ctx: ctx}, []Extractor{&pdfExtractor{}})
require.ErrorIs(t, err, context.Canceled)
require.Empty(t, text)
})
}
type slowExtractor struct {
delay time.Duration
done chan struct{}
}
func (se *slowExtractor) Name() string { return "slowExtractor" }
func (se *slowExtractor) Match(filename string) bool { return true }
func (se *slowExtractor) Extract(_ context.Context, filename string, r io.ReadSeeker, _ int64) (string, error) {
defer func() {
if se.done != nil {
close(se.done)
}
}()
time.Sleep(se.delay)
return "done", nil
}
func TestExtractTimeout(t *testing.T) {
logger := mlog.CreateConsoleTestLogger(t)
data := []byte("hello world")
t.Run("aborts a slow extraction once the timeout elapses", func(t *testing.T) {
extractDone := make(chan struct{})
start := time.Now()
text, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), ExtractSettings{Timeout: 50 * time.Millisecond}, []Extractor{&slowExtractor{delay: 500 * time.Millisecond, done: extractDone}})
elapsed := time.Since(start)
require.Error(t, err)
require.Empty(t, text)
assert.Contains(t, err.Error(), "timed out")
assert.Less(t, elapsed, 200*time.Millisecond, "should return shortly after the timeout, not wait for the extraction")
select {
case <-extractDone:
case <-time.After(2 * time.Second):
require.FailNow(t, "detached extraction did not finish within the deadline")
}
})
t.Run("returns the result when extraction finishes within the timeout", func(t *testing.T) {
text, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), ExtractSettings{Timeout: 5 * time.Second}, []Extractor{&slowExtractor{delay: 10 * time.Millisecond}})
require.NoError(t, err)
require.Equal(t, "done", text)
})
t.Run("a zero timeout disables the bound", func(t *testing.T) {
text, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), ExtractSettings{Timeout: 0}, []Extractor{&slowExtractor{delay: 10 * time.Millisecond}})
require.NoError(t, err)
require.Equal(t, "done", text)
})
t.Run("a panic in the detached extraction is converted to an error", func(t *testing.T) {
text, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), ExtractSettings{Timeout: time.Second}, []Extractor{&panickingExtractor{}})
require.Error(t, err)
require.Empty(t, text)
require.Contains(t, err.Error(), "panic")
})
}
type panickingExtractor struct{}
func (pe *panickingExtractor) Name() string { return "panickingExtractor" }
func (pe *panickingExtractor) Match(filename string) bool { return true }
func (pe *panickingExtractor) Extract(_ context.Context, filename string, r io.ReadSeeker, _ int64) (string, error) {
panic("boom")
}
type recordingCloser struct {
closed atomic.Bool
}
func (c *recordingCloser) Close() error {
c.closed.Store(true)
return nil
}
// blockingExtractor blocks inside Extract until release is closed, simulating a
// converter that is still using the reader after an extraction timeout fires.
type blockingExtractor struct {
started chan struct{}
release chan struct{}
done chan struct{}
}
func (be *blockingExtractor) Name() string { return "blockingExtractor" }
func (be *blockingExtractor) Match(filename string) bool { return true }
func (be *blockingExtractor) Extract(_ context.Context, filename string, r io.ReadSeeker, _ int64) (string, error) {
close(be.started)
<-be.release
if be.done != nil {
close(be.done)
}
return "done", nil
}
// waitForExtractionSlotsIdle polls until every extraction slot has been
// released. be.done only signals that the extractor finished; the detached
// goroutine may still be running defers that release its slot.
func waitForExtractionSlotsIdle(t *testing.T, limit int) {
t.Helper()
require.Eventually(t, func() bool {
acquired := 0
ok := true
for range limit {
if !tryAcquireExtractionSlot() {
ok = false
break
}
acquired++
}
for range acquired {
releaseExtractionSlot()
}
return ok
}, 2*time.Second, 10*time.Millisecond, "extraction slots did not become idle")
}
func TestExtractReaderCloserOwnership(t *testing.T) {
logger := mlog.CreateConsoleTestLogger(t)
t.Run("reader is closed only after the detached extraction finishes on timeout", func(t *testing.T) {
closer := &recordingCloser{}
be := &blockingExtractor{started: make(chan struct{}), release: make(chan struct{})}
settings := ExtractSettings{Timeout: 50 * time.Millisecond, ReaderCloser: closer}
_, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader([]byte("hi")), settings, []Extractor{be})
require.Error(t, err)
require.Contains(t, err.Error(), "timed out")
// Wait (with a deadline) for the detached extraction to start so the
// test fails fast instead of hanging if it never runs.
select {
case <-be.started:
case <-time.After(2 * time.Second):
require.FailNow(t, "extraction did not start within the deadline")
}
// The extraction goroutine is still running, so closing the reader now
// would race with it; it must stay open.
require.False(t, closer.closed.Load(), "reader must not be closed while the extraction goroutine is still running")
close(be.release)
require.Eventually(t, closer.closed.Load, 2*time.Second, 5*time.Millisecond, "reader should be closed once the extraction goroutine finishes")
})
t.Run("reader is closed on the synchronous path", func(t *testing.T) {
closer := &recordingCloser{}
_, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader([]byte("hi")), ExtractSettings{ReaderCloser: closer}, []Extractor{&slowExtractor{delay: 0}})
require.NoError(t, err)
require.True(t, closer.closed.Load(), "reader should be closed after synchronous extraction")
})
}
func TestExtractWithTimeout(t *testing.T) {
content, err := testutils.ReadTestFile("sample-doc.pdf")
require.NoError(t, err)
t.Run("no-timeout path propagates cancelled context", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
text, err := extractWithTimeout(
&pdfExtractor{},
"sample-doc.pdf",
bytes.NewReader(content),
ExtractSettings{Ctx: ctx})
require.ErrorIs(t, err, context.Canceled)
require.Empty(t, text)
})
// context.WithTimeout is derived from the (already cancelled) ExtractSettings.Ctx,
// so ctx.Done() fires immediately and the select returns a cancellation error
// before extraction can complete.
t.Run("goroutine/select path returns cancellation error", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
text, err := extractWithTimeout(&pdfExtractor{},
"sample-doc.pdf",
bytes.NewReader(content),
ExtractSettings{
Ctx: ctx,
Timeout: time.Second,
})
require.Error(t, err)
require.Empty(t, text)
// Error comes from either the select's ctx.Done() branch ("cancelled") or
// directly from pdfExtractor propagating the cancelled context ("canceled").
require.Contains(t, strings.ToLower(err.Error()), "cancel")
})
}
func TestDocumentMaxFileSize(t *testing.T) {
logger := mlog.CreateConsoleTestLogger(t)
data, err := testutils.ReadTestFile("sample-doc.docx")
require.NoError(t, err)
t.Run("a generous limit extracts the document content", func(t *testing.T) {
text, err := Extract(logger, "sample-doc.docx", bytes.NewReader(data), ExtractSettings{MaxFileSize: 10 * 1024 * 1024})
require.NoError(t, err)
assert.Contains(t, text, "simple")
})
t.Run("a tiny limit prevents the document content from being extracted", func(t *testing.T) {
text, err := Extract(logger, "sample-doc.docx", bytes.NewReader(data), ExtractSettings{MaxFileSize: 16})
require.NoError(t, err)
assert.NotContains(t, text, "simple")
})
}
func TestArchiveMaxFileSize(t *testing.T) {
t.Parallel()
tests := []struct {
name string
file string
recursion bool
limit int64
expectContains []string
expectMissing []string
}{
{
name: "Zip with recursion and large limit extracts fully",
file: "Fake_Team_Import.zip",
recursion: true,
limit: 10 * 1024 * 1024,
expectContains: []string{"purpose", "announcements"},
},
{
name: "Zip with recursion and tiny limit rejects oversized entries",
file: "Fake_Team_Import.zip",
recursion: true,
limit: 1,
expectMissing: []string{"purpose", "announcements"},
},
{
name: "Zip with recursion and zero limit means unlimited",
file: "Fake_Team_Import.zip",
recursion: true,
limit: 0,
expectContains: []string{"purpose", "announcements"},
},
{
name: "Zip without recursion lists paths regardless of limit",
file: "Fake_Team_Import.zip",
recursion: false,
limit: 1,
expectContains: []string{"channels"},
},
{
name: "Tar.gz with recursion and tiny limit rejects oversized entries",
file: "Fake_Team_Import.tar.gz",
recursion: true,
limit: 1,
expectMissing: []string{"purpose", "announcements"},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
data, err := testutils.ReadTestFile(tc.file)
require.NoError(t, err)
settings := ExtractSettings{ArchiveRecursion: tc.recursion, MaxFileSize: tc.limit}
text, err := Extract(mlog.CreateConsoleTestLogger(t), tc.file, bytes.NewReader(data), settings)
require.NoError(t, err)
for _, s := range tc.expectContains {
assert.Contains(t, text, s)
}
for _, s := range tc.expectMissing {
assert.NotContains(t, text, s)
}
})
}
}
func TestExtractConcurrency(t *testing.T) {
t.Skip("Skipped due to flakiness — tracked in https://mattermost.atlassian.net/browse/MM-70109")
logger := mlog.CreateConsoleTestLogger(t)
resetExtractionConcurrencyForTest(1)
t.Cleanup(func() {
waitForExtractionSlotsIdle(t, 1)
resetExtractionConcurrencyForTest(runtime.NumCPU())
})
data := []byte("hello world")
t.Run("rejects new extractions while the concurrency limit is reached", func(t *testing.T) {
be := &blockingExtractor{started: make(chan struct{}), release: make(chan struct{}), done: make(chan struct{})}
settings := ExtractSettings{Timeout: 50 * time.Millisecond, ReaderCloser: &recordingCloser{}}
go func() {
_, _ = ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), settings, []Extractor{be})
}()
select {
case <-be.started:
case <-time.After(2 * time.Second):
require.FailNow(t, "first extraction did not start within the deadline")
}
text, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), ExtractSettings{Timeout: time.Second}, []Extractor{&slowExtractor{delay: 0}})
require.Error(t, err)
require.Empty(t, text)
require.Contains(t, err.Error(), "capacity exhausted")
close(be.release)
select {
case <-be.done:
case <-time.After(2 * time.Second):
require.FailNow(t, "detached extraction did not finish within the deadline")
}
waitForExtractionSlotsIdle(t, 1)
})
t.Run("a timed-out extraction keeps its slot until the detached goroutine finishes", func(t *testing.T) {
be := &blockingExtractor{started: make(chan struct{}), release: make(chan struct{}), done: make(chan struct{})}
settings := ExtractSettings{Timeout: 50 * time.Millisecond, ReaderCloser: &recordingCloser{}}
_, err := ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), settings, []Extractor{be})
require.Error(t, err)
require.Contains(t, err.Error(), "timed out")
select {
case <-be.started:
case <-time.After(2 * time.Second):
require.FailNow(t, "detached extraction did not start within the deadline")
}
_, err = ExtractWithExtraExtractors(logger, "file.txt", bytes.NewReader(data), ExtractSettings{Timeout: time.Second}, []Extractor{&slowExtractor{delay: 0}})
require.Error(t, err)
require.Contains(t, err.Error(), "capacity exhausted")
close(be.release)
select {
case <-be.done:
case <-time.After(2 * time.Second):
require.FailNow(t, "detached extraction did not finish within the deadline")
}
waitForExtractionSlotsIdle(t, 1)
})
}

View File

@@ -0,0 +1,69 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package docextractor
import (
"context"
"errors"
"io"
"path"
"strings"
"code.sajari.com/docconv/v2"
"github.com/mattermost/mattermost/server/v8/channels/utils"
)
type documentExtractor struct{}
var doconvConverterByExtensions = map[string]func(io.Reader) (string, map[string]string, error){
"doc": docconv.ConvertDoc,
"docx": docconv.ConvertDocx,
"pptx": docconv.ConvertPptx,
"odt": docconv.ConvertODT,
"html": func(r io.Reader) (string, map[string]string, error) { return docconv.ConvertHTML(r, true) },
// Temporarily disabled to avoid crashes on malicious .pages files
// "pages": docconv.ConvertPages,
"rtf": docconv.ConvertRTF,
"pdf": docconv.ConvertPDF,
}
func (de *documentExtractor) Name() string {
return "documentExtractor"
}
func (de *documentExtractor) Match(filename string) bool {
extension := strings.TrimPrefix(path.Ext(filename), ".")
_, ok := doconvConverterByExtensions[extension]
return ok
}
func (de *documentExtractor) Extract(_ context.Context, filename string, r io.ReadSeeker, maxFileSize int64) (out string, outErr error) {
defer func() {
if r := recover(); r != nil {
out = ""
outErr = errors.New("error extracting document text")
}
}()
extension := strings.TrimPrefix(path.Ext(filename), ".")
converter, ok := doconvConverterByExtensions[extension]
if !ok {
return "", errors.New("unknown converter")
}
// Bound how much data the converter is allowed to read so a small upload
// cannot expand into an unbounded amount of in-memory work.
var reader io.Reader = r
if maxFileSize > 0 {
reader = utils.NewLimitedReaderWithError(r, maxFileSize)
}
text, _, err := converter(reader)
if err != nil {
return "", err
}
return text, nil
}

View File

@@ -0,0 +1,16 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package docextractor
import (
"context"
"io"
)
// Extractors define the interface needed to extract file content
type Extractor interface {
Match(filename string) bool
Extract(ctx context.Context, filename string, file io.ReadSeeker, maxFileSize int64) (string, error)
Name() string
}

View File

@@ -0,0 +1,90 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package docextractor
// MMPreview is a micro-service to convert from any libreoffice supported
// format into a PDF file, and then we use the regular pdf extractor to convert
// it into plain text.
import (
"bytes"
"context"
"io"
"mime/multipart"
"net/http"
"path"
"strings"
"github.com/pkg/errors"
)
type mmPreviewExtractor struct {
url string
secret string
pdfExtractor pdfExtractor
}
var mmpreviewSupportedExtensions = map[string]bool{
"ppt": true,
"odp": true,
"xls": true,
"xlsx": true,
"ods": true,
}
func newMMPreviewExtractor(url string, secret string, pdfExtractor pdfExtractor) *mmPreviewExtractor {
return &mmPreviewExtractor{url: url, secret: secret, pdfExtractor: pdfExtractor}
}
func (mpe *mmPreviewExtractor) Name() string {
return "mmPreviewExtractor"
}
func (mpe *mmPreviewExtractor) Match(filename string) bool {
extension := strings.TrimPrefix(path.Ext(filename), ".")
return mmpreviewSupportedExtensions[extension]
}
func (mpe *mmPreviewExtractor) Extract(ctx context.Context, filename string, file io.ReadSeeker, maxFileSize int64) (string, error) {
b, w, err := createMultipartFormData("file", filename, file)
if err != nil {
return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
}
req, err := http.NewRequestWithContext(ctx, "POST", mpe.url+"/toPDF", &b)
if err != nil {
return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
}
req.Header.Set("Content-Type", w.FormDataContentType())
if mpe.secret != "" {
req.Header.Add("Authentication", mpe.secret)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return "", errors.New("Unable to generate file preview using mmpreview (The server has replied with an error)")
}
data, err := io.ReadAll(resp.Body)
if err != nil {
return "", errors.Wrap(err, "unable to read the response from mmpreview")
}
return mpe.pdfExtractor.Extract(ctx, filename, bytes.NewReader(data), maxFileSize)
}
func createMultipartFormData(fieldName, fileName string, fileData io.ReadSeeker) (bytes.Buffer, *multipart.Writer, error) {
var b bytes.Buffer
var err error
w := multipart.NewWriter(&b)
var fw io.Writer
if fw, err = w.CreateFormFile(fieldName, fileName); err != nil {
return b, nil, err
}
if _, err = io.Copy(fw, fileData); err != nil {
return b, nil, err
}
w.Close()
return b, w, nil
}

View File

@@ -0,0 +1,72 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package docextractor
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"path"
"strings"
"github.com/mattermost/pdf"
"github.com/mattermost/mattermost/server/v8/channels/utils"
)
type pdfExtractor struct{}
func (pe *pdfExtractor) Name() string {
return "pdfExtractor"
}
func (pe *pdfExtractor) Match(filename string) bool {
supportedExtensions := map[string]bool{
"pdf": true,
}
extension := strings.TrimPrefix(path.Ext(filename), ".")
return supportedExtensions[extension]
}
func (pe *pdfExtractor) Extract(ctx context.Context, filename string, r io.ReadSeeker, maxFileSize int64) (out string, outErr error) {
defer func() {
if r := recover(); r != nil {
out = ""
outErr = errors.New("error extracting pdf text")
}
}()
f, err := os.CreateTemp(os.TempDir(), "pdflib")
if err != nil {
return "", fmt.Errorf("error creating temporary file: %v", err)
}
defer f.Close()
defer os.Remove(f.Name())
// Bound how much data is copied to disk so a small upload cannot expand
// into an unbounded amount of temporary storage.
var src io.Reader = r
if maxFileSize > 0 {
src = utils.NewLimitedReaderWithError(r, maxFileSize)
}
size, err := io.Copy(f, src)
if err != nil {
return "", fmt.Errorf("error copying data into temporary file: %v", err)
}
reader, err := pdf.NewReader(f, size)
if err != nil {
return "", err
}
var buf bytes.Buffer
b, err := reader.GetPlainText(ctx)
if err != nil {
return "", err
}
buf.ReadFrom(b)
return buf.String(), nil
}

View File

@@ -0,0 +1,92 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package docextractor
import (
"bytes"
"context"
"testing"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/v8/channels/utils/testutils"
)
func TestPdfEmptyFile(t *testing.T) {
extractor := pdfExtractor{}
_, err := extractor.Extract(context.Background(), "test.pdf", bytes.NewReader([]byte{}), 0)
require.Error(t, err)
}
func TestPdfFile(t *testing.T) {
extractor := pdfExtractor{}
contentText := "\nThis is a simple document that contains some text."
content, err := testutils.ReadTestFile("sample-doc.pdf")
require.NoError(t, err)
extractedText, err := extractor.Extract(context.Background(), "sample-doc.pdf", bytes.NewReader(content), 0)
require.NoError(t, err)
require.Equal(t, contentText, extractedText)
}
func TestPdfDeeplyNestedObjects(t *testing.T) {
// Test for MM-63434
var buf bytes.Buffer
buf.WriteString("%PDF-1.0\n")
for range 10_000 {
buf.WriteString("0\n0\nobj\n")
}
buf.WriteString("startxref\n0\n%%EOF\n")
extractor := pdfExtractor{}
text, err := extractor.Extract(context.Background(), "excessive-nests.pdf", bytes.NewReader(buf.Bytes()), 0)
require.Error(t, err)
require.Empty(t, text)
}
func TestWrongPdfFile(t *testing.T) {
extractor := pdfExtractor{}
content, err := testutils.ReadTestFile("sample-doc.docx")
require.NoError(t, err)
_, err = extractor.Extract(context.Background(), "sample-doc.pdf", bytes.NewReader(content), 0)
require.Error(t, err)
}
func TestPdfCancelledContext(t *testing.T) {
extractor := pdfExtractor{}
content, err := testutils.ReadTestFile("sample-doc.pdf")
require.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err = extractor.Extract(ctx, "sample-doc.pdf", bytes.NewReader(content), 0)
require.ErrorIs(t, err, context.Canceled)
}
func TestPdfMaxFileSize(t *testing.T) {
extractor := pdfExtractor{}
content, err := testutils.ReadTestFile("sample-doc.pdf")
require.NoError(t, err)
require.Greater(t, len(content), 16, "fixture must be larger than the tight limit under test")
t.Run("a zero limit means unlimited and extracts the content", func(t *testing.T) {
text, err := extractor.Extract(context.Background(), "sample-doc.pdf", bytes.NewReader(content), 0)
require.NoError(t, err)
require.Contains(t, text, "simple")
})
t.Run("a generous limit extracts the content", func(t *testing.T) {
text, err := extractor.Extract(context.Background(), "sample-doc.pdf", bytes.NewReader(content), 10*1024*1024)
require.NoError(t, err)
require.Contains(t, text, "simple")
})
t.Run("a tight limit prevents extraction", func(t *testing.T) {
// The reader errors once it reads past the limit, so io.Copy to the
// temp file fails and no text is extracted.
text, err := extractor.Extract(context.Background(), "sample-doc.pdf", bytes.NewReader(content), 16)
require.Error(t, err)
require.Empty(t, text)
})
}

View File

@@ -0,0 +1,56 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package docextractor
import (
"context"
"io"
"unicode"
"unicode/utf8"
)
type plainExtractor struct{}
func (pe *plainExtractor) Name() string {
return "plainExtractor"
}
func (pe *plainExtractor) Match(filename string) bool {
return true
}
func (pe *plainExtractor) Extract(_ context.Context, filename string, r io.ReadSeeker, _ int64) (string, error) {
// This detects any visible character plus any whitespace
validRanges := append(unicode.GraphicRanges, unicode.White_Space)
runes := make([]byte, 1024)
total, err := r.Read(runes)
if err != nil && err != io.EOF {
return "", err
}
if total == 0 {
return "", nil
}
count := 0
for {
c, size := utf8.DecodeRune(runes[count:])
if !unicode.In(c, validRanges...) {
return "", nil
}
if size == 0 {
break
}
count += size
// subtract the max rune size to prevent accidentally splitted runes at the end of first 1024 bytes
if count > total-utf8.UTFMax {
break
}
}
text, _ := io.ReadAll(r)
return string(runes[0:total]) + string(text), nil
}

View File

@@ -0,0 +1,54 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package docextractor
import (
"bytes"
"context"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func TestPlainEmptyFile(t *testing.T) {
extractor := plainExtractor{}
extractedText, err := extractor.Extract(context.Background(), "test.txt", bytes.NewReader([]byte{}), 0)
require.NoError(t, err)
require.Equal(t, "", extractedText)
}
func TestPlainTextSmallFile(t *testing.T) {
extractor := plainExtractor{}
content := strings.Repeat("test \n", 5)
extractedText, err := extractor.Extract(context.Background(), "test.txt", bytes.NewReader([]byte(content)), 0)
require.NoError(t, err)
require.Equal(t, content, extractedText)
}
func TestPlainBigFile(t *testing.T) {
extractor := plainExtractor{}
content := strings.Repeat("test \n", 1000)
extractedText, err := extractor.Extract(context.Background(), "test.txt", bytes.NewReader([]byte(content)), 0)
require.NoError(t, err)
require.Equal(t, content, extractedText)
}
func TestSmallBinaryFile(t *testing.T) {
extractor := plainExtractor{}
notUTF8Char := byte(0x7)
content := bytes.Repeat([]byte{notUTF8Char}, 1000)
extractedText, err := extractor.Extract(context.Background(), "test.bin", bytes.NewReader(content), 0)
require.NoError(t, err)
require.Equal(t, "", extractedText)
}
func TestBigBinaryFile(t *testing.T) {
extractor := plainExtractor{}
notUTF8Char := byte(0x7)
content := bytes.Repeat([]byte{notUTF8Char}, 10000)
extractedText, err := extractor.Extract(context.Background(), "test.bin", bytes.NewReader(content), 0)
require.NoError(t, err)
require.Equal(t, "", extractedText)
}