snapshot of https://github.com/go-gitea/gitea.git (main, 2026-09-15; blobs > 131072B omitted)

modules/indexer/code/bleve/bleve.go

@@ -0,0 +1,400 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package bleve
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"time"
+
+	repo_model "gitea.dev/models/repo"
+	"gitea.dev/modules/analyze"
+	"gitea.dev/modules/charset"
+	"gitea.dev/modules/git"
+	"gitea.dev/modules/git/gitcmd"
+	"gitea.dev/modules/indexer"
+	path_filter "gitea.dev/modules/indexer/code/bleve/token/path"
+	"gitea.dev/modules/indexer/code/internal"
+	indexer_internal "gitea.dev/modules/indexer/internal"
+	inner_bleve "gitea.dev/modules/indexer/internal/bleve"
+	"gitea.dev/modules/json"
+	"gitea.dev/modules/setting"
+	"gitea.dev/modules/timeutil"
+	"gitea.dev/modules/typesniffer"
+	"gitea.dev/modules/util"
+
+	"github.com/blevesearch/bleve/v2"
+	analyzer_custom "github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
+	analyzer_keyword "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
+	"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
+	"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
+	"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
+	"github.com/blevesearch/bleve/v2/mapping"
+	"github.com/blevesearch/bleve/v2/search/query"
+	"github.com/go-enry/go-enry/v2"
+)
+
+const (
+	unicodeNormalizeName = "unicodeNormalize"
+	maxBatchSize         = 16
+)
+
+func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error {
+	return m.AddCustomTokenFilter(unicodeNormalizeName, map[string]any{
+		"type": unicodenorm.Name,
+		"form": unicodenorm.NFC,
+	})
+}
+
+// RepoIndexerData data stored in the repo indexer
+type RepoIndexerData struct {
+	RepoID    int64
+	CommitID  string
+	Content   string
+	Filename  string
+	Language  string
+	UpdatedAt time.Time
+}
+
+// Type returns the document type, for bleve's mapping.Classifier interface.
+func (d *RepoIndexerData) Type() string {
+	return repoIndexerDocType
+}
+
+const (
+	repoIndexerAnalyzer      = "repoIndexerAnalyzer"
+	filenameIndexerAnalyzer  = "filenameIndexerAnalyzer"
+	repoIndexerDocType       = "repoIndexerDocType"
+	repoIndexerLatestVersion = 10
+)
+
+// generateBleveIndexMapping generates a bleve index mapping for the repo indexer
+func generateBleveIndexMapping() (mapping.IndexMapping, error) {
+	docMapping := bleve.NewDocumentMapping()
+	numericFieldMapping := bleve.NewNumericFieldMapping()
+	numericFieldMapping.IncludeInAll = false
+	docMapping.AddFieldMappingsAt("RepoID", numericFieldMapping)
+
+	textFieldMapping := bleve.NewTextFieldMapping()
+	textFieldMapping.IncludeInAll = false
+	docMapping.AddFieldMappingsAt("Content", textFieldMapping)
+
+	fileNamedMapping := bleve.NewTextFieldMapping()
+	fileNamedMapping.IncludeInAll = false
+	fileNamedMapping.Analyzer = filenameIndexerAnalyzer
+	docMapping.AddFieldMappingsAt("Filename", fileNamedMapping)
+
+	termFieldMapping := bleve.NewTextFieldMapping()
+	termFieldMapping.IncludeInAll = false
+	termFieldMapping.Analyzer = analyzer_keyword.Name
+	docMapping.AddFieldMappingsAt("Language", termFieldMapping)
+	docMapping.AddFieldMappingsAt("CommitID", termFieldMapping)
+
+	timeFieldMapping := bleve.NewDateTimeFieldMapping()
+	timeFieldMapping.IncludeInAll = false
+	docMapping.AddFieldMappingsAt("UpdatedAt", timeFieldMapping)
+
+	mapping := bleve.NewIndexMapping()
+
+	if err := addUnicodeNormalizeTokenFilter(mapping); err != nil {
+		return nil, err
+	} else if err := mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]any{
+		"type":          analyzer_custom.Name,
+		"char_filters":  []string{},
+		"tokenizer":     codeTokenizerName,
+		"token_filters": []string{unicodeNormalizeName, codeTokenFilterName, lowercase.Name},
+	}); err != nil {
+		return nil, err
+	}
+
+	if err := mapping.AddCustomAnalyzer(filenameIndexerAnalyzer, map[string]any{
+		"type":          analyzer_custom.Name,
+		"char_filters":  []string{},
+		"tokenizer":     unicode.Name,
+		"token_filters": []string{unicodeNormalizeName, path_filter.Name, lowercase.Name},
+	}); err != nil {
+		return nil, err
+	}
+
+	mapping.DefaultAnalyzer = repoIndexerAnalyzer
+	mapping.AddDocumentMapping(repoIndexerDocType, docMapping)
+	mapping.AddDocumentMapping("_all", bleve.NewDocumentDisabledMapping())
+
+	return mapping, nil
+}
+
+var _ internal.Indexer = &Indexer{}
+
+// Indexer represents a bleve indexer implementation
+type Indexer struct {
+	inner                    *inner_bleve.Indexer
+	indexer_internal.Indexer // do not composite inner_bleve.Indexer directly to avoid exposing too much
+}
+
+func (b *Indexer) SupportedSearchModes() []indexer.SearchMode {
+	return indexer.SearchModesExactWords()
+}
+
+// NewIndexer creates a new bleve local indexer
+func NewIndexer(indexDir string) *Indexer {
+	inner := inner_bleve.NewIndexer(indexDir, repoIndexerLatestVersion, generateBleveIndexMapping)
+	return &Indexer{
+		Indexer: inner,
+		inner:   inner,
+	}
+}
+
+func (b *Indexer) addUpdate(ctx context.Context, catFileBatch git.CatFileBatch, commitSha string,
+	update internal.FileUpdate, repo *repo_model.Repository, batch *inner_bleve.FlushingBatch,
+) error {
+	// Ignore vendored files in code search
+	if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) {
+		return nil
+	}
+
+	size := update.Size
+
+	var err error
+	if !update.Sized {
+		var stdout string
+		stdout, _, err = gitcmd.NewCommand("cat-file", "-s").AddDynamicArguments(update.BlobSha).WithRepo(repo).RunStdString(ctx)
+		if err != nil {
+			return err
+		}
+		if size, err = strconv.ParseInt(strings.TrimSpace(stdout), 10, 64); err != nil {
+			return fmt.Errorf("misformatted git cat-file output: %w", err)
+		}
+	}
+
+	if size > setting.Indexer.MaxIndexerFileSize {
+		return b.addDelete(update.Filename, repo, batch)
+	}
+
+	info, batchReader, err := catFileBatch.QueryContent(update.BlobSha)
+	if err != nil {
+		return err
+	}
+	fileContents, err := io.ReadAll(io.LimitReader(batchReader, info.Size))
+	if err != nil {
+		return err
+	} else if !typesniffer.DetectContentType(fileContents).IsText() {
+		// FIXME: UTF-16 files will probably fail here
+		// Even if the file is not recognized as a "text file", we could still put its name into the indexers to make the filename become searchable, while leave the content to empty.
+		fileContents = nil
+	}
+
+	if _, err = batchReader.Discard(1); err != nil {
+		return err
+	}
+	id := internal.FilenameIndexerID(repo.ID, update.Filename)
+	return batch.Index(id, &RepoIndexerData{
+		RepoID:    repo.ID,
+		CommitID:  commitSha,
+		Filename:  update.Filename,
+		Content:   string(charset.ToUTF8DropErrors(fileContents)),
+		Language:  analyze.GetCodeLanguage(update.Filename, fileContents),
+		UpdatedAt: time.Now().UTC(),
+	})
+}
+
+func (b *Indexer) addDelete(filename string, repo *repo_model.Repository, batch *inner_bleve.FlushingBatch) error {
+	id := internal.FilenameIndexerID(repo.ID, filename)
+	return batch.Delete(id)
+}
+
+// Index indexes the data
+func (b *Indexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *internal.RepoChanges) error {
+	batch := inner_bleve.NewFlushingBatch(b.inner.Indexer, maxBatchSize)
+	if len(changes.Updates) > 0 {
+		catfileBatch, err := git.NewBatch(ctx, repo)
+		if err != nil {
+			return err
+		}
+		defer catfileBatch.Close()
+
+		for _, update := range changes.Updates {
+			if err := b.addUpdate(ctx, catfileBatch, sha, update, repo, batch); err != nil {
+				return err
+			}
+		}
+	}
+	for _, filename := range changes.RemovedFilenames {
+		if err := b.addDelete(filename, repo, batch); err != nil {
+			return err
+		}
+	}
+	return batch.Flush()
+}
+
+// Delete deletes indexes by ids
+func (b *Indexer) Delete(_ context.Context, repoID int64) error {
+	query := inner_bleve.NumericEqualityQuery(repoID, "RepoID")
+	searchRequest := bleve.NewSearchRequestOptions(query, 2147483647, 0, false)
+	result, err := b.inner.Indexer.Search(searchRequest)
+	if err != nil {
+		return err
+	}
+	batch := inner_bleve.NewFlushingBatch(b.inner.Indexer, maxBatchSize)
+	for _, hit := range result.Hits {
+		if err = batch.Delete(hit.ID); err != nil {
+			return err
+		}
+	}
+	return batch.Flush()
+}
+
+// Search searches for files in the specified repo.
+// Returns the matching file-paths
+func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
+	var (
+		indexerQuery query.Query
+		keywordQuery query.Query
+		contentQuery query.Query
+	)
+
+	pathQuery := bleve.NewPrefixQuery(strings.ToLower(opts.Keyword))
+	pathQuery.FieldVal = "Filename"
+	pathQuery.SetBoost(10)
+
+	searchMode := util.IfZero(opts.SearchMode, b.SupportedSearchModes()[0].ModeValue)
+	if searchMode == indexer.SearchModeExact {
+		// 1.21 used NewPrefixQuery, but it seems not working well, and later releases changed to NewMatchPhraseQuery
+		q := bleve.NewMatchPhraseQuery(opts.Keyword)
+		q.Analyzer = repoIndexerAnalyzer
+		q.FieldVal = "Content"
+		contentQuery = q
+	} else /* words */ {
+		q := bleve.NewMatchQuery(opts.Keyword)
+		q.FieldVal = "Content"
+		q.Analyzer = repoIndexerAnalyzer
+		if searchMode == indexer.SearchModeFuzzy {
+			// this logic doesn't seem right, it is only used to pass the test-case `Keyword:    "dESCRIPTION"`, which doesn't seem to be a real-life use-case.
+			q.Fuzziness = inner_bleve.GuessFuzzinessByKeyword(opts.Keyword)
+		} else {
+			q.Operator = query.MatchQueryOperatorAnd
+		}
+		contentQuery = q
+	}
+
+	keywordQuery = bleve.NewDisjunctionQuery(contentQuery, pathQuery)
+
+	if len(opts.RepoIDs) > 0 {
+		repoQueries := make([]query.Query, 0, len(opts.RepoIDs))
+		for _, repoID := range opts.RepoIDs {
+			repoQueries = append(repoQueries, inner_bleve.NumericEqualityQuery(repoID, "RepoID"))
+		}
+
+		indexerQuery = bleve.NewConjunctionQuery(
+			bleve.NewDisjunctionQuery(repoQueries...),
+			keywordQuery,
+		)
+	} else {
+		indexerQuery = keywordQuery
+	}
+
+	// Save for reuse without language filter
+	facetQuery := indexerQuery
+	if len(opts.Language) > 0 {
+		languageQuery := bleve.NewMatchQuery(opts.Language)
+		languageQuery.FieldVal = "Language"
+		languageQuery.Analyzer = analyzer_keyword.Name
+
+		indexerQuery = bleve.NewConjunctionQuery(
+			indexerQuery,
+			languageQuery,
+		)
+	}
+
+	from, pageSize := opts.GetSkipTake()
+	searchRequest := bleve.NewSearchRequestOptions(indexerQuery, pageSize, from, false)
+	searchRequest.Fields = []string{"Content", "Filename", "RepoID", "Language", "CommitID", "UpdatedAt"}
+	searchRequest.IncludeLocations = true
+
+	if len(opts.Language) == 0 {
+		searchRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
+	}
+
+	searchRequest.SortBy([]string{"-_score", "UpdatedAt"})
+
+	result, err := b.inner.Indexer.SearchInContext(ctx, searchRequest)
+	if err != nil {
+		return 0, nil, nil, err
+	}
+
+	total := int64(result.Total)
+
+	searchResults := make([]*internal.SearchResult, len(result.Hits))
+	for i, hit := range result.Hits {
+		content, okContent := hit.Fields["Content"].(string)
+		language, okLanguage := hit.Fields["Language"].(string)
+		commitID, okCommitID := hit.Fields["CommitID"].(string)
+		updatedAt, okUpdatedAt := hit.Fields["UpdatedAt"].(string)
+		repoID, okRepoID := hit.Fields["RepoID"].(float64)
+		if !okContent || !okLanguage || !okCommitID || !okUpdatedAt || !okRepoID {
+			hitFieldsJson, _ := json.Marshal(hit.Fields)
+			setting.PanicInDevOrTesting("unexpected field types in search hit %q: %s", hit.ID, string(hitFieldsJson))
+			return 0, nil, nil, fmt.Errorf("unexpected field types in search hit %q", hit.ID)
+		}
+
+		startIndex, endIndex := -1, -1
+		for _, locations := range hit.Locations["Content"] {
+			location := locations[0]
+			locationStart := int(location.Start)
+			locationEnd := int(location.End)
+			if startIndex < 0 || locationStart < startIndex {
+				startIndex = locationStart
+			}
+			if endIndex < 0 || locationEnd > endIndex {
+				endIndex = locationEnd
+			}
+		}
+		if len(hit.Locations["Filename"]) > 0 {
+			startIndex, endIndex = internal.FilenameMatchIndexPos(content)
+		}
+
+		var updatedUnix timeutil.TimeStamp
+		if t, err := time.Parse(time.RFC3339, updatedAt); err == nil {
+			updatedUnix = timeutil.TimeStamp(t.Unix())
+		}
+		searchResults[i] = &internal.SearchResult{
+			RepoID:      int64(repoID),
+			StartIndex:  startIndex,
+			EndIndex:    endIndex,
+			Filename:    internal.FilenameOfIndexerID(hit.ID),
+			Content:     content,
+			CommitID:    commitID,
+			UpdatedUnix: updatedUnix,
+			Language:    language,
+			Color:       enry.GetColor(language),
+		}
+	}
+
+	searchResultLanguages := make([]*internal.SearchResultLanguages, 0, 10)
+	if len(opts.Language) > 0 {
+		// Use separate query to go get all language counts
+		facetRequest := bleve.NewSearchRequestOptions(facetQuery, 1, 0, false)
+		facetRequest.Fields = []string{"Content", "RepoID", "Language", "CommitID", "UpdatedAt"}
+		facetRequest.IncludeLocations = true
+		facetRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
+
+		if result, err = b.inner.Indexer.Search(facetRequest); err != nil {
+			return 0, nil, nil, err
+		}
+	}
+	languagesFacet := result.Facets["languages"]
+	for _, term := range languagesFacet.Terms.Terms() {
+		if len(term.Term) == 0 {
+			continue
+		}
+		searchResultLanguages = append(searchResultLanguages, &internal.SearchResultLanguages{
+			Language: term.Term,
+			Color:    enry.GetColor(term.Term),
+			Count:    term.Count,
+		})
+	}
+	return total, searchResults, searchResultLanguages, nil
+}

modules/indexer/code/bleve/bleve_test.go

@@ -0,0 +1,59 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package bleve
+
+import (
+	"testing"
+	"time"
+
+	"gitea.dev/models/db"
+	"gitea.dev/modules/indexer/code/internal"
+	inner_bleve "gitea.dev/modules/indexer/internal/bleve"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBleveIndexerTokenFilter(t *testing.T) {
+	dir := t.TempDir()
+	indexer := NewIndexer(dir)
+	defer indexer.Close()
+
+	_, err := indexer.Init(t.Context())
+	require.NoError(t, err)
+
+	batch := inner_bleve.NewFlushingBatch(indexer.inner.Indexer, maxBatchSize)
+	batch.Index("2", &RepoIndexerData{RepoID: 2, Content: "mDNS.port2=12345", UpdatedAt: time.Now()})
+	batch.Flush()
+
+	testCases := []struct {
+		keyword     string
+		expectedIDs []int64
+	}{
+		{keyword: "12345", expectedIDs: []int64{2}},
+		{keyword: "DNS", expectedIDs: []int64{}},
+		{keyword: "mdns", expectedIDs: []int64{2}},
+		{keyword: "port", expectedIDs: []int64{2}},
+		{keyword: "port2", expectedIDs: []int64{2}},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.keyword, func(t *testing.T) {
+			_, results, _, err := indexer.Search(t.Context(), &internal.SearchOptions{
+				Paginator: &db.ListOptions{Page: 1, PageSize: 1},
+				Keyword:   testCase.keyword,
+			})
+			require.NoError(t, err)
+			assert.ElementsMatch(t, testCase.expectedIDs, searchResultIDs(results))
+		})
+	}
+}
+
+func searchResultIDs(result []*internal.SearchResult) []int64 {
+	ids := make([]int64, 0, len(result))
+	for _, hit := range result {
+		ids = append(ids, hit.RepoID)
+	}
+	return ids
+}

modules/indexer/code/bleve/code_token.go

@@ -0,0 +1,65 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package bleve
+
+import (
+	"regexp"
+	"unicode"
+
+	"gitea.dev/modules/util"
+
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/analysis/tokenizer/character"
+	"github.com/blevesearch/bleve/v2/registry"
+)
+
+const codeTokenizerName = "codeTokenizer"
+
+func codeTokenizerConstructor(_ map[string]any, _ *registry.Cache) (analysis.Tokenizer, error) {
+	// Old code used "letter" tokenizer which doesn't support CJK.
+	// Here it still doesn't support CJK, since there is no usable CJK tokenizer at the moment.
+	return character.NewCharacterTokenizer(func(r rune) bool {
+		return unicode.IsLetter(r) || unicode.IsNumber(r)
+	}), nil
+}
+
+const codeTokenFilterName = "codeTokenFilter"
+
+type codeTokenFilter struct {
+	re *regexp.Regexp
+}
+
+func (c codeTokenFilter) Filter(stream analysis.TokenStream) (ret analysis.TokenStream) {
+	// split one token to "letter" parts and "number" parts (to keep the old behavior).
+	// e.g.: input token="port123", then the output tokens are "port123", "port", "123"
+	for _, token := range stream {
+		ret = append(ret, token)
+		m := c.re.FindAllIndex(token.Term, -1)
+		if len(m) > 1 {
+			for _, it := range m {
+				p1, p2 := it[0], it[1]
+				t := &analysis.Token{
+					Start:    token.Start + p1,
+					End:      token.Start + p2,
+					Term:     token.Term[p1:p2],
+					Position: token.Position,
+					Type:     analysis.AlphaNumeric,
+				}
+				ret = append(ret, t)
+			}
+		}
+	}
+	return ret
+}
+
+func codeTokenFilterConstructor(_ map[string]any, _ *registry.Cache) (analysis.TokenFilter, error) {
+	return &codeTokenFilter{
+		re: regexp.MustCompile("[a-zA-Z]+|[0-9]+"),
+	}, nil
+}
+
+func init() {
+	util.MustNoError(registry.RegisterTokenizer(codeTokenizerName, codeTokenizerConstructor))
+	util.MustNoError(registry.RegisterTokenFilter(codeTokenFilterName, codeTokenFilterConstructor))
+}

modules/indexer/code/bleve/token/path/path.go

@@ -0,0 +1,102 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package path
+
+import (
+	"slices"
+	"strings"
+
+	"gitea.dev/modules/util"
+
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/registry"
+)
+
+const Name = "gitea/path"
+
+type TokenFilter struct{}
+
+func NewTokenFilter() *TokenFilter {
+	return &TokenFilter{}
+}
+
+func TokenFilterConstructor(config map[string]any, cache *registry.Cache) (analysis.TokenFilter, error) {
+	return NewTokenFilter(), nil
+}
+
+func (s *TokenFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
+	if len(input) == 1 {
+		// if there is only one token, we don't need to generate the reversed chain
+		return generatePathTokens(input, false)
+	}
+
+	normal := generatePathTokens(input, false)
+	reversed := generatePathTokens(input, true)
+
+	return append(normal, reversed...)
+}
+
+// Generates path tokens from the input tokens.
+// This mimics the behavior of the path hierarchy tokenizer in ES. It takes the input tokens and combine them, generating a term for each component
+// in tree (e.g., foo/bar/baz.md will generate foo, foo/bar, and foo/bar/baz.md).
+//
+// If the reverse flag is set, the order of the tokens is reversed (the same input will generate baz.md, baz.md/bar, baz.md/bar/foo). This is useful
+// to efficiently search for filenames without supplying the fullpath.
+func generatePathTokens(input analysis.TokenStream, reversed bool) analysis.TokenStream {
+	terms := make([]string, 0, len(input))
+	longestTerm := 0
+
+	if reversed {
+		slices.Reverse(input)
+	}
+
+	for i := range input {
+		var sb strings.Builder
+		sb.Write(input[0].Term)
+
+		for j := 1; j < i; j++ {
+			sb.WriteString("/")
+			sb.Write(input[j].Term)
+		}
+
+		term := sb.String()
+
+		if longestTerm < len(term) {
+			longestTerm = len(term)
+		}
+
+		terms = append(terms, term)
+	}
+
+	output := make(analysis.TokenStream, 0, len(terms))
+
+	for _, term := range terms {
+		var start, end int
+
+		if reversed {
+			start = 0
+			end = len(term)
+		} else {
+			start = longestTerm - len(term)
+			end = longestTerm
+		}
+
+		token := analysis.Token{
+			Position: 1,
+			Start:    start,
+			End:      end,
+			Type:     analysis.AlphaNumeric,
+			Term:     []byte(term),
+		}
+
+		output = append(output, &token)
+	}
+
+	return output
+}
+
+func init() {
+	// FIXME: move it to the bleve's init function, but do not call it in global init
+	util.MustNoError(registry.RegisterTokenFilter(Name, TokenFilterConstructor))
+}

modules/indexer/code/bleve/token/path/path_test.go

@@ -0,0 +1,76 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package path
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
+	"github.com/stretchr/testify/assert"
+)
+
+type Scenario struct {
+	Input  string
+	Tokens []string
+}
+
+func TestTokenFilter(t *testing.T) {
+	scenarios := []struct {
+		Input string
+		Terms []string
+	}{
+		{
+			Input: "Dockerfile",
+			Terms: []string{"Dockerfile"},
+		},
+		{
+			Input: "Dockerfile.rootless",
+			Terms: []string{"Dockerfile.rootless"},
+		},
+		{
+			Input: "a/b/c/Dockerfile.rootless",
+			Terms: []string{"a", "a/b", "a/b/c", "a/b/c/Dockerfile.rootless", "Dockerfile.rootless", "Dockerfile.rootless/c", "Dockerfile.rootless/c/b", "Dockerfile.rootless/c/b/a"},
+		},
+		{
+			Input: "",
+			Terms: []string{},
+		},
+	}
+
+	for _, scenario := range scenarios {
+		t.Run(fmt.Sprintf("ensure terms of '%s'", scenario.Input), func(t *testing.T) {
+			terms := extractTerms(scenario.Input)
+
+			assert.Len(t, terms, len(scenario.Terms))
+
+			for _, term := range terms {
+				assert.Contains(t, scenario.Terms, term)
+			}
+		})
+	}
+}
+
+func extractTerms(input string) []string {
+	tokens := tokenize(input)
+	filteredTokens := filter(tokens)
+	terms := make([]string, 0, len(filteredTokens))
+
+	for _, token := range filteredTokens {
+		terms = append(terms, string(token.Term))
+	}
+
+	return terms
+}
+
+func filter(input analysis.TokenStream) analysis.TokenStream {
+	filter := NewTokenFilter()
+	return filter.Filter(input)
+}
+
+func tokenize(input string) analysis.TokenStream {
+	tokenizer := unicode.NewUnicodeTokenizer()
+	return tokenizer.Tokenize([]byte(input))
+}