snapshot of https://github.com/blevesearch/bleve.git (master, 2026-09-15; blobs > 131072B omitted)
Some checks failed
Coverage / coverage (push) Failing after 6s
Tests / test (1.24.x, ubuntu-latest) (push) Failing after 42s
Tests / test (1.26.x, ubuntu-latest) (push) Failing after 38s
Tests / test (1.25.x, ubuntu-latest) (push) Failing after 35s
Tests / test (1.25.x, windows-latest) (push) Has been cancelled
Tests / test (1.26.x, macos-latest) (push) Has been cancelled
Tests / test (1.26.x, windows-latest) (push) Has been cancelled
Tests / test (1.24.x, macos-latest) (push) Has been cancelled
Tests / test (1.24.x, windows-latest) (push) Has been cancelled
Tests / test (1.25.x, macos-latest) (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-15 22:27:16 +05:00
commit f8d3d9f0cf
797 changed files with 159570 additions and 0 deletions

View File

@@ -0,0 +1,163 @@
// Copyright (c) 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package facet
import (
"reflect"
"sort"
"time"
"github.com/blevesearch/bleve/v2/numeric"
"github.com/blevesearch/bleve/v2/search"
"github.com/blevesearch/bleve/v2/size"
)
var (
reflectStaticSizeDateTimeFacetBuilder int
reflectStaticSizedateTimeRange int
)
func init() {
var dtfb DateTimeFacetBuilder
reflectStaticSizeDateTimeFacetBuilder = int(reflect.TypeOf(dtfb).Size())
var dtr dateTimeRange
reflectStaticSizedateTimeRange = int(reflect.TypeOf(dtr).Size())
}
type dateTimeRange struct {
start time.Time
end time.Time
}
type DateTimeFacetBuilder struct {
size int
field string
termsCount map[string]int
total int
missing int
ranges map[string]*dateTimeRange
sawValue bool
}
func NewDateTimeFacetBuilder(field string, size int) *DateTimeFacetBuilder {
return &DateTimeFacetBuilder{
size: size,
field: field,
termsCount: make(map[string]int),
ranges: make(map[string]*dateTimeRange, 0),
}
}
func (fb *DateTimeFacetBuilder) Size() int {
sizeInBytes := reflectStaticSizeDateTimeFacetBuilder + size.SizeOfPtr +
len(fb.field)
for k := range fb.termsCount {
sizeInBytes += size.SizeOfString + len(k) +
size.SizeOfInt
}
for k := range fb.ranges {
sizeInBytes += size.SizeOfString + len(k) +
size.SizeOfPtr + reflectStaticSizedateTimeRange
}
return sizeInBytes
}
func (fb *DateTimeFacetBuilder) AddRange(name string, start, end time.Time) {
r := dateTimeRange{
start: start,
end: end,
}
fb.ranges[name] = &r
}
func (fb *DateTimeFacetBuilder) Field() string {
return fb.field
}
func (fb *DateTimeFacetBuilder) UpdateVisitor(term []byte) {
fb.sawValue = true
// only consider the values which are shifted 0
prefixCoded := numeric.PrefixCoded(term)
shift, err := prefixCoded.Shift()
if err == nil && shift == 0 {
i64, err := prefixCoded.Int64()
if err == nil {
t := time.Unix(0, i64)
// look at each of the ranges for a match
for rangeName, r := range fb.ranges {
if (r.start.IsZero() || t.After(r.start) || t.Equal(r.start)) && (r.end.IsZero() || t.Before(r.end)) {
fb.termsCount[rangeName] = fb.termsCount[rangeName] + 1
fb.total++
}
}
}
}
}
func (fb *DateTimeFacetBuilder) StartDoc() {
fb.sawValue = false
}
func (fb *DateTimeFacetBuilder) EndDoc() {
if !fb.sawValue {
fb.missing++
}
}
func (fb *DateTimeFacetBuilder) Result() *search.FacetResult {
rv := search.FacetResult{
Field: fb.field,
Total: fb.total,
Missing: fb.missing,
}
rv.DateRanges = make([]*search.DateRangeFacet, 0, len(fb.termsCount))
for term, count := range fb.termsCount {
dateRange := fb.ranges[term]
tf := &search.DateRangeFacet{
Name: term,
Count: count,
}
if !dateRange.start.IsZero() {
start := dateRange.start.Format(time.RFC3339Nano)
tf.Start = &start
}
if !dateRange.end.IsZero() {
end := dateRange.end.Format(time.RFC3339Nano)
tf.End = &end
}
rv.DateRanges = append(rv.DateRanges, tf)
}
sort.Sort(rv.DateRanges)
// we now have the list of the top N facets
if fb.size < len(rv.DateRanges) {
rv.DateRanges = rv.DateRanges[:fb.size]
}
notOther := 0
for _, nr := range rv.DateRanges {
notOther += nr.Count
}
rv.Other = fb.total - notOther
return &rv
}

View File

@@ -0,0 +1,157 @@
// Copyright (c) 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package facet
import (
"reflect"
"sort"
"github.com/blevesearch/bleve/v2/numeric"
"github.com/blevesearch/bleve/v2/search"
"github.com/blevesearch/bleve/v2/size"
)
var (
reflectStaticSizeNumericFacetBuilder int
reflectStaticSizenumericRange int
)
func init() {
var nfb NumericFacetBuilder
reflectStaticSizeNumericFacetBuilder = int(reflect.TypeOf(nfb).Size())
var nr numericRange
reflectStaticSizenumericRange = int(reflect.TypeOf(nr).Size())
}
type numericRange struct {
min *float64
max *float64
}
type NumericFacetBuilder struct {
size int
field string
termsCount map[string]int
total int
missing int
ranges map[string]*numericRange
sawValue bool
}
func NewNumericFacetBuilder(field string, size int) *NumericFacetBuilder {
return &NumericFacetBuilder{
size: size,
field: field,
termsCount: make(map[string]int),
ranges: make(map[string]*numericRange, 0),
}
}
func (fb *NumericFacetBuilder) Size() int {
sizeInBytes := reflectStaticSizeNumericFacetBuilder + size.SizeOfPtr +
len(fb.field)
for k := range fb.termsCount {
sizeInBytes += size.SizeOfString + len(k) +
size.SizeOfInt
}
for k := range fb.ranges {
sizeInBytes += size.SizeOfString + len(k) +
size.SizeOfPtr + reflectStaticSizenumericRange
}
return sizeInBytes
}
func (fb *NumericFacetBuilder) AddRange(name string, min, max *float64) {
r := numericRange{
min: min,
max: max,
}
fb.ranges[name] = &r
}
func (fb *NumericFacetBuilder) Field() string {
return fb.field
}
func (fb *NumericFacetBuilder) UpdateVisitor(term []byte) {
fb.sawValue = true
// only consider the values which are shifted 0
prefixCoded := numeric.PrefixCoded(term)
shift, err := prefixCoded.Shift()
if err == nil && shift == 0 {
i64, err := prefixCoded.Int64()
if err == nil {
f64 := numeric.Int64ToFloat64(i64)
// look at each of the ranges for a match
for rangeName, r := range fb.ranges {
if (r.min == nil || f64 >= *r.min) && (r.max == nil || f64 < *r.max) {
fb.termsCount[rangeName] = fb.termsCount[rangeName] + 1
fb.total++
}
}
}
}
}
func (fb *NumericFacetBuilder) StartDoc() {
fb.sawValue = false
}
func (fb *NumericFacetBuilder) EndDoc() {
if !fb.sawValue {
fb.missing++
}
}
func (fb *NumericFacetBuilder) Result() *search.FacetResult {
rv := search.FacetResult{
Field: fb.field,
Total: fb.total,
Missing: fb.missing,
}
rv.NumericRanges = make([]*search.NumericRangeFacet, 0, len(fb.termsCount))
for term, count := range fb.termsCount {
numericRange := fb.ranges[term]
tf := &search.NumericRangeFacet{
Name: term,
Count: count,
Min: numericRange.min,
Max: numericRange.max,
}
rv.NumericRanges = append(rv.NumericRanges, tf)
}
sort.Sort(rv.NumericRanges)
// we now have the list of the top N facets
if fb.size < len(rv.NumericRanges) {
rv.NumericRanges = rv.NumericRanges[:fb.size]
}
notOther := 0
for _, nr := range rv.NumericRanges {
notOther += nr.Count
}
rv.Other = fb.total - notOther
return &rv
}

View File

@@ -0,0 +1,64 @@
// Copyright (c) 2016 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package facet
import (
"strconv"
"testing"
"github.com/blevesearch/bleve/v2/numeric"
)
var pcodedvalues []numeric.PrefixCoded
func init() {
pcodedvalues = []numeric.PrefixCoded{{0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, {0x20, 0x0, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f}, {0x20, 0x0, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7a, 0x1d, 0xa}, {0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x1, 0x16, 0x9, 0x4a, 0x7b}}
}
func BenchmarkNumericFacet10(b *testing.B) {
numericFacetN(b, 10)
}
func BenchmarkNumericFacet100(b *testing.B) {
numericFacetN(b, 100)
}
func BenchmarkNumericFacet1000(b *testing.B) {
numericFacetN(b, 1000)
}
func numericFacetN(b *testing.B, numTerms int) {
field := "test"
nfb := NewNumericFacetBuilder(field, numTerms)
min, max := 0.0, 9999999998.0
for i := 0; i <= numTerms; i++ {
max++
min--
nfb.AddRange("rangename"+strconv.Itoa(i), &min, &max)
for _, pv := range pcodedvalues {
nfb.StartDoc()
nfb.UpdateVisitor(pv)
nfb.EndDoc()
}
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
nfb.Result()
}
}

View File

@@ -0,0 +1,157 @@
// Copyright (c) 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package facet
import (
"bytes"
"reflect"
"regexp"
"sort"
"github.com/blevesearch/bleve/v2/search"
"github.com/blevesearch/bleve/v2/size"
)
var reflectStaticSizeTermsFacetBuilder int
func init() {
var tfb TermsFacetBuilder
reflectStaticSizeTermsFacetBuilder = int(reflect.TypeOf(tfb).Size())
}
type TermsFacetBuilder struct {
size int
field string
prefixBytes []byte
regex *regexp.Regexp
termsCount map[string]int
total int
missing int
sawValue bool
}
func NewTermsFacetBuilder(field string, size int) *TermsFacetBuilder {
return &TermsFacetBuilder{
size: size,
field: field,
termsCount: make(map[string]int),
}
}
func (fb *TermsFacetBuilder) Size() int {
sizeInBytes := reflectStaticSizeTermsFacetBuilder + size.SizeOfPtr +
len(fb.field) +
len(fb.prefixBytes) +
size.SizeOfPtr // regex pointer (does not include actual regexp.Regexp object size)
// Estimate regex object size if present.
if fb.regex != nil {
// This is only the static size of regexp.Regexp struct, not including heap allocations.
sizeInBytes += int(reflect.TypeOf(*fb.regex).Size())
// NOTE: Actual memory usage of regexp.Regexp may be higher due to internal allocations.
}
for k := range fb.termsCount {
sizeInBytes += size.SizeOfString + len(k) +
size.SizeOfInt
}
return sizeInBytes
}
func (fb *TermsFacetBuilder) Field() string {
return fb.field
}
// SetPrefixFilter sets the prefix filter for term facets.
func (fb *TermsFacetBuilder) SetPrefixFilter(prefix string) {
if prefix != "" {
fb.prefixBytes = []byte(prefix)
} else {
fb.prefixBytes = nil
}
}
// SetRegexFilter sets the compiled regex filter for term facets.
func (fb *TermsFacetBuilder) SetRegexFilter(regex *regexp.Regexp) {
fb.regex = regex
}
func (fb *TermsFacetBuilder) UpdateVisitor(term []byte) {
// Total represents all terms visited, not just matching ones.
// This is necessary for the "Other" calculation.
fb.total++
// Fast prefix check on []byte - zero allocation
if len(fb.prefixBytes) > 0 && !bytes.HasPrefix(term, fb.prefixBytes) {
return
}
// Fast regex check on []byte - zero allocation
if fb.regex != nil && !fb.regex.Match(term) {
return
}
// Only convert to string if term matches filters
termStr := string(term)
fb.sawValue = true
fb.termsCount[termStr] = fb.termsCount[termStr] + 1
}
func (fb *TermsFacetBuilder) StartDoc() {
fb.sawValue = false
}
func (fb *TermsFacetBuilder) EndDoc() {
if !fb.sawValue {
fb.missing++
}
}
func (fb *TermsFacetBuilder) Result() *search.FacetResult {
rv := search.FacetResult{
Field: fb.field,
Total: fb.total,
Missing: fb.missing,
}
rv.Terms = &search.TermFacets{}
for term, count := range fb.termsCount {
tf := &search.TermFacet{
Term: term,
Count: count,
}
rv.Terms.Add(tf)
}
sort.Sort(rv.Terms)
// we now have the list of the top N facets
trimTopN := fb.size
if trimTopN > rv.Terms.Len() {
trimTopN = rv.Terms.Len()
}
rv.Terms.TrimToTopN(trimTopN)
notOther := 0
for _, tf := range rv.Terms.Terms() {
notOther += tf.Count
}
rv.Other = fb.total - notOther
return &rv
}

View File

@@ -0,0 +1,290 @@
// Copyright (c) 2016 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package facet
import (
"os"
"regexp"
"testing"
)
var terms []string
func init() {
wsRegexp := regexp.MustCompile(`\W+`)
input, err := os.ReadFile("benchmark_data.txt")
if err != nil {
panic(err)
}
terms = wsRegexp.Split(string(input), -1)
}
func BenchmarkTermsFacet10(b *testing.B) {
termsFacetN(b, 10)
}
func BenchmarkTermsFacet100(b *testing.B) {
termsFacetN(b, 100)
}
func BenchmarkTermsFacet1000(b *testing.B) {
termsFacetN(b, 1000)
}
func BenchmarkTermsFacet10000(b *testing.B) {
termsFacetN(b, 10000)
}
// func BenchmarkTermsFacet100000(b *testing.B) {
// termsFacetN(b, 100000)
// }
func termsFacetN(b *testing.B, numTerms int) {
field := "test"
termsLen := len(terms)
tfb := NewTermsFacetBuilder(field, 3)
i := 0
for len(tfb.termsCount) < numTerms && i <= termsLen {
j := i % termsLen
term := terms[j]
tfb.StartDoc()
tfb.UpdateVisitor([]byte(term))
tfb.EndDoc()
i++
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
tfb.Result()
}
}
func TestTermsFacetPrefix(t *testing.T) {
field := "category"
tfb := NewTermsFacetBuilder(field, 10)
tfb.SetPrefixFilter("prod-")
// Add terms with various prefixes
terms := []string{
"prod-server",
"prod-database",
"dev-server",
"dev-database",
"test-server",
"prod-cache",
}
for _, term := range terms {
tfb.StartDoc()
tfb.UpdateVisitor([]byte(term))
tfb.EndDoc()
}
result := tfb.Result()
// Should only have terms with "prod-" prefix
if result.Terms.Len() != 3 {
t.Fatalf("expected 3 matching terms, got %d", result.Terms.Len())
}
// Verify the terms are correct
expectedTerms := map[string]bool{
"prod-server": true,
"prod-database": true,
"prod-cache": true,
}
for _, facet := range result.Terms.Terms() {
if !expectedTerms[facet.Term] {
t.Errorf("unexpected term in results: %s", facet.Term)
}
if facet.Count != 1 {
t.Errorf("expected count 1 for %s, got %d", facet.Term, facet.Count)
}
}
// Total should include all terms (matching + non-matching)
if result.Total != 6 {
t.Errorf("expected total 6, got %d", result.Total)
}
// Other should be 3 (the non-matching terms)
if result.Other != 3 {
t.Errorf("expected other 3, got %d", result.Other)
}
}
func TestTermsFacetRegex(t *testing.T) {
field := "product_code"
// Match pattern: ABC-#### (3 letters, dash, 4 digits) - pattern: ^[A-Z]{3}-\\d{4}$
tfb := NewTermsFacetBuilder(field, 10)
regex, err := regexp.Compile("^[A-Z]{3}-\\d{4}$")
if err != nil {
t.Fatal(err)
}
tfb.SetRegexFilter(regex)
// Add terms with various formats
terms := []string{
"ABC-1234",
"XYZ-5678",
"ABC-999", // too few digits
"ABCD-1234", // too many letters
"ABC-ABCD", // letters instead of digits
"DEF-0000",
}
for _, term := range terms {
tfb.StartDoc()
tfb.UpdateVisitor([]byte(term))
tfb.EndDoc()
}
result := tfb.Result()
// Should only have 3 terms matching the pattern
if result.Terms.Len() != 3 {
t.Fatalf("expected 3 matching terms, got %d", result.Terms.Len())
}
// Verify the terms are correct
expectedTerms := map[string]bool{
"ABC-1234": true,
"XYZ-5678": true,
"DEF-0000": true,
}
for _, facet := range result.Terms.Terms() {
if !expectedTerms[facet.Term] {
t.Errorf("unexpected term in results: %s", facet.Term)
}
if facet.Count != 1 {
t.Errorf("expected count 1 for %s, got %d", facet.Term, facet.Count)
}
}
// Total should include all terms
if result.Total != 6 {
t.Errorf("expected total 6, got %d", result.Total)
}
// Other should be 3 (the non-matching terms)
if result.Other != 3 {
t.Errorf("expected other 3, got %d", result.Other)
}
}
func TestTermsFacetPrefixAndRegex(t *testing.T) {
field := "tag"
// Both prefix "env:" and regex pattern for prod/staging only
tfb := NewTermsFacetBuilder(field, 10)
tfb.SetPrefixFilter("env:")
regex, err := regexp.Compile("^env:(prod|staging)$")
if err != nil {
t.Fatal(err)
}
tfb.SetRegexFilter(regex)
// Add various terms
terms := []string{
"env:prod",
"env:staging",
"env:dev", // has prefix but doesn't match regex
"env:test", // has prefix but doesn't match regex
"type:server", // no prefix
"env:prod", // duplicate
"env:staging", // duplicate
}
for _, term := range terms {
tfb.StartDoc()
tfb.UpdateVisitor([]byte(term))
tfb.EndDoc()
}
result := tfb.Result()
// Should only have 2 unique terms (env:prod and env:staging)
if result.Terms.Len() != 2 {
t.Fatalf("expected 2 matching terms, got %d", result.Terms.Len())
}
// Verify the terms and counts
termCounts := make(map[string]int)
for _, facet := range result.Terms.Terms() {
termCounts[facet.Term] = facet.Count
}
if termCounts["env:prod"] != 2 {
t.Errorf("expected count 2 for env:prod, got %d", termCounts["env:prod"])
}
if termCounts["env:staging"] != 2 {
t.Errorf("expected count 2 for env:staging, got %d", termCounts["env:staging"])
}
// Total should be all 7 terms
if result.Total != 7 {
t.Errorf("expected total 7, got %d", result.Total)
}
// Other should be 3 (env:dev, env:test, type:server)
if result.Other != 3 {
t.Errorf("expected other 3, got %d", result.Other)
}
}
func TestTermsFacetInvalidRegex(t *testing.T) {
// Invalid regex pattern (unmatched bracket)
_, err := regexp.Compile("[invalid")
if err == nil {
t.Fatal("expected error for invalid regex, got nil")
}
}
func TestTermsFacetNoFilter(t *testing.T) {
field := "tag"
tfb := NewTermsFacetBuilder(field, 2)
terms := []string{"apple", "banana", "cherry", "apple"}
for _, term := range terms {
tfb.StartDoc()
tfb.UpdateVisitor([]byte(term))
tfb.EndDoc()
}
result := tfb.Result()
// Should return top 2 by count
if result.Terms.Len() != 2 {
t.Fatalf("expected 2 terms, got %d", result.Terms.Len())
}
// Apple should be first with count 2
facets := result.Terms.Terms()
if facets[0].Term != "apple" || facets[0].Count != 2 {
t.Errorf("expected apple with count 2, got %s with count %d", facets[0].Term, facets[0].Count)
}
// Total should be 4
if result.Total != 4 {
t.Errorf("expected total 4, got %d", result.Total)
}
// Other should be 1 (cherry was trimmed)
if result.Other != 1 {
t.Errorf("expected other 1, got %d", result.Other)
}
}