snapshot of https://github.com/blevesearch/bleve.git (master, 2026-09-15; blobs > 131072B omitted)
Some checks failed
Coverage / coverage (push) Failing after 6s
Tests / test (1.24.x, ubuntu-latest) (push) Failing after 42s
Tests / test (1.26.x, ubuntu-latest) (push) Failing after 38s
Tests / test (1.25.x, ubuntu-latest) (push) Failing after 35s
Tests / test (1.25.x, windows-latest) (push) Has been cancelled
Tests / test (1.26.x, macos-latest) (push) Has been cancelled
Tests / test (1.26.x, windows-latest) (push) Has been cancelled
Tests / test (1.24.x, macos-latest) (push) Has been cancelled
Tests / test (1.24.x, windows-latest) (push) Has been cancelled
Tests / test (1.25.x, macos-latest) (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-15 22:27:16 +05:00
commit f8d3d9f0cf
797 changed files with 159570 additions and 0 deletions

130
util/bitset.go Normal file
View File

@@ -0,0 +1,130 @@
// Copyright (c) 2026 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package util
import (
"math/bits"
"github.com/RoaringBitmap/roaring/v2"
)
type Bitset struct {
data []uint64
numBits int
exclude *roaring.Bitmap
}
// NewBitset initializes a bitset capable of holding numbers up to maxVal
func NewBitset(maxVal int, exclude *roaring.Bitmap) *Bitset {
// We need (maxVal / 64) + 1 buckets to hold up to maxVal
size := (maxVal / 64) + 1
return &Bitset{
data: make([]uint64, size),
numBits: maxVal,
exclude: exclude,
}
}
// Add inserts a value into the bitset (safely handles duplicates)
func (b *Bitset) Add(val int) {
if b.exclude != nil && b.exclude.Contains(uint32(val)) {
return
}
bucket := val >> 6 // Equivalent to val / 64
bit := uint(val & 63) // Equivalent to val % 64
// Set the bit to 1 using bitwise OR
b.data[bucket] |= (1 << bit)
}
// Remove deletes a value from the bitset
func (b *Bitset) Remove(val int) {
bucket := val >> 6
bit := uint(val & 63)
// Set the bit to 0 using bitwise AND with the complement
b.data[bucket] &^= (1 << bit)
}
// Contains checks if a value exists in the bitset
func (b *Bitset) Contains(val int) bool {
bucket := val >> 6
bit := uint(val & 63)
return (b.data[bucket] & (1 << bit)) != 0
}
// Invert flips all bits in the bitset,
// effectively turning all 1s to 0s and vice versa
func (b *Bitset) Invert() {
for i := range b.data {
b.data[i] = ^b.data[i]
}
// the flip above sets the trailing bits beyond numBits in the last
// bucket(s), which do not correspond to valid values - clear them so
// that Iterate and Count never see them
lastBucket := b.numBits >> 6
if lastBucket < len(b.data) {
b.data[lastBucket] &= (1 << uint(b.numBits&63)) - 1
for i := lastBucket + 1; i < len(b.data); i++ {
b.data[i] = 0
}
}
if b.exclude != nil {
it := b.exclude.Iterator()
for it.HasNext() {
bit := uint64(it.Next())
word := bit / 64
if word < uint64(len(b.data)) {
b.data[word] &^= uint64(1) << (bit % 64)
}
}
}
}
// Iterate calls the provided function for every integer recorded in the bitset, in ascending order
func (b *Bitset) Iterate(f func(int)) {
for bucketIdx, bucket := range b.data {
// If the entire 64-bit block is 0, skip it entirely for speed
if bucket == 0 {
continue
}
// Check all 64 bits in this bucket
for bitIdx := 0; bitIdx < 64; bitIdx++ {
if (bucket & (1 << uint(bitIdx))) != 0 {
// Reconstruct the original integer
originalVal := (bucketIdx << 6) + bitIdx
f(originalVal)
}
}
}
}
func (b *Bitset) Count() int {
count := 0
for _, word := range b.data {
count += bits.OnesCount64(word)
}
return count
}
func (b *Bitset) Clear() {
for i := range b.data {
b.data[i] = 0
}
}

205
util/bitset_test.go Normal file
View File

@@ -0,0 +1,205 @@
// Copyright (c) 2026 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package util
import (
"reflect"
"testing"
"github.com/RoaringBitmap/roaring/v2"
)
func TestBitsetAddContains(t *testing.T) {
b := NewBitset(100, nil)
// values not yet added must not be present
for _, v := range []int{0, 1, 63, 64, 65, 100} {
if b.Contains(v) {
t.Fatalf("expected %d to be absent from an empty bitset", v)
}
}
// exercise word boundaries explicitly: 63 is the last bit of word 0,
// 64 is the first bit of word 1, 65 the second
added := []int{0, 1, 63, 64, 65, 100}
for _, v := range added {
b.Add(v)
}
for _, v := range added {
if !b.Contains(v) {
t.Fatalf("expected %d to be present after Add", v)
}
}
// a value between two set bits must remain absent
if b.Contains(50) {
t.Fatalf("expected 50 to be absent")
}
}
func TestBitsetAddDuplicate(t *testing.T) {
b := NewBitset(100, nil)
b.Add(42)
b.Add(42)
if got := b.Count(); got != 1 {
t.Fatalf("expected duplicate Add to keep count at 1, got %d", got)
}
if !b.Contains(42) {
t.Fatalf("expected 42 to be present")
}
}
func TestBitsetRemove(t *testing.T) {
b := NewBitset(100, nil)
b.Add(10)
b.Add(64)
b.Remove(10)
if b.Contains(10) {
t.Fatalf("expected 10 to be absent after Remove")
}
if !b.Contains(64) {
t.Fatalf("expected 64 to still be present")
}
// removing an absent value must be a no-op
b.Remove(99)
if b.Count() != 1 {
t.Fatalf("expected count 1 after removing an absent value, got %d", b.Count())
}
}
func TestBitsetCountAndClear(t *testing.T) {
b := NewBitset(200, nil)
for _, v := range []int{0, 5, 63, 64, 128, 200} {
b.Add(v)
}
if got := b.Count(); got != 6 {
t.Fatalf("expected count 6, got %d", got)
}
b.Clear()
if got := b.Count(); got != 0 {
t.Fatalf("expected count 0 after Clear, got %d", got)
}
}
func TestBitsetIterateAscending(t *testing.T) {
b := NewBitset(200, nil)
// add out of order to confirm Iterate returns ascending order
for _, v := range []int{130, 0, 64, 63, 7} {
b.Add(v)
}
var got []int
b.Iterate(func(v int) {
got = append(got, v)
})
want := []int{0, 7, 63, 64, 130}
if !reflect.DeepEqual(got, want) {
t.Fatalf("expected Iterate to yield %v in ascending order, got %v", want, got)
}
}
func TestBitsetExcludeBlocksAdd(t *testing.T) {
exclude := roaring.New()
exclude.AddInt(5)
exclude.AddInt(70)
b := NewBitset(100, exclude)
b.Add(5) // excluded, must be ignored
b.Add(6) // allowed
b.Add(70) // excluded, must be ignored
if b.Contains(5) {
t.Fatalf("expected excluded value 5 to be blocked by Add")
}
if b.Contains(70) {
t.Fatalf("expected excluded value 70 to be blocked by Add")
}
if !b.Contains(6) {
t.Fatalf("expected non-excluded value 6 to be present")
}
}
func TestBitsetInvertRespectsNumBits(t *testing.T) {
// numDocs = 10 means the valid doc IDs are 0..9. NewBitset allocates a
// full 64-bit word, so bits 10..63 are unused and must never surface.
numDocs := 10
b := NewBitset(numDocs, nil)
b.Add(3)
b.Add(7)
b.Invert()
var got []int
b.Iterate(func(v int) {
got = append(got, v)
})
// after inverting, exactly the doc IDs in [0, 10) that were NOT set
// should be present - and nothing >= 10
want := []int{0, 1, 2, 4, 5, 6, 8, 9}
if !reflect.DeepEqual(got, want) {
t.Fatalf("expected inverted bitset to yield %v, got %v", want, got)
}
if got := b.Count(); got != len(want) {
t.Fatalf("expected inverted count %d, got %d", len(want), got)
}
}
func TestBitsetInvertWordBoundaries(t *testing.T) {
// exercise numBits at and around 64-bit word boundaries to make sure
// the trailing-bit mask is computed correctly
for _, numDocs := range []int{1, 63, 64, 65, 128, 129} {
b := NewBitset(numDocs, nil)
b.Invert()
max := -1
b.Iterate(func(v int) {
if v > max {
max = v
}
})
// every valid doc ID [0, numDocs) should be present after inverting
// an empty bitset, and none at or beyond numDocs
if got := b.Count(); got != numDocs {
t.Fatalf("numDocs=%d: expected inverted-empty count %d, got %d",
numDocs, numDocs, got)
}
if max >= numDocs {
t.Fatalf("numDocs=%d: Invert surfaced out-of-range value %d",
numDocs, max)
}
}
}
func TestBitsetInvertClearsExcluded(t *testing.T) {
// excluded docs must remain unset even after Invert, since they are
// never valid hits
exclude := roaring.New()
exclude.AddInt(2)
exclude.AddInt(8)
b := NewBitset(10, exclude)
b.Add(3)
b.Invert()
if b.Contains(2) || b.Contains(8) {
t.Fatalf("expected excluded docs to stay unset after Invert")
}
if b.Contains(3) {
t.Fatalf("expected the originally-set doc 3 to be cleared after Invert")
}
// a normal, non-excluded, originally-unset doc should now be set
if !b.Contains(0) {
t.Fatalf("expected doc 0 to be set after Invert")
}
}

170
util/bolt.go Normal file
View File

@@ -0,0 +1,170 @@
// Copyright (c) 2026 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package util
import (
"fmt"
"os"
bolt "go.etcd.io/bbolt"
)
// All of the bolt impls provide a layer of indirection to allow for processing
// of values as they are read/written to bolt depending on the key or bucket name
// This is used to allow better support for file callbacks
// wrapper around bolt.DB
type RootBoltImpl struct {
*bolt.DB
}
// wrapper around bolt.Tx
type BoltTxImpl struct {
*bolt.Tx
}
// wrapper around bolt.Bucket
type BoltBucketImpl struct {
*bolt.Bucket
name string // store the name of the bucket during creation
}
func OpenBolt(path string, mode os.FileMode, options *bolt.Options) (*RootBoltImpl, error) {
db, err := bolt.Open(path, mode, options)
if err != nil {
return nil, err
}
return &RootBoltImpl{DB: db}, nil
}
func (r *RootBoltImpl) Begin(writable bool) (*BoltTxImpl, error) {
tx, err := r.DB.Begin(writable)
if err != nil {
return nil, err
}
return &BoltTxImpl{Tx: tx}, nil
}
func (r *RootBoltImpl) View(fn func(*BoltTxImpl) error) error {
return r.DB.View(func(tx *bolt.Tx) error {
return fn(&BoltTxImpl{Tx: tx})
})
}
func (r *RootBoltImpl) Update(fn func(*BoltTxImpl) error) error {
return r.DB.Update(func(tx *bolt.Tx) error {
return fn(&BoltTxImpl{Tx: tx})
})
}
func (tx *BoltTxImpl) CreateBucketIfNotExists(name []byte) (*BoltBucketImpl, error) {
bucket, err := tx.Tx.CreateBucketIfNotExists(name)
if err != nil {
return nil, err
}
return &BoltBucketImpl{
name: string(name),
Bucket: bucket,
}, nil
}
func (tx *BoltTxImpl) Bucket(name []byte) *BoltBucketImpl {
bucket := tx.Tx.Bucket(name)
if bucket == nil {
return nil
}
return &BoltBucketImpl{
name: string(name),
Bucket: bucket,
}
}
func (b *BoltBucketImpl) GetBucket(name []byte) *BoltBucketImpl {
bucket := b.Bucket.Bucket(name)
if bucket == nil {
return nil
}
return &BoltBucketImpl{
name: string(name),
Bucket: bucket,
}
}
func (b *BoltBucketImpl) CreateBucketIfNotExists(name []byte) (*BoltBucketImpl, error) {
bucket, err := b.Bucket.CreateBucketIfNotExists(name)
if err != nil {
return nil, err
}
return &BoltBucketImpl{
name: string(name),
Bucket: bucket,
}, nil
}
// Process values during ForEach if the bucket name or key is in the boltKeysProcessed map
func (b *BoltBucketImpl) ForEach(fn func(key []byte, value []byte) error, reader FileReader) error {
_, ok1 := boltKeysProcessed[b.name]
return b.Bucket.ForEach(func(k, v []byte) error {
v = append([]byte(nil), v...)
if _, ok2 := boltKeysProcessed[string(k)]; ok1 || ok2 {
if reader == nil {
return fmt.Errorf("reader callback is required for bucket %s", b.name)
}
processedValue, err := reader.Process(v)
if err != nil {
return err
}
return fn(k, processedValue)
}
return fn(k, v)
})
}
// Process values during Put/Get if the bucket name or key is in the boltKeysProcessed map
func (b *BoltBucketImpl) Put(key []byte, value []byte, writer FileWriter) error {
_, ok1 := boltKeysProcessed[string(key)]
_, ok2 := boltKeysProcessed[b.name]
value = append([]byte(nil), value...)
if ok1 || ok2 {
if writer == nil {
return fmt.Errorf("writer callback is required for key %s", string(key))
}
processedValue := writer.Process(value)
return b.Bucket.Put(key, processedValue)
}
return b.Bucket.Put(key, value)
}
// Process values during Put/Get if the bucket name or key is in the boltKeysProcessed map
func (b *BoltBucketImpl) Get(key []byte, reader FileReader) ([]byte, error) {
_, ok1 := boltKeysProcessed[string(key)]
_, ok2 := boltKeysProcessed[b.name]
if ok1 || ok2 {
if reader == nil {
return nil, fmt.Errorf("reader callback is required for key %s", string(key))
}
val := b.Bucket.Get(key)
if val == nil {
return nil, nil
}
processedVal, err := reader.Process(val)
if err != nil {
return nil, err
}
return processedVal, nil
}
return b.Bucket.Get(key), nil
}

66
util/extract.go Normal file
View File

@@ -0,0 +1,66 @@
// Copyright (c) 2023 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package util
import (
"math"
"reflect"
)
// extract numeric value (if possible) and returns a float64
func ExtractNumericValFloat64(v interface{}) (float64, bool) {
val := reflect.ValueOf(v)
if !val.IsValid() {
return 0, false
}
switch {
case val.CanFloat():
return val.Float(), true
case val.CanInt():
return float64(val.Int()), true
case val.CanUint():
return float64(val.Uint()), true
}
return 0, false
}
// extract numeric value (if possible) and returns a float32
func ExtractNumericValFloat32(v interface{}) (float32, bool) {
val := reflect.ValueOf(v)
if !val.IsValid() {
return 0, false
}
switch {
case val.CanFloat():
floatVal := val.Float()
if !IsValidFloat32(floatVal) {
return 0, false
}
return float32(floatVal), true
case val.CanInt():
return float32(val.Int()), true
case val.CanUint():
return float32(val.Uint()), true
}
return 0, false
}
func IsValidFloat32(val float64) bool {
return !math.IsNaN(val) && !math.IsInf(val, 0) && val <= math.MaxFloat32
}

129
util/file_callbacks.go Normal file
View File

@@ -0,0 +1,129 @@
// Copyright (c) 2026 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package util
import (
"fmt"
index "github.com/blevesearch/bleve_index_api"
)
// This file provides a mechanism for users of bleve to provide callbacks
// that can process data before it is written to disk, and after it is read
// from disk. This can be used for things like encryption, compression, etc.
// The user is responsible for ensuring that the writer and reader callbacks
// are compatible with each other, and that any state needed by the callbacks
// is managed appropriately. For example, if the writer callback uses a
// unique key or nonce per write, the reader callback must be able to
// determine the correct key or nonce to use for each read.
// The callbacks are identified by an id string, which is returned by the
// WriterHook. The same id string is passed to the ReaderHook when creating a reader.
// This allows the reader to determine which callback to use for a given file.
// Support for identifying all callbacks used by a given index and to remove
// selected callbacks associated with ids is provided via index.WriterIdsInUse()
// and index.DropWriterIds().
const DefaultFileCallbackId = ""
// FileWriter and FileReader interfaces are wrappers around the callback functions
// provided by the user. They provide a convenient way to apply the callbacks to data
// being written to or read from a file. They also store the id the callbacks,
// which can be useful for managing state across multiple reads and writes.
type FileWriter interface {
Process(data []byte) []byte
Id() string
}
type fileWriterImpl struct {
id string
processor func(data []byte) []byte
}
func NewFileWriter(context []byte) (FileWriter, error) {
rv := &fileWriterImpl{}
if index.WriterHook != nil {
var err error
rv.id, rv.processor, err = index.WriterHook(context)
if err != nil {
return nil, err
}
}
return rv, nil
}
func (w *fileWriterImpl) Process(data []byte) []byte {
if w.processor != nil {
return w.processor(data)
}
return data
}
func (w *fileWriterImpl) Id() string {
return w.id
}
type FileReader interface {
Process(data []byte) ([]byte, error)
Id() string
}
type fileReaderImpl struct {
id string
processor func(data []byte) ([]byte, error)
}
func NewFileReader(id string, context []byte) (FileReader, error) {
rv := &fileReaderImpl{
id: id,
}
if index.ReaderHook != nil {
var err error
rv.processor, err = index.ReaderHook(id, context)
if err != nil {
return nil, err
}
} else if id != "" {
return nil, fmt.Errorf("reader callback id %s provided but no ReaderHook is set", id)
}
return rv, nil
}
func (r *fileReaderImpl) Process(data []byte) ([]byte, error) {
if r.processor != nil {
return r.processor(data)
}
return data, nil
}
func (r *fileReaderImpl) Id() string {
return r.id
}
// -----------------------------------------------------------------------
// set of bolt keys and bucket names that require processing by the reader
// and writer callbacks.
var boltKeysProcessed = map[string]struct{}{
string(BoltDeletedKey): {},
string(BoltInternalKey): {},
string(BoltStatsKey): {},
string(BoltUpdatedFieldsKey): {},
}

25
util/json.go Normal file
View File

@@ -0,0 +1,25 @@
// Copyright (c) 2023 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package util
import (
"encoding/json"
)
// Should only be overwritten during process init()'ialization.
var (
MarshalJSON = json.Marshal
UnmarshalJSON = json.Unmarshal
)

36
util/keys.go Normal file
View File

@@ -0,0 +1,36 @@
// Copyright (c) 2025 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package util
var (
// Bolt keys
BoltSnapshotsBucket = []byte{'s'}
BoltTrainerKey = []byte{'t'}
BoltTrainCompleteKey = []byte{'c'}
BoltTrainedSamplesKey = []byte{'n'}
BoltPathKey = []byte{'p'}
BoltDeletedKey = []byte{'d'}
BoltInternalKey = []byte{'i'}
BoltMetaDataKey = []byte{'m'}
BoltMetaDataSegmentTypeKey = []byte("type")
BoltMetaDataSegmentVersionKey = []byte("version")
BoltMetaDataTimeStamp = []byte("timeStamp")
BoltStatsKey = []byte("stats")
BoltUpdatedFieldsKey = []byte("fields")
TotBytesWrittenKey = []byte("TotBytesWritten")
BoltMetaDataFileWriterIDKey = []byte("fileWriterID")
MappingInternalKey = []byte("_mapping")
)