snapshot of https://github.com/blevesearch/bleve.git (master, 2026-09-15; blobs > 131072B omitted)
Some checks failed
Coverage / coverage (push) Failing after 6s
Tests / test (1.24.x, ubuntu-latest) (push) Failing after 42s
Tests / test (1.26.x, ubuntu-latest) (push) Failing after 38s
Tests / test (1.25.x, ubuntu-latest) (push) Failing after 35s
Tests / test (1.25.x, windows-latest) (push) Has been cancelled
Tests / test (1.26.x, macos-latest) (push) Has been cancelled
Tests / test (1.26.x, windows-latest) (push) Has been cancelled
Tests / test (1.24.x, macos-latest) (push) Has been cancelled
Tests / test (1.24.x, windows-latest) (push) Has been cancelled
Tests / test (1.25.x, macos-latest) (push) Has been cancelled
Some checks failed
Coverage / coverage (push) Failing after 6s
Tests / test (1.24.x, ubuntu-latest) (push) Failing after 42s
Tests / test (1.26.x, ubuntu-latest) (push) Failing after 38s
Tests / test (1.25.x, ubuntu-latest) (push) Failing after 35s
Tests / test (1.25.x, windows-latest) (push) Has been cancelled
Tests / test (1.26.x, macos-latest) (push) Has been cancelled
Tests / test (1.26.x, windows-latest) (push) Has been cancelled
Tests / test (1.24.x, macos-latest) (push) Has been cancelled
Tests / test (1.24.x, windows-latest) (push) Has been cancelled
Tests / test (1.25.x, macos-latest) (push) Has been cancelled
This commit is contained in:
130
util/bitset.go
Normal file
130
util/bitset.go
Normal file
@@ -0,0 +1,130 @@
|
||||
// Copyright (c) 2026 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package util
|
||||
|
||||
import (
|
||||
"math/bits"
|
||||
|
||||
"github.com/RoaringBitmap/roaring/v2"
|
||||
)
|
||||
|
||||
type Bitset struct {
|
||||
data []uint64
|
||||
numBits int
|
||||
exclude *roaring.Bitmap
|
||||
}
|
||||
|
||||
// NewBitset initializes a bitset capable of holding numbers up to maxVal
|
||||
func NewBitset(maxVal int, exclude *roaring.Bitmap) *Bitset {
|
||||
// We need (maxVal / 64) + 1 buckets to hold up to maxVal
|
||||
size := (maxVal / 64) + 1
|
||||
return &Bitset{
|
||||
data: make([]uint64, size),
|
||||
numBits: maxVal,
|
||||
exclude: exclude,
|
||||
}
|
||||
}
|
||||
|
||||
// Add inserts a value into the bitset (safely handles duplicates)
|
||||
func (b *Bitset) Add(val int) {
|
||||
if b.exclude != nil && b.exclude.Contains(uint32(val)) {
|
||||
return
|
||||
}
|
||||
bucket := val >> 6 // Equivalent to val / 64
|
||||
bit := uint(val & 63) // Equivalent to val % 64
|
||||
|
||||
// Set the bit to 1 using bitwise OR
|
||||
b.data[bucket] |= (1 << bit)
|
||||
}
|
||||
|
||||
// Remove deletes a value from the bitset
|
||||
func (b *Bitset) Remove(val int) {
|
||||
bucket := val >> 6
|
||||
bit := uint(val & 63)
|
||||
|
||||
// Set the bit to 0 using bitwise AND with the complement
|
||||
b.data[bucket] &^= (1 << bit)
|
||||
}
|
||||
|
||||
// Contains checks if a value exists in the bitset
|
||||
func (b *Bitset) Contains(val int) bool {
|
||||
bucket := val >> 6
|
||||
bit := uint(val & 63)
|
||||
|
||||
return (b.data[bucket] & (1 << bit)) != 0
|
||||
}
|
||||
|
||||
// Invert flips all bits in the bitset,
|
||||
// effectively turning all 1s to 0s and vice versa
|
||||
func (b *Bitset) Invert() {
|
||||
for i := range b.data {
|
||||
b.data[i] = ^b.data[i]
|
||||
}
|
||||
// the flip above sets the trailing bits beyond numBits in the last
|
||||
// bucket(s), which do not correspond to valid values - clear them so
|
||||
// that Iterate and Count never see them
|
||||
lastBucket := b.numBits >> 6
|
||||
if lastBucket < len(b.data) {
|
||||
b.data[lastBucket] &= (1 << uint(b.numBits&63)) - 1
|
||||
for i := lastBucket + 1; i < len(b.data); i++ {
|
||||
b.data[i] = 0
|
||||
}
|
||||
}
|
||||
if b.exclude != nil {
|
||||
it := b.exclude.Iterator()
|
||||
for it.HasNext() {
|
||||
bit := uint64(it.Next())
|
||||
word := bit / 64
|
||||
if word < uint64(len(b.data)) {
|
||||
b.data[word] &^= uint64(1) << (bit % 64)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Iterate calls the provided function for every integer recorded in the bitset, in ascending order
|
||||
func (b *Bitset) Iterate(f func(int)) {
|
||||
for bucketIdx, bucket := range b.data {
|
||||
// If the entire 64-bit block is 0, skip it entirely for speed
|
||||
if bucket == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check all 64 bits in this bucket
|
||||
for bitIdx := 0; bitIdx < 64; bitIdx++ {
|
||||
if (bucket & (1 << uint(bitIdx))) != 0 {
|
||||
// Reconstruct the original integer
|
||||
originalVal := (bucketIdx << 6) + bitIdx
|
||||
f(originalVal)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Bitset) Count() int {
|
||||
count := 0
|
||||
|
||||
for _, word := range b.data {
|
||||
count += bits.OnesCount64(word)
|
||||
}
|
||||
|
||||
return count
|
||||
}
|
||||
|
||||
func (b *Bitset) Clear() {
|
||||
for i := range b.data {
|
||||
b.data[i] = 0
|
||||
}
|
||||
}
|
||||
205
util/bitset_test.go
Normal file
205
util/bitset_test.go
Normal file
@@ -0,0 +1,205 @@
|
||||
// Copyright (c) 2026 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package util
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/RoaringBitmap/roaring/v2"
|
||||
)
|
||||
|
||||
func TestBitsetAddContains(t *testing.T) {
|
||||
b := NewBitset(100, nil)
|
||||
|
||||
// values not yet added must not be present
|
||||
for _, v := range []int{0, 1, 63, 64, 65, 100} {
|
||||
if b.Contains(v) {
|
||||
t.Fatalf("expected %d to be absent from an empty bitset", v)
|
||||
}
|
||||
}
|
||||
|
||||
// exercise word boundaries explicitly: 63 is the last bit of word 0,
|
||||
// 64 is the first bit of word 1, 65 the second
|
||||
added := []int{0, 1, 63, 64, 65, 100}
|
||||
for _, v := range added {
|
||||
b.Add(v)
|
||||
}
|
||||
for _, v := range added {
|
||||
if !b.Contains(v) {
|
||||
t.Fatalf("expected %d to be present after Add", v)
|
||||
}
|
||||
}
|
||||
|
||||
// a value between two set bits must remain absent
|
||||
if b.Contains(50) {
|
||||
t.Fatalf("expected 50 to be absent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitsetAddDuplicate(t *testing.T) {
|
||||
b := NewBitset(100, nil)
|
||||
b.Add(42)
|
||||
b.Add(42)
|
||||
if got := b.Count(); got != 1 {
|
||||
t.Fatalf("expected duplicate Add to keep count at 1, got %d", got)
|
||||
}
|
||||
if !b.Contains(42) {
|
||||
t.Fatalf("expected 42 to be present")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitsetRemove(t *testing.T) {
|
||||
b := NewBitset(100, nil)
|
||||
b.Add(10)
|
||||
b.Add(64)
|
||||
b.Remove(10)
|
||||
if b.Contains(10) {
|
||||
t.Fatalf("expected 10 to be absent after Remove")
|
||||
}
|
||||
if !b.Contains(64) {
|
||||
t.Fatalf("expected 64 to still be present")
|
||||
}
|
||||
// removing an absent value must be a no-op
|
||||
b.Remove(99)
|
||||
if b.Count() != 1 {
|
||||
t.Fatalf("expected count 1 after removing an absent value, got %d", b.Count())
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitsetCountAndClear(t *testing.T) {
|
||||
b := NewBitset(200, nil)
|
||||
for _, v := range []int{0, 5, 63, 64, 128, 200} {
|
||||
b.Add(v)
|
||||
}
|
||||
if got := b.Count(); got != 6 {
|
||||
t.Fatalf("expected count 6, got %d", got)
|
||||
}
|
||||
b.Clear()
|
||||
if got := b.Count(); got != 0 {
|
||||
t.Fatalf("expected count 0 after Clear, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitsetIterateAscending(t *testing.T) {
|
||||
b := NewBitset(200, nil)
|
||||
// add out of order to confirm Iterate returns ascending order
|
||||
for _, v := range []int{130, 0, 64, 63, 7} {
|
||||
b.Add(v)
|
||||
}
|
||||
var got []int
|
||||
b.Iterate(func(v int) {
|
||||
got = append(got, v)
|
||||
})
|
||||
want := []int{0, 7, 63, 64, 130}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("expected Iterate to yield %v in ascending order, got %v", want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitsetExcludeBlocksAdd(t *testing.T) {
|
||||
exclude := roaring.New()
|
||||
exclude.AddInt(5)
|
||||
exclude.AddInt(70)
|
||||
|
||||
b := NewBitset(100, exclude)
|
||||
b.Add(5) // excluded, must be ignored
|
||||
b.Add(6) // allowed
|
||||
b.Add(70) // excluded, must be ignored
|
||||
|
||||
if b.Contains(5) {
|
||||
t.Fatalf("expected excluded value 5 to be blocked by Add")
|
||||
}
|
||||
if b.Contains(70) {
|
||||
t.Fatalf("expected excluded value 70 to be blocked by Add")
|
||||
}
|
||||
if !b.Contains(6) {
|
||||
t.Fatalf("expected non-excluded value 6 to be present")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitsetInvertRespectsNumBits(t *testing.T) {
|
||||
// numDocs = 10 means the valid doc IDs are 0..9. NewBitset allocates a
|
||||
// full 64-bit word, so bits 10..63 are unused and must never surface.
|
||||
numDocs := 10
|
||||
b := NewBitset(numDocs, nil)
|
||||
b.Add(3)
|
||||
b.Add(7)
|
||||
|
||||
b.Invert()
|
||||
|
||||
var got []int
|
||||
b.Iterate(func(v int) {
|
||||
got = append(got, v)
|
||||
})
|
||||
|
||||
// after inverting, exactly the doc IDs in [0, 10) that were NOT set
|
||||
// should be present - and nothing >= 10
|
||||
want := []int{0, 1, 2, 4, 5, 6, 8, 9}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("expected inverted bitset to yield %v, got %v", want, got)
|
||||
}
|
||||
if got := b.Count(); got != len(want) {
|
||||
t.Fatalf("expected inverted count %d, got %d", len(want), got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitsetInvertWordBoundaries(t *testing.T) {
|
||||
// exercise numBits at and around 64-bit word boundaries to make sure
|
||||
// the trailing-bit mask is computed correctly
|
||||
for _, numDocs := range []int{1, 63, 64, 65, 128, 129} {
|
||||
b := NewBitset(numDocs, nil)
|
||||
b.Invert()
|
||||
max := -1
|
||||
b.Iterate(func(v int) {
|
||||
if v > max {
|
||||
max = v
|
||||
}
|
||||
})
|
||||
// every valid doc ID [0, numDocs) should be present after inverting
|
||||
// an empty bitset, and none at or beyond numDocs
|
||||
if got := b.Count(); got != numDocs {
|
||||
t.Fatalf("numDocs=%d: expected inverted-empty count %d, got %d",
|
||||
numDocs, numDocs, got)
|
||||
}
|
||||
if max >= numDocs {
|
||||
t.Fatalf("numDocs=%d: Invert surfaced out-of-range value %d",
|
||||
numDocs, max)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitsetInvertClearsExcluded(t *testing.T) {
|
||||
// excluded docs must remain unset even after Invert, since they are
|
||||
// never valid hits
|
||||
exclude := roaring.New()
|
||||
exclude.AddInt(2)
|
||||
exclude.AddInt(8)
|
||||
|
||||
b := NewBitset(10, exclude)
|
||||
b.Add(3)
|
||||
b.Invert()
|
||||
|
||||
if b.Contains(2) || b.Contains(8) {
|
||||
t.Fatalf("expected excluded docs to stay unset after Invert")
|
||||
}
|
||||
if b.Contains(3) {
|
||||
t.Fatalf("expected the originally-set doc 3 to be cleared after Invert")
|
||||
}
|
||||
// a normal, non-excluded, originally-unset doc should now be set
|
||||
if !b.Contains(0) {
|
||||
t.Fatalf("expected doc 0 to be set after Invert")
|
||||
}
|
||||
}
|
||||
170
util/bolt.go
Normal file
170
util/bolt.go
Normal file
@@ -0,0 +1,170 @@
|
||||
// Copyright (c) 2026 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package util
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
bolt "go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
// All of the bolt impls provide a layer of indirection to allow for processing
|
||||
// of values as they are read/written to bolt depending on the key or bucket name
|
||||
// This is used to allow better support for file callbacks
|
||||
|
||||
// wrapper around bolt.DB
|
||||
type RootBoltImpl struct {
|
||||
*bolt.DB
|
||||
}
|
||||
|
||||
// wrapper around bolt.Tx
|
||||
type BoltTxImpl struct {
|
||||
*bolt.Tx
|
||||
}
|
||||
|
||||
// wrapper around bolt.Bucket
|
||||
type BoltBucketImpl struct {
|
||||
*bolt.Bucket
|
||||
|
||||
name string // store the name of the bucket during creation
|
||||
}
|
||||
|
||||
func OpenBolt(path string, mode os.FileMode, options *bolt.Options) (*RootBoltImpl, error) {
|
||||
db, err := bolt.Open(path, mode, options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &RootBoltImpl{DB: db}, nil
|
||||
}
|
||||
|
||||
func (r *RootBoltImpl) Begin(writable bool) (*BoltTxImpl, error) {
|
||||
tx, err := r.DB.Begin(writable)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &BoltTxImpl{Tx: tx}, nil
|
||||
}
|
||||
|
||||
func (r *RootBoltImpl) View(fn func(*BoltTxImpl) error) error {
|
||||
return r.DB.View(func(tx *bolt.Tx) error {
|
||||
return fn(&BoltTxImpl{Tx: tx})
|
||||
})
|
||||
}
|
||||
|
||||
func (r *RootBoltImpl) Update(fn func(*BoltTxImpl) error) error {
|
||||
return r.DB.Update(func(tx *bolt.Tx) error {
|
||||
return fn(&BoltTxImpl{Tx: tx})
|
||||
})
|
||||
}
|
||||
|
||||
func (tx *BoltTxImpl) CreateBucketIfNotExists(name []byte) (*BoltBucketImpl, error) {
|
||||
bucket, err := tx.Tx.CreateBucketIfNotExists(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &BoltBucketImpl{
|
||||
name: string(name),
|
||||
Bucket: bucket,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (tx *BoltTxImpl) Bucket(name []byte) *BoltBucketImpl {
|
||||
bucket := tx.Tx.Bucket(name)
|
||||
if bucket == nil {
|
||||
return nil
|
||||
}
|
||||
return &BoltBucketImpl{
|
||||
name: string(name),
|
||||
Bucket: bucket,
|
||||
}
|
||||
}
|
||||
|
||||
func (b *BoltBucketImpl) GetBucket(name []byte) *BoltBucketImpl {
|
||||
bucket := b.Bucket.Bucket(name)
|
||||
if bucket == nil {
|
||||
return nil
|
||||
}
|
||||
return &BoltBucketImpl{
|
||||
name: string(name),
|
||||
Bucket: bucket,
|
||||
}
|
||||
}
|
||||
|
||||
func (b *BoltBucketImpl) CreateBucketIfNotExists(name []byte) (*BoltBucketImpl, error) {
|
||||
bucket, err := b.Bucket.CreateBucketIfNotExists(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &BoltBucketImpl{
|
||||
name: string(name),
|
||||
Bucket: bucket,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Process values during ForEach if the bucket name or key is in the boltKeysProcessed map
|
||||
func (b *BoltBucketImpl) ForEach(fn func(key []byte, value []byte) error, reader FileReader) error {
|
||||
_, ok1 := boltKeysProcessed[b.name]
|
||||
return b.Bucket.ForEach(func(k, v []byte) error {
|
||||
v = append([]byte(nil), v...)
|
||||
if _, ok2 := boltKeysProcessed[string(k)]; ok1 || ok2 {
|
||||
if reader == nil {
|
||||
return fmt.Errorf("reader callback is required for bucket %s", b.name)
|
||||
}
|
||||
processedValue, err := reader.Process(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return fn(k, processedValue)
|
||||
}
|
||||
return fn(k, v)
|
||||
})
|
||||
}
|
||||
|
||||
// Process values during Put/Get if the bucket name or key is in the boltKeysProcessed map
|
||||
func (b *BoltBucketImpl) Put(key []byte, value []byte, writer FileWriter) error {
|
||||
_, ok1 := boltKeysProcessed[string(key)]
|
||||
_, ok2 := boltKeysProcessed[b.name]
|
||||
value = append([]byte(nil), value...)
|
||||
if ok1 || ok2 {
|
||||
if writer == nil {
|
||||
return fmt.Errorf("writer callback is required for key %s", string(key))
|
||||
}
|
||||
processedValue := writer.Process(value)
|
||||
return b.Bucket.Put(key, processedValue)
|
||||
}
|
||||
return b.Bucket.Put(key, value)
|
||||
}
|
||||
|
||||
// Process values during Put/Get if the bucket name or key is in the boltKeysProcessed map
|
||||
func (b *BoltBucketImpl) Get(key []byte, reader FileReader) ([]byte, error) {
|
||||
_, ok1 := boltKeysProcessed[string(key)]
|
||||
_, ok2 := boltKeysProcessed[b.name]
|
||||
if ok1 || ok2 {
|
||||
if reader == nil {
|
||||
return nil, fmt.Errorf("reader callback is required for key %s", string(key))
|
||||
}
|
||||
val := b.Bucket.Get(key)
|
||||
if val == nil {
|
||||
return nil, nil
|
||||
}
|
||||
processedVal, err := reader.Process(val)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return processedVal, nil
|
||||
}
|
||||
return b.Bucket.Get(key), nil
|
||||
}
|
||||
66
util/extract.go
Normal file
66
util/extract.go
Normal file
@@ -0,0 +1,66 @@
|
||||
// Copyright (c) 2023 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package util
|
||||
|
||||
import (
|
||||
"math"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
// extract numeric value (if possible) and returns a float64
|
||||
func ExtractNumericValFloat64(v interface{}) (float64, bool) {
|
||||
val := reflect.ValueOf(v)
|
||||
if !val.IsValid() {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
switch {
|
||||
case val.CanFloat():
|
||||
return val.Float(), true
|
||||
case val.CanInt():
|
||||
return float64(val.Int()), true
|
||||
case val.CanUint():
|
||||
return float64(val.Uint()), true
|
||||
}
|
||||
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// extract numeric value (if possible) and returns a float32
|
||||
func ExtractNumericValFloat32(v interface{}) (float32, bool) {
|
||||
val := reflect.ValueOf(v)
|
||||
if !val.IsValid() {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
switch {
|
||||
case val.CanFloat():
|
||||
floatVal := val.Float()
|
||||
if !IsValidFloat32(floatVal) {
|
||||
return 0, false
|
||||
}
|
||||
return float32(floatVal), true
|
||||
case val.CanInt():
|
||||
return float32(val.Int()), true
|
||||
case val.CanUint():
|
||||
return float32(val.Uint()), true
|
||||
}
|
||||
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func IsValidFloat32(val float64) bool {
|
||||
return !math.IsNaN(val) && !math.IsInf(val, 0) && val <= math.MaxFloat32
|
||||
}
|
||||
129
util/file_callbacks.go
Normal file
129
util/file_callbacks.go
Normal file
@@ -0,0 +1,129 @@
|
||||
// Copyright (c) 2026 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package util
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
// This file provides a mechanism for users of bleve to provide callbacks
|
||||
// that can process data before it is written to disk, and after it is read
|
||||
// from disk. This can be used for things like encryption, compression, etc.
|
||||
|
||||
// The user is responsible for ensuring that the writer and reader callbacks
|
||||
// are compatible with each other, and that any state needed by the callbacks
|
||||
// is managed appropriately. For example, if the writer callback uses a
|
||||
// unique key or nonce per write, the reader callback must be able to
|
||||
// determine the correct key or nonce to use for each read.
|
||||
|
||||
// The callbacks are identified by an id string, which is returned by the
|
||||
// WriterHook. The same id string is passed to the ReaderHook when creating a reader.
|
||||
// This allows the reader to determine which callback to use for a given file.
|
||||
|
||||
// Support for identifying all callbacks used by a given index and to remove
|
||||
// selected callbacks associated with ids is provided via index.WriterIdsInUse()
|
||||
// and index.DropWriterIds().
|
||||
|
||||
const DefaultFileCallbackId = ""
|
||||
|
||||
// FileWriter and FileReader interfaces are wrappers around the callback functions
|
||||
// provided by the user. They provide a convenient way to apply the callbacks to data
|
||||
// being written to or read from a file. They also store the id the callbacks,
|
||||
// which can be useful for managing state across multiple reads and writes.
|
||||
type FileWriter interface {
|
||||
Process(data []byte) []byte
|
||||
Id() string
|
||||
}
|
||||
type fileWriterImpl struct {
|
||||
id string
|
||||
processor func(data []byte) []byte
|
||||
}
|
||||
|
||||
func NewFileWriter(context []byte) (FileWriter, error) {
|
||||
rv := &fileWriterImpl{}
|
||||
|
||||
if index.WriterHook != nil {
|
||||
var err error
|
||||
rv.id, rv.processor, err = index.WriterHook(context)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return rv, nil
|
||||
}
|
||||
|
||||
func (w *fileWriterImpl) Process(data []byte) []byte {
|
||||
if w.processor != nil {
|
||||
return w.processor(data)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func (w *fileWriterImpl) Id() string {
|
||||
return w.id
|
||||
}
|
||||
|
||||
type FileReader interface {
|
||||
Process(data []byte) ([]byte, error)
|
||||
Id() string
|
||||
}
|
||||
|
||||
type fileReaderImpl struct {
|
||||
id string
|
||||
processor func(data []byte) ([]byte, error)
|
||||
}
|
||||
|
||||
func NewFileReader(id string, context []byte) (FileReader, error) {
|
||||
rv := &fileReaderImpl{
|
||||
id: id,
|
||||
}
|
||||
|
||||
if index.ReaderHook != nil {
|
||||
var err error
|
||||
rv.processor, err = index.ReaderHook(id, context)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else if id != "" {
|
||||
return nil, fmt.Errorf("reader callback id %s provided but no ReaderHook is set", id)
|
||||
}
|
||||
|
||||
return rv, nil
|
||||
}
|
||||
|
||||
func (r *fileReaderImpl) Process(data []byte) ([]byte, error) {
|
||||
if r.processor != nil {
|
||||
return r.processor(data)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (r *fileReaderImpl) Id() string {
|
||||
return r.id
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
// set of bolt keys and bucket names that require processing by the reader
|
||||
// and writer callbacks.
|
||||
var boltKeysProcessed = map[string]struct{}{
|
||||
string(BoltDeletedKey): {},
|
||||
string(BoltInternalKey): {},
|
||||
string(BoltStatsKey): {},
|
||||
string(BoltUpdatedFieldsKey): {},
|
||||
}
|
||||
25
util/json.go
Normal file
25
util/json.go
Normal file
@@ -0,0 +1,25 @@
|
||||
// Copyright (c) 2023 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package util
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
// Should only be overwritten during process init()'ialization.
|
||||
var (
|
||||
MarshalJSON = json.Marshal
|
||||
UnmarshalJSON = json.Unmarshal
|
||||
)
|
||||
36
util/keys.go
Normal file
36
util/keys.go
Normal file
@@ -0,0 +1,36 @@
|
||||
// Copyright (c) 2025 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package util
|
||||
|
||||
var (
|
||||
// Bolt keys
|
||||
BoltSnapshotsBucket = []byte{'s'}
|
||||
BoltTrainerKey = []byte{'t'}
|
||||
BoltTrainCompleteKey = []byte{'c'}
|
||||
BoltTrainedSamplesKey = []byte{'n'}
|
||||
BoltPathKey = []byte{'p'}
|
||||
BoltDeletedKey = []byte{'d'}
|
||||
BoltInternalKey = []byte{'i'}
|
||||
BoltMetaDataKey = []byte{'m'}
|
||||
BoltMetaDataSegmentTypeKey = []byte("type")
|
||||
BoltMetaDataSegmentVersionKey = []byte("version")
|
||||
BoltMetaDataTimeStamp = []byte("timeStamp")
|
||||
BoltStatsKey = []byte("stats")
|
||||
BoltUpdatedFieldsKey = []byte("fields")
|
||||
TotBytesWrittenKey = []byte("TotBytesWritten")
|
||||
BoltMetaDataFileWriterIDKey = []byte("fileWriterID")
|
||||
|
||||
MappingInternalKey = []byte("_mapping")
|
||||
)
|
||||
Reference in New Issue
Block a user