snapshot of https://github.com/nats-io/nats-server.git (main, 2026-09-15; blobs > 131072B omitted)
Some checks failed
NATS Server Tests / Lint (push) Failing after 3s
NATS Server Tests / Build (Latest Go) (push) Has been skipped
NATS Server Tests / Build (Minimum Go) (push) Has been skipped
NATS Server Tests / Build (Minimum Go, windows-2022) (push) Has been skipped
NATS Server Tests / Build (Minimum Go, windows-2025) (push) Has been skipped
NATS Server Tests / Test Stores (push) Has been skipped
NATS Server Tests / Test JetStream (push) Has been skipped
NATS Server Tests / Test Raft (push) Has been skipped
NATS Server Tests / Test JetStream Consumers (push) Has been skipped
NATS Server Tests / Test JetStream Cluster 1 (push) Has been skipped
NATS Server Tests / Test JetStream Cluster 2 (push) Has been skipped
NATS Server Tests / Test JetStream Cluster 3 (push) Has been skipped
NATS Server Tests / Test JetStream Cluster 4 (push) Has been skipped
NATS Server Tests / Test JetStream Supercluster (push) Has been skipped
NATS Server Tests / Test No-Race 1 (push) Has been skipped
NATS Server Tests / Test No-Race 2 (push) Has been skipped
NATS Server Tests / Test MQTT (push) Has been skipped
NATS Server Tests / Test Message Tracing (push) Has been skipped
NATS Server Tests / Test JWT (push) Has been skipped
NATS Server Tests / Test Remaining Server Tests (push) Has been skipped
NATS Server Tests / Test Other Packages (push) Has been skipped
Vulnerability Scan / govulncheck (push) Failing after 12m2s
Some checks failed
NATS Server Tests / Lint (push) Failing after 3s
NATS Server Tests / Build (Latest Go) (push) Has been skipped
NATS Server Tests / Build (Minimum Go) (push) Has been skipped
NATS Server Tests / Build (Minimum Go, windows-2022) (push) Has been skipped
NATS Server Tests / Build (Minimum Go, windows-2025) (push) Has been skipped
NATS Server Tests / Test Stores (push) Has been skipped
NATS Server Tests / Test JetStream (push) Has been skipped
NATS Server Tests / Test Raft (push) Has been skipped
NATS Server Tests / Test JetStream Consumers (push) Has been skipped
NATS Server Tests / Test JetStream Cluster 1 (push) Has been skipped
NATS Server Tests / Test JetStream Cluster 2 (push) Has been skipped
NATS Server Tests / Test JetStream Cluster 3 (push) Has been skipped
NATS Server Tests / Test JetStream Cluster 4 (push) Has been skipped
NATS Server Tests / Test JetStream Supercluster (push) Has been skipped
NATS Server Tests / Test No-Race 1 (push) Has been skipped
NATS Server Tests / Test No-Race 2 (push) Has been skipped
NATS Server Tests / Test MQTT (push) Has been skipped
NATS Server Tests / Test Message Tracing (push) Has been skipped
NATS Server Tests / Test JWT (push) Has been skipped
NATS Server Tests / Test Remaining Server Tests (push) Has been skipped
NATS Server Tests / Test Other Packages (push) Has been skipped
Vulnerability Scan / govulncheck (push) Failing after 12m2s
This commit is contained in:
198
server/avl/norace_test.go
Normal file
198
server/avl/norace_test.go
Normal file
@@ -0,0 +1,198 @@
|
||||
// Copyright 2023-2026 The NATS Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//go:build !race && !skip_no_race_tests && !skip_no_race_1_tests
|
||||
|
||||
package avl
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"math"
|
||||
"math/rand/v2"
|
||||
"runtime"
|
||||
"runtime/debug"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Print Results: go test -v --args --results
|
||||
var printResults = flag.Bool("results", false, "Enable Results Logging")
|
||||
|
||||
// SequenceSet memory tests vs dmaps.
|
||||
func TestNoRaceSeqSetSizeComparison(t *testing.T) {
|
||||
// Create 5M random entries (dupes possible but ok for this test) out of 8M range.
|
||||
num := 5_000_000
|
||||
max := 7_000_000
|
||||
|
||||
seqs := make([]uint64, 0, num)
|
||||
for i := 0; i < num; i++ {
|
||||
n := uint64(rand.Int64N(int64(max + 1)))
|
||||
seqs = append(seqs, n)
|
||||
}
|
||||
|
||||
runtime.GC()
|
||||
// Disable to get stable results.
|
||||
gcp := debug.SetGCPercent(-1)
|
||||
defer debug.SetGCPercent(gcp)
|
||||
|
||||
mem := runtime.MemStats{}
|
||||
runtime.ReadMemStats(&mem)
|
||||
inUseBefore := mem.HeapInuse
|
||||
|
||||
dmap := make(map[uint64]struct{}, num)
|
||||
for _, n := range seqs {
|
||||
dmap[n] = struct{}{}
|
||||
}
|
||||
runtime.ReadMemStats(&mem)
|
||||
dmapUse := mem.HeapInuse - inUseBefore
|
||||
inUseBefore = mem.HeapInuse
|
||||
|
||||
// Now do SequenceSet on same dataset.
|
||||
var sset SequenceSet
|
||||
for _, n := range seqs {
|
||||
sset.Insert(n)
|
||||
}
|
||||
|
||||
runtime.ReadMemStats(&mem)
|
||||
seqSetUse := mem.HeapInuse - inUseBefore
|
||||
|
||||
if seqSetUse > 2*1024*1024 {
|
||||
t.Fatalf("Expected SequenceSet size to be < 2M, got %v", friendlyBytes(seqSetUse))
|
||||
}
|
||||
if seqSetUse*50 > dmapUse {
|
||||
t.Fatalf("Expected SequenceSet to be at least 50x better then dmap approach: %v vs %v",
|
||||
friendlyBytes(seqSetUse),
|
||||
friendlyBytes(dmapUse),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoRaceSeqSetEncodeLarge(t *testing.T) {
|
||||
num := 2_500_000
|
||||
max := 5_000_000
|
||||
|
||||
dmap := make(map[uint64]struct{}, num)
|
||||
var ss SequenceSet
|
||||
for i := 0; i < num; i++ {
|
||||
n := uint64(rand.Int64N(int64(max + 1)))
|
||||
ss.Insert(n)
|
||||
dmap[n] = struct{}{}
|
||||
}
|
||||
|
||||
// Disable to get stable results.
|
||||
gcp := debug.SetGCPercent(-1)
|
||||
defer debug.SetGCPercent(gcp)
|
||||
|
||||
// In general should be about the same, but can see some variability.
|
||||
expected := time.Millisecond
|
||||
|
||||
start := time.Now()
|
||||
b := ss.Encode(nil)
|
||||
|
||||
if elapsed := time.Since(start); elapsed > expected {
|
||||
t.Fatalf("Expected encode of %d items with encoded size %v to take less than %v, got %v",
|
||||
num, friendlyBytes(len(b)), expected, elapsed)
|
||||
} else {
|
||||
logResults("Encode time for %d items was %v, encoded size is %v\n", num, elapsed, friendlyBytes(len(b)))
|
||||
}
|
||||
|
||||
start = time.Now()
|
||||
ss2, _, err := Decode(b)
|
||||
require_NoError(t, err)
|
||||
if elapsed := time.Since(start); elapsed > expected {
|
||||
t.Fatalf("Expected decode to take less than %v, got %v", expected, elapsed)
|
||||
} else {
|
||||
logResults("Decode time is %v\n", elapsed)
|
||||
}
|
||||
require_True(t, ss.Nodes() == ss2.Nodes())
|
||||
require_True(t, ss.Size() == ss2.Size())
|
||||
}
|
||||
|
||||
func TestNoRaceSeqSetRelativeSpeed(t *testing.T) {
|
||||
// Create 1M random entries (dupes possible but ok for this test) out of 3M range.
|
||||
num := 1_000_000
|
||||
max := 3_000_000
|
||||
|
||||
seqs := make([]uint64, 0, num)
|
||||
for i := 0; i < num; i++ {
|
||||
n := uint64(rand.Int64N(int64(max + 1)))
|
||||
seqs = append(seqs, n)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
// Now do SequenceSet on same dataset.
|
||||
var sset SequenceSet
|
||||
for _, n := range seqs {
|
||||
sset.Insert(n)
|
||||
}
|
||||
ssInsertElapsed := time.Since(start)
|
||||
logResults("Inserts SequenceSet: %v for %d items\n", ssInsertElapsed, num)
|
||||
|
||||
start = time.Now()
|
||||
for _, n := range seqs {
|
||||
if ok := sset.Exists(n); !ok {
|
||||
t.Fatalf("Should exist")
|
||||
}
|
||||
}
|
||||
ssLookupElapsed := time.Since(start)
|
||||
logResults("Lookups: %v\n", ssLookupElapsed)
|
||||
|
||||
// Now do a map.
|
||||
dmap := make(map[uint64]struct{})
|
||||
start = time.Now()
|
||||
for _, n := range seqs {
|
||||
dmap[n] = struct{}{}
|
||||
}
|
||||
mapInsertElapsed := time.Since(start)
|
||||
logResults("Inserts Map[uint64]: %v for %d items\n", mapInsertElapsed, num)
|
||||
|
||||
start = time.Now()
|
||||
for _, n := range seqs {
|
||||
if _, ok := dmap[n]; !ok {
|
||||
t.Fatalf("Should exist")
|
||||
}
|
||||
}
|
||||
mapLookupElapsed := time.Since(start)
|
||||
logResults("Lookups: %v\n", mapLookupElapsed)
|
||||
|
||||
// In general we are between 1.5 and 1.75 times slower atm then a straight map.
|
||||
// Let's test an upper bound of 2x for now.
|
||||
if mapInsertElapsed*2 <= ssInsertElapsed {
|
||||
t.Fatalf("Expected SequenceSet insert to be no more than 2x slower (%v vs %v)", mapInsertElapsed, ssInsertElapsed)
|
||||
}
|
||||
|
||||
if mapLookupElapsed*3 <= ssLookupElapsed {
|
||||
t.Fatalf("Expected SequenceSet lookups to be no more than 3x slower (%v vs %v)", mapLookupElapsed, ssLookupElapsed)
|
||||
}
|
||||
}
|
||||
|
||||
// friendlyBytes returns a string with the given bytes int64
|
||||
// represented as a size, such as 1KB, 10MB, etc...
|
||||
func friendlyBytes[T int | uint64 | int64](bytes T) string {
|
||||
fbytes := float64(bytes)
|
||||
base := 1024
|
||||
pre := []string{"K", "M", "G", "T", "P", "E"}
|
||||
if fbytes < float64(base) {
|
||||
return fmt.Sprintf("%v B", fbytes)
|
||||
}
|
||||
exp := int(math.Log(fbytes) / math.Log(float64(base)))
|
||||
index := exp - 1
|
||||
return fmt.Sprintf("%.2f %sB", fbytes/math.Pow(float64(base), float64(exp)), pre[index])
|
||||
}
|
||||
|
||||
func logResults(format string, args ...any) {
|
||||
if *printResults {
|
||||
fmt.Printf(format, args...)
|
||||
}
|
||||
}
|
||||
715
server/avl/seqset.go
Normal file
715
server/avl/seqset.go
Normal file
@@ -0,0 +1,715 @@
|
||||
// Copyright 2023-2026 The NATS Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package avl
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"math/bits"
|
||||
"slices"
|
||||
)
|
||||
|
||||
// SequenceSet is a memory and encoding optimized set for storing unsigned ints.
|
||||
//
|
||||
// SequenceSet is ~80-100 times more efficient memory wise than a map[uint64]struct{}.
|
||||
// SequenceSet is ~1.75 times slower at inserts than the same map.
|
||||
// SequenceSet is not thread safe.
|
||||
//
|
||||
// We use an AVL tree with nodes that hold bitmasks for set membership.
|
||||
//
|
||||
// Encoding will convert to a space optimized encoding using bitmasks.
|
||||
type SequenceSet struct {
|
||||
root *node // root node
|
||||
size int // number of items
|
||||
nodes int // number of nodes
|
||||
// Having this here vs on the stack in Insert/Delete
|
||||
// makes a difference in memory usage.
|
||||
changed bool
|
||||
}
|
||||
|
||||
// Insert will insert the sequence into the set.
|
||||
// The tree will be balanced inline.
|
||||
func (ss *SequenceSet) Insert(seq uint64) {
|
||||
// If a node covering seq already exists, setting a bit can not change the
|
||||
// tree shape, so skip the recursive descent and rebalance checks.
|
||||
for n := ss.root; n != nil; {
|
||||
if seq < n.base {
|
||||
n = n.l
|
||||
} else if seq >= n.base+numEntries {
|
||||
n = n.r
|
||||
} else {
|
||||
n.set(seq, &ss.changed)
|
||||
if ss.changed {
|
||||
ss.changed = false
|
||||
ss.size++
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
if ss.root = ss.root.insert(seq, &ss.changed, &ss.nodes); ss.changed {
|
||||
ss.changed = false
|
||||
ss.size++
|
||||
}
|
||||
}
|
||||
|
||||
// Exists will return true iff the sequence is a member of this set.
|
||||
func (ss *SequenceSet) Exists(seq uint64) bool {
|
||||
for n := ss.root; n != nil; {
|
||||
if seq < n.base {
|
||||
n = n.l
|
||||
continue
|
||||
} else if seq >= n.base+numEntries {
|
||||
n = n.r
|
||||
continue
|
||||
}
|
||||
return n.exists(seq)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SetInitialMin should be used to set the initial minimum sequence when known.
|
||||
// This will more effectively utilize space versus self selecting.
|
||||
// The set should be empty.
|
||||
func (ss *SequenceSet) SetInitialMin(min uint64) error {
|
||||
if !ss.IsEmpty() {
|
||||
return ErrSetNotEmpty
|
||||
}
|
||||
ss.root, ss.nodes = &node{base: min, h: 1}, 1
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete will remove the sequence from the set.
|
||||
// Will optionally remove nodes and rebalance.
|
||||
// Returns where the sequence was set.
|
||||
func (ss *SequenceSet) Delete(seq uint64) bool {
|
||||
if ss == nil || ss.root == nil {
|
||||
return false
|
||||
}
|
||||
ss.root = ss.root.delete(seq, &ss.changed, &ss.nodes)
|
||||
if ss.changed {
|
||||
ss.changed = false
|
||||
ss.size--
|
||||
if ss.size == 0 {
|
||||
ss.Empty()
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Size returns the number of items in the set.
|
||||
func (ss *SequenceSet) Size() int {
|
||||
return ss.size
|
||||
}
|
||||
|
||||
// Nodes returns the number of nodes in the tree.
|
||||
func (ss *SequenceSet) Nodes() int {
|
||||
return ss.nodes
|
||||
}
|
||||
|
||||
// Empty will clear all items from a set.
|
||||
func (ss *SequenceSet) Empty() {
|
||||
ss.root = nil
|
||||
ss.size = 0
|
||||
ss.nodes = 0
|
||||
}
|
||||
|
||||
// IsEmpty is a fast check of the set being empty.
|
||||
func (ss *SequenceSet) IsEmpty() bool {
|
||||
if ss == nil || ss.root == nil {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Range will invoke the given function for each item in the set.
|
||||
// They will range over the set in ascending order.
|
||||
// If the callback returns false we terminate the iteration.
|
||||
func (ss *SequenceSet) Range(f func(uint64) bool) {
|
||||
ss.root.iter(f)
|
||||
}
|
||||
|
||||
// Heights returns the left and right heights of the tree.
|
||||
func (ss *SequenceSet) Heights() (l, r int) {
|
||||
if ss.root == nil {
|
||||
return 0, 0
|
||||
}
|
||||
if ss.root.l != nil {
|
||||
l = ss.root.l.h
|
||||
}
|
||||
if ss.root.r != nil {
|
||||
r = ss.root.r.h
|
||||
}
|
||||
return l, r
|
||||
}
|
||||
|
||||
// Returns min, max and number of set items.
|
||||
func (ss *SequenceSet) State() (min, max, num uint64) {
|
||||
if ss == nil || ss.root == nil {
|
||||
return 0, 0, 0
|
||||
}
|
||||
min, max = ss.MinMax()
|
||||
return min, max, uint64(ss.Size())
|
||||
}
|
||||
|
||||
// MinMax will return the minunum and maximum values in the set.
|
||||
func (ss *SequenceSet) MinMax() (min, max uint64) {
|
||||
if ss.root == nil {
|
||||
return 0, 0
|
||||
}
|
||||
for l := ss.root; l != nil; l = l.l {
|
||||
if l.l == nil {
|
||||
min = l.min()
|
||||
}
|
||||
}
|
||||
for r := ss.root; r != nil; r = r.r {
|
||||
if r.r == nil {
|
||||
max = r.max()
|
||||
}
|
||||
}
|
||||
return min, max
|
||||
}
|
||||
|
||||
func clone(src *node, target **node) {
|
||||
if src == nil {
|
||||
return
|
||||
}
|
||||
n := &node{base: src.base, bits: src.bits, h: src.h}
|
||||
*target = n
|
||||
clone(src.l, &n.l)
|
||||
clone(src.r, &n.r)
|
||||
}
|
||||
|
||||
// Clone will return a clone of the given SequenceSet.
|
||||
func (ss *SequenceSet) Clone() *SequenceSet {
|
||||
if ss == nil {
|
||||
return nil
|
||||
}
|
||||
css := &SequenceSet{nodes: ss.nodes, size: ss.size}
|
||||
clone(ss.root, &css.root)
|
||||
|
||||
return css
|
||||
}
|
||||
|
||||
// Equal returns whether the two sets contain exactly the same sequences.
|
||||
func (ss *SequenceSet) Equal(other *SequenceSet) bool {
|
||||
if ss.IsEmpty() || other.IsEmpty() {
|
||||
return ss.IsEmpty() && other.IsEmpty()
|
||||
}
|
||||
if ss.size != other.size {
|
||||
return false
|
||||
}
|
||||
// Sizes are equal, so a one-way membership check suffices.
|
||||
equal := true
|
||||
ss.Range(func(seq uint64) bool {
|
||||
equal = other.Exists(seq)
|
||||
return equal
|
||||
})
|
||||
return equal
|
||||
}
|
||||
|
||||
// Union will union this SequenceSet with ssa.
|
||||
func (ss *SequenceSet) Union(ssa ...*SequenceSet) {
|
||||
for _, sa := range ssa {
|
||||
sa.root.nodeIter(func(n *node) {
|
||||
for nb, b := range n.bits {
|
||||
base := n.base + uint64(nb)*bitsPerBucket
|
||||
for b != 0 {
|
||||
ss.Insert(base + uint64(bits.TrailingZeros64(b)))
|
||||
b &= b - 1
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Union will return a union of all sets.
|
||||
func Union(ssa ...*SequenceSet) *SequenceSet {
|
||||
if len(ssa) == 0 {
|
||||
return nil
|
||||
}
|
||||
// Sort so we can clone largest.
|
||||
slices.SortFunc(ssa, func(i, j *SequenceSet) int { return -cmp.Compare(i.Size(), j.Size()) }) // reverse order
|
||||
ss := ssa[0].Clone()
|
||||
|
||||
// Insert the rest through range call.
|
||||
for i := 1; i < len(ssa); i++ {
|
||||
ssa[i].Range(func(n uint64) bool {
|
||||
ss.Insert(n)
|
||||
return true
|
||||
})
|
||||
}
|
||||
return ss
|
||||
}
|
||||
|
||||
const (
|
||||
// Magic is used to identify the encode binary state..
|
||||
magic = uint8(22)
|
||||
// Version
|
||||
version = uint8(2)
|
||||
// hdrLen
|
||||
hdrLen = 2
|
||||
// minimum length of an encoded SequenceSet.
|
||||
minLen = 2 + 8 // magic + version + num nodes + num entries.
|
||||
)
|
||||
|
||||
// EncodeLen returns the bytes needed for encoding.
|
||||
func (ss SequenceSet) EncodeLen() int {
|
||||
return minLen + (ss.Nodes() * ((numBuckets+1)*8 + 2))
|
||||
}
|
||||
|
||||
func (ss SequenceSet) Encode(buf []byte) []byte {
|
||||
nn, encLen := ss.Nodes(), ss.EncodeLen()
|
||||
|
||||
if cap(buf) < encLen {
|
||||
buf = make([]byte, encLen)
|
||||
} else {
|
||||
buf = buf[:encLen]
|
||||
}
|
||||
|
||||
// TODO(dlc) - Go 1.19 introduced Append to not have to keep track.
|
||||
// Once 1.20 is out we could change this over.
|
||||
// Also binary.Write() is way slower, do not use.
|
||||
|
||||
var le = binary.LittleEndian
|
||||
buf[0], buf[1] = magic, version
|
||||
i := hdrLen
|
||||
le.PutUint32(buf[i:], uint32(nn))
|
||||
le.PutUint32(buf[i+4:], uint32(ss.size))
|
||||
i += 8
|
||||
ss.root.nodeIter(func(n *node) {
|
||||
le.PutUint64(buf[i:], n.base)
|
||||
i += 8
|
||||
for _, b := range n.bits {
|
||||
le.PutUint64(buf[i:], b)
|
||||
i += 8
|
||||
}
|
||||
le.PutUint16(buf[i:], uint16(n.h))
|
||||
i += 2
|
||||
})
|
||||
return buf[:i]
|
||||
}
|
||||
|
||||
// ErrBadEncoding is returned when we can not decode properly.
|
||||
var (
|
||||
ErrBadEncoding = errors.New("ss: bad encoding")
|
||||
ErrBadVersion = errors.New("ss: bad version")
|
||||
ErrSetNotEmpty = errors.New("ss: set not empty")
|
||||
)
|
||||
|
||||
// Decode returns the sequence set and number of bytes read from the buffer on success.
|
||||
func Decode(buf []byte) (*SequenceSet, int, error) {
|
||||
if len(buf) < minLen || buf[0] != magic {
|
||||
return nil, -1, ErrBadEncoding
|
||||
}
|
||||
|
||||
switch v := buf[1]; v {
|
||||
case 1:
|
||||
return decodev1(buf)
|
||||
case 2:
|
||||
return decodev2(buf)
|
||||
default:
|
||||
return nil, -1, ErrBadVersion
|
||||
}
|
||||
}
|
||||
|
||||
// Helper to decode v2.
|
||||
func decodev2(buf []byte) (*SequenceSet, int, error) {
|
||||
var le = binary.LittleEndian
|
||||
index := 2
|
||||
nn := int(le.Uint32(buf[index:]))
|
||||
sz := int(le.Uint32(buf[index+4:]))
|
||||
index += 8
|
||||
|
||||
// nn is decoded as a uint32 but held in an int. On 32-bit builds a value
|
||||
// above MaxInt32 turns negative and nn*perNode below overflows, so the
|
||||
// length check would pass for a short buffer and the following make/reads
|
||||
// run off the end. Compare with division so the bound holds on every arch.
|
||||
if nn < 0 || nn > (len(buf)-minLen)/((numBuckets+1)*8+2) {
|
||||
return nil, -1, ErrBadEncoding
|
||||
}
|
||||
|
||||
ss, nodes := SequenceSet{size: sz}, make([]node, nn)
|
||||
|
||||
for i := 0; i < nn; i++ {
|
||||
n := &nodes[i]
|
||||
n.base = le.Uint64(buf[index:])
|
||||
index += 8
|
||||
for bi := range n.bits {
|
||||
n.bits[bi] = le.Uint64(buf[index:])
|
||||
index += 8
|
||||
}
|
||||
n.h = int(le.Uint16(buf[index:]))
|
||||
index += 2
|
||||
ss.insertNode(n)
|
||||
}
|
||||
|
||||
return &ss, index, nil
|
||||
}
|
||||
|
||||
// Helper to decode v1 into v2 which has fixed buckets of 32 vs 64 originally.
|
||||
func decodev1(buf []byte) (*SequenceSet, int, error) {
|
||||
var le = binary.LittleEndian
|
||||
index := 2
|
||||
nn := int(le.Uint32(buf[index:]))
|
||||
sz := int(le.Uint32(buf[index+4:]))
|
||||
index += 8
|
||||
|
||||
const v1NumBuckets = 64
|
||||
|
||||
// See decodev2: guard the node count without overflowing the multiply so
|
||||
// the bound stays correct on 32-bit builds too.
|
||||
if nn < 0 || nn > (len(buf)-minLen)/((v1NumBuckets+1)*8+2) {
|
||||
return nil, -1, ErrBadEncoding
|
||||
}
|
||||
|
||||
var ss SequenceSet
|
||||
for i := 0; i < nn; i++ {
|
||||
base := le.Uint64(buf[index:])
|
||||
index += 8
|
||||
for nb := uint64(0); nb < v1NumBuckets; nb++ {
|
||||
n := le.Uint64(buf[index:])
|
||||
// Walk all set bits and insert sequences manually for this decode from v1.
|
||||
for n != 0 {
|
||||
ss.Insert(base + (nb * uint64(bitsPerBucket)) + uint64(bits.TrailingZeros64(n)))
|
||||
n &= n - 1
|
||||
}
|
||||
index += 8
|
||||
}
|
||||
// Skip over encoded height.
|
||||
index += 2
|
||||
}
|
||||
|
||||
// Sanity check.
|
||||
if ss.Size() != sz {
|
||||
return nil, -1, ErrBadEncoding
|
||||
}
|
||||
|
||||
return &ss, index, nil
|
||||
|
||||
}
|
||||
|
||||
// insertNode places a decoded node into the tree.
|
||||
// These should be done in tree order as defined by Encode()
|
||||
// This allows us to not have to calculate height or do rebalancing.
|
||||
// So much better performance this way.
|
||||
func (ss *SequenceSet) insertNode(n *node) {
|
||||
ss.nodes++
|
||||
|
||||
if ss.root == nil {
|
||||
ss.root = n
|
||||
return
|
||||
}
|
||||
// Walk our way to the insertion point.
|
||||
for p := ss.root; p != nil; {
|
||||
if n.base < p.base {
|
||||
if p.l == nil {
|
||||
p.l = n
|
||||
return
|
||||
}
|
||||
p = p.l
|
||||
} else {
|
||||
if p.r == nil {
|
||||
p.r = n
|
||||
return
|
||||
}
|
||||
p = p.r
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
bitsPerBucket = 64 // bits in uint64
|
||||
numBuckets = 32
|
||||
numEntries = numBuckets * bitsPerBucket
|
||||
)
|
||||
|
||||
type node struct {
|
||||
//v dvalue
|
||||
base uint64
|
||||
bits [numBuckets]uint64
|
||||
l *node
|
||||
r *node
|
||||
h int
|
||||
}
|
||||
|
||||
// Set the proper bit.
|
||||
// seq should have already been qualified and inserted should be non nil.
|
||||
func (n *node) set(seq uint64, inserted *bool) {
|
||||
seq -= n.base
|
||||
i := seq / bitsPerBucket
|
||||
mask := uint64(1) << (seq % bitsPerBucket)
|
||||
if (n.bits[i] & mask) == 0 {
|
||||
n.bits[i] |= mask
|
||||
*inserted = true
|
||||
}
|
||||
}
|
||||
|
||||
func (n *node) insert(seq uint64, inserted *bool, nodes *int) *node {
|
||||
if n == nil {
|
||||
base := (seq / numEntries) * numEntries
|
||||
n := &node{base: base, h: 1}
|
||||
n.set(seq, inserted)
|
||||
*nodes++
|
||||
return n
|
||||
}
|
||||
|
||||
if seq < n.base {
|
||||
n.l = n.l.insert(seq, inserted, nodes)
|
||||
} else if seq >= n.base+numEntries {
|
||||
n.r = n.r.insert(seq, inserted, nodes)
|
||||
} else {
|
||||
n.set(seq, inserted)
|
||||
}
|
||||
|
||||
n.h = maxH(n) + 1
|
||||
|
||||
// Don't make a function, impacts performance.
|
||||
if bf := balanceF(n); bf > 1 {
|
||||
// Left unbalanced.
|
||||
if balanceF(n.l) < 0 {
|
||||
n.l = n.l.rotateL()
|
||||
}
|
||||
return n.rotateR()
|
||||
} else if bf < -1 {
|
||||
// Right unbalanced.
|
||||
if balanceF(n.r) > 0 {
|
||||
n.r = n.r.rotateR()
|
||||
}
|
||||
return n.rotateL()
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (n *node) rotateL() *node {
|
||||
r := n.r
|
||||
if r != nil {
|
||||
n.r = r.l
|
||||
r.l = n
|
||||
n.h = maxH(n) + 1
|
||||
r.h = maxH(r) + 1
|
||||
} else {
|
||||
n.r = nil
|
||||
n.h = maxH(n) + 1
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func (n *node) rotateR() *node {
|
||||
l := n.l
|
||||
if l != nil {
|
||||
n.l = l.r
|
||||
l.r = n
|
||||
n.h = maxH(n) + 1
|
||||
l.h = maxH(l) + 1
|
||||
} else {
|
||||
n.l = nil
|
||||
n.h = maxH(n) + 1
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
func balanceF(n *node) int {
|
||||
if n == nil {
|
||||
return 0
|
||||
}
|
||||
var lh, rh int
|
||||
if n.l != nil {
|
||||
lh = n.l.h
|
||||
}
|
||||
if n.r != nil {
|
||||
rh = n.r.h
|
||||
}
|
||||
return lh - rh
|
||||
}
|
||||
|
||||
func maxH(n *node) int {
|
||||
if n == nil {
|
||||
return 0
|
||||
}
|
||||
var lh, rh int
|
||||
if n.l != nil {
|
||||
lh = n.l.h
|
||||
}
|
||||
if n.r != nil {
|
||||
rh = n.r.h
|
||||
}
|
||||
if lh > rh {
|
||||
return lh
|
||||
}
|
||||
return rh
|
||||
}
|
||||
|
||||
// Clear the proper bit.
|
||||
// seq should have already been qualified and deleted should be non nil.
|
||||
// Will return true if this node is now empty.
|
||||
func (n *node) clear(seq uint64, deleted *bool) bool {
|
||||
seq -= n.base
|
||||
i := seq / bitsPerBucket
|
||||
mask := uint64(1) << (seq % bitsPerBucket)
|
||||
if (n.bits[i] & mask) == 0 {
|
||||
// Nothing cleared, and nodes in the tree are never empty,
|
||||
// so no need to scan the buckets.
|
||||
return false
|
||||
}
|
||||
n.bits[i] &^= mask
|
||||
*deleted = true
|
||||
for _, b := range n.bits {
|
||||
if b != 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (n *node) delete(seq uint64, deleted *bool, nodes *int) *node {
|
||||
if n == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if seq < n.base {
|
||||
n.l = n.l.delete(seq, deleted, nodes)
|
||||
} else if seq >= n.base+numEntries {
|
||||
n.r = n.r.delete(seq, deleted, nodes)
|
||||
} else if empty := n.clear(seq, deleted); empty {
|
||||
*nodes--
|
||||
if n.l == nil {
|
||||
n = n.r
|
||||
} else if n.r == nil {
|
||||
n = n.l
|
||||
} else {
|
||||
// We have both children.
|
||||
n.r = n.r.insertNodePrev(n.l)
|
||||
n = n.r
|
||||
}
|
||||
}
|
||||
|
||||
if n != nil {
|
||||
n.h = maxH(n) + 1
|
||||
}
|
||||
|
||||
// Check balance.
|
||||
if bf := balanceF(n); bf > 1 {
|
||||
// Left unbalanced.
|
||||
if balanceF(n.l) < 0 {
|
||||
n.l = n.l.rotateL()
|
||||
}
|
||||
return n.rotateR()
|
||||
} else if bf < -1 {
|
||||
// right unbalanced.
|
||||
if balanceF(n.r) > 0 {
|
||||
n.r = n.r.rotateR()
|
||||
}
|
||||
return n.rotateL()
|
||||
}
|
||||
|
||||
return n
|
||||
}
|
||||
|
||||
// Will insert nn into the node assuming it is less than all other nodes in n.
|
||||
// Will re-calculate height and balance.
|
||||
func (n *node) insertNodePrev(nn *node) *node {
|
||||
if n.l == nil {
|
||||
n.l = nn
|
||||
} else {
|
||||
n.l = n.l.insertNodePrev(nn)
|
||||
}
|
||||
n.h = maxH(n) + 1
|
||||
|
||||
// Check balance.
|
||||
if bf := balanceF(n); bf > 1 {
|
||||
// Left unbalanced.
|
||||
if balanceF(n.l) < 0 {
|
||||
n.l = n.l.rotateL()
|
||||
}
|
||||
return n.rotateR()
|
||||
} else if bf < -1 {
|
||||
// right unbalanced.
|
||||
if balanceF(n.r) > 0 {
|
||||
n.r = n.r.rotateR()
|
||||
}
|
||||
return n.rotateL()
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (n *node) exists(seq uint64) bool {
|
||||
seq -= n.base
|
||||
i := seq / bitsPerBucket
|
||||
mask := uint64(1) << (seq % bitsPerBucket)
|
||||
return n.bits[i]&mask != 0
|
||||
}
|
||||
|
||||
// Return minimum sequence in the set.
|
||||
// This node can not be empty.
|
||||
func (n *node) min() uint64 {
|
||||
for i, b := range n.bits {
|
||||
if b != 0 {
|
||||
return n.base +
|
||||
uint64(i*bitsPerBucket) +
|
||||
uint64(bits.TrailingZeros64(b))
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Return maximum sequence in the set.
|
||||
// This node can not be empty.
|
||||
func (n *node) max() uint64 {
|
||||
for i := numBuckets - 1; i >= 0; i-- {
|
||||
if b := n.bits[i]; b != 0 {
|
||||
return n.base +
|
||||
uint64(i*bitsPerBucket) +
|
||||
uint64(bitsPerBucket-bits.LeadingZeros64(b>>1))
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// This is done in tree order.
|
||||
func (n *node) nodeIter(f func(n *node)) {
|
||||
if n == nil {
|
||||
return
|
||||
}
|
||||
f(n)
|
||||
n.l.nodeIter(f)
|
||||
n.r.nodeIter(f)
|
||||
}
|
||||
|
||||
// iter will iterate through the set's items in this node.
|
||||
// If the supplied function returns false we terminate the iteration.
|
||||
func (n *node) iter(f func(uint64) bool) bool {
|
||||
if n == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
if ok := n.l.iter(f); !ok {
|
||||
return false
|
||||
}
|
||||
for i, b := range n.bits {
|
||||
base := n.base + uint64(i)*bitsPerBucket
|
||||
for b != 0 {
|
||||
if ok := f(base + uint64(bits.TrailingZeros64(b))); !ok {
|
||||
return false
|
||||
}
|
||||
b &= b - 1
|
||||
}
|
||||
}
|
||||
if ok := n.r.iter(f); !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
556
server/avl/seqset_test.go
Normal file
556
server/avl/seqset_test.go
Normal file
@@ -0,0 +1,556 @@
|
||||
// Copyright 2023-2026 The NATS Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package avl
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"math/rand/v2"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSeqSetBasics(t *testing.T) {
|
||||
var ss SequenceSet
|
||||
|
||||
seqs := []uint64{22, 222, 2000, 2, 2, 4}
|
||||
for _, seq := range seqs {
|
||||
ss.Insert(seq)
|
||||
require_True(t, ss.Exists(seq))
|
||||
}
|
||||
|
||||
require_True(t, ss.Nodes() == 1)
|
||||
require_True(t, ss.Size() == len(seqs)-1) // We have one dup in there.
|
||||
lh, rh := ss.Heights()
|
||||
require_True(t, lh == 0)
|
||||
require_True(t, rh == 0)
|
||||
}
|
||||
|
||||
func TestSeqSetLeftLean(t *testing.T) {
|
||||
var ss SequenceSet
|
||||
|
||||
for i := uint64(4 * numEntries); i > 0; i-- {
|
||||
ss.Insert(i)
|
||||
}
|
||||
|
||||
require_True(t, ss.Nodes() == 5)
|
||||
require_True(t, ss.Size() == 4*numEntries)
|
||||
lh, rh := ss.Heights()
|
||||
require_True(t, lh == 2)
|
||||
require_True(t, rh == 1)
|
||||
}
|
||||
|
||||
func TestSeqSetRightLean(t *testing.T) {
|
||||
var ss SequenceSet
|
||||
|
||||
for i := uint64(0); i < uint64(4*numEntries); i++ {
|
||||
ss.Insert(i)
|
||||
}
|
||||
|
||||
require_True(t, ss.Nodes() == 4)
|
||||
require_True(t, ss.Size() == 4*numEntries)
|
||||
lh, rh := ss.Heights()
|
||||
require_True(t, lh == 1)
|
||||
require_True(t, rh == 2)
|
||||
}
|
||||
|
||||
func TestSeqSetCorrectness(t *testing.T) {
|
||||
// Generate 100k sequences across 500k range.
|
||||
num := 100_000
|
||||
max := 500_000
|
||||
|
||||
set := make(map[uint64]struct{}, num)
|
||||
var ss SequenceSet
|
||||
for i := 0; i < num; i++ {
|
||||
n := uint64(rand.Int64N(int64(max + 1)))
|
||||
ss.Insert(n)
|
||||
set[n] = struct{}{}
|
||||
}
|
||||
|
||||
for i := uint64(0); i <= uint64(max); i++ {
|
||||
_, exists := set[i]
|
||||
require_True(t, ss.Exists(i) == exists)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeqSetRange(t *testing.T) {
|
||||
num := 2*numEntries + 22
|
||||
nums := make([]uint64, 0, num)
|
||||
for i := 0; i < num; i++ {
|
||||
nums = append(nums, uint64(i))
|
||||
}
|
||||
rand.Shuffle(len(nums), func(i, j int) { nums[i], nums[j] = nums[j], nums[i] })
|
||||
|
||||
var ss SequenceSet
|
||||
for _, n := range nums {
|
||||
ss.Insert(n)
|
||||
}
|
||||
|
||||
nums = nums[:0]
|
||||
ss.Range(func(n uint64) bool {
|
||||
nums = append(nums, n)
|
||||
return true
|
||||
})
|
||||
require_True(t, len(nums) == num)
|
||||
for i := uint64(0); i < uint64(num); i++ {
|
||||
require_True(t, nums[i] == i)
|
||||
}
|
||||
|
||||
// Test truncating the range call.
|
||||
nums = nums[:0]
|
||||
ss.Range(func(n uint64) bool {
|
||||
if n >= 10 {
|
||||
return false
|
||||
}
|
||||
nums = append(nums, n)
|
||||
return true
|
||||
})
|
||||
require_True(t, len(nums) == 10)
|
||||
for i := uint64(0); i < 10; i++ {
|
||||
require_True(t, nums[i] == i)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeqSetDelete(t *testing.T) {
|
||||
var ss SequenceSet
|
||||
|
||||
// Simple single node.
|
||||
seqs := []uint64{22, 222, 2222, 2, 2, 4}
|
||||
for _, seq := range seqs {
|
||||
ss.Insert(seq)
|
||||
}
|
||||
|
||||
for _, seq := range seqs {
|
||||
ss.Delete(seq)
|
||||
require_True(t, !ss.Exists(seq))
|
||||
}
|
||||
require_True(t, ss.root == nil)
|
||||
}
|
||||
|
||||
func TestSeqSetInsertAndDeletePedantic(t *testing.T) {
|
||||
var ss SequenceSet
|
||||
|
||||
num := 50*numEntries + 22
|
||||
nums := make([]uint64, 0, num)
|
||||
for i := 0; i < num; i++ {
|
||||
nums = append(nums, uint64(i))
|
||||
}
|
||||
rand.Shuffle(len(nums), func(i, j int) { nums[i], nums[j] = nums[j], nums[i] })
|
||||
|
||||
// Make sure always balanced.
|
||||
testBalanced := func() {
|
||||
t.Helper()
|
||||
// Check heights.
|
||||
ss.root.nodeIter(func(n *node) {
|
||||
if n != nil && n.h != maxH(n)+1 {
|
||||
t.Fatalf("Node height is wrong: %+v", n)
|
||||
}
|
||||
})
|
||||
// Check balance factor.
|
||||
if bf := balanceF(ss.root); bf > 1 || bf < -1 {
|
||||
t.Fatalf("Unbalanced tree")
|
||||
}
|
||||
}
|
||||
|
||||
for _, n := range nums {
|
||||
ss.Insert(n)
|
||||
testBalanced()
|
||||
}
|
||||
require_True(t, ss.root != nil)
|
||||
|
||||
for _, n := range nums {
|
||||
ss.Delete(n)
|
||||
testBalanced()
|
||||
require_True(t, !ss.Exists(n))
|
||||
if ss.Size() > 0 {
|
||||
require_True(t, ss.root != nil)
|
||||
}
|
||||
}
|
||||
require_True(t, ss.root == nil)
|
||||
}
|
||||
|
||||
func TestSeqSetMinMax(t *testing.T) {
|
||||
var ss SequenceSet
|
||||
|
||||
// Simple single node.
|
||||
seqs := []uint64{22, 222, 2222, 2, 2, 4}
|
||||
for _, seq := range seqs {
|
||||
ss.Insert(seq)
|
||||
}
|
||||
|
||||
min, max := ss.MinMax()
|
||||
require_True(t, min == 2 && max == 2222)
|
||||
|
||||
// Multi-node
|
||||
ss.Empty()
|
||||
|
||||
num := 22*numEntries + 22
|
||||
nums := make([]uint64, 0, num)
|
||||
for i := 0; i < num; i++ {
|
||||
nums = append(nums, uint64(i))
|
||||
}
|
||||
rand.Shuffle(len(nums), func(i, j int) { nums[i], nums[j] = nums[j], nums[i] })
|
||||
for _, n := range nums {
|
||||
ss.Insert(n)
|
||||
}
|
||||
|
||||
min, max = ss.MinMax()
|
||||
require_True(t, min == 0 && max == uint64(num-1))
|
||||
}
|
||||
|
||||
func TestSeqSetClone(t *testing.T) {
|
||||
// Generate 100k sequences across 500k range.
|
||||
num := 100_000
|
||||
max := 500_000
|
||||
|
||||
var ss SequenceSet
|
||||
for i := 0; i < num; i++ {
|
||||
ss.Insert(uint64(rand.Int64N(int64(max + 1))))
|
||||
}
|
||||
|
||||
ssc := ss.Clone()
|
||||
require_True(t, ss.Size() == ssc.Size())
|
||||
require_True(t, ss.Nodes() == ssc.Nodes())
|
||||
}
|
||||
|
||||
func TestSeqSetEqual(t *testing.T) {
|
||||
var ss1, ss2, ss3 SequenceSet
|
||||
|
||||
// Empty and nil sets are all equal.
|
||||
var nilSet *SequenceSet
|
||||
require_True(t, ss1.Equal(&ss2))
|
||||
require_True(t, ss1.Equal(nilSet))
|
||||
require_True(t, nilSet.Equal(&ss1))
|
||||
|
||||
// Same state (first, last, size) but different contents.
|
||||
for _, seq := range []uint64{1, 3, 5} {
|
||||
ss1.Insert(seq)
|
||||
}
|
||||
for _, seq := range []uint64{1, 4, 5} {
|
||||
ss2.Insert(seq)
|
||||
}
|
||||
require_False(t, ss1.Equal(&ss2))
|
||||
require_False(t, ss2.Equal(&ss1))
|
||||
require_False(t, ss1.Equal(nilSet))
|
||||
require_False(t, nilSet.Equal(&ss1))
|
||||
|
||||
// Identical contents, insertion order should not matter.
|
||||
for _, seq := range []uint64{5, 1, 3} {
|
||||
ss3.Insert(seq)
|
||||
}
|
||||
require_True(t, ss1.Equal(&ss3))
|
||||
require_True(t, ss3.Equal(&ss1))
|
||||
|
||||
// Differing sizes.
|
||||
ss3.Insert(7)
|
||||
require_False(t, ss1.Equal(&ss3))
|
||||
}
|
||||
|
||||
func TestSeqSetUnion(t *testing.T) {
|
||||
var ss1, ss2 SequenceSet
|
||||
|
||||
seqs1 := []uint64{22, 222, 2222, 2, 2, 4}
|
||||
for _, seq := range seqs1 {
|
||||
ss1.Insert(seq)
|
||||
}
|
||||
|
||||
seqs2 := []uint64{33, 333, 3333, 3, 33_333, 333_333}
|
||||
for _, seq := range seqs2 {
|
||||
ss2.Insert(seq)
|
||||
}
|
||||
|
||||
ss := Union(&ss1, &ss2)
|
||||
require_True(t, ss.Size() == 11)
|
||||
|
||||
seqs := append(seqs1, seqs2...)
|
||||
for _, n := range seqs {
|
||||
require_True(t, ss.Exists(n))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeqSetFirst(t *testing.T) {
|
||||
var ss SequenceSet
|
||||
|
||||
seqs := []uint64{22, 222, 2222, 222_222}
|
||||
for _, seq := range seqs {
|
||||
// Normal case where we pick first/base.
|
||||
ss.Insert(seq)
|
||||
require_True(t, ss.root.base == (seq/numEntries)*numEntries)
|
||||
ss.Empty()
|
||||
// Where we set the minimum start value.
|
||||
ss.SetInitialMin(seq)
|
||||
ss.Insert(seq)
|
||||
require_True(t, ss.root.base == seq)
|
||||
ss.Empty()
|
||||
}
|
||||
}
|
||||
|
||||
// Test that we can union with nodes vs individual sequence insertion.
|
||||
func TestSeqSetDistinctUnion(t *testing.T) {
|
||||
// Distinct sets.
|
||||
var ss1 SequenceSet
|
||||
seqs1 := []uint64{1, 10, 100, 200}
|
||||
for _, seq := range seqs1 {
|
||||
ss1.Insert(seq)
|
||||
}
|
||||
|
||||
var ss2 SequenceSet
|
||||
seqs2 := []uint64{5000, 6100, 6200, 6222}
|
||||
for _, seq := range seqs2 {
|
||||
ss2.Insert(seq)
|
||||
}
|
||||
|
||||
ss := ss1.Clone()
|
||||
allSeqs := append(seqs1, seqs2...)
|
||||
|
||||
ss.Union(&ss2)
|
||||
require_True(t, ss.Size() == len(allSeqs))
|
||||
for _, seq := range allSeqs {
|
||||
require_True(t, ss.Exists(seq))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeqSetDecodeV1(t *testing.T) {
|
||||
// Encoding from v1 which was 64 buckets.
|
||||
seqs := []uint64{22, 222, 2222, 222_222, 2_222_222}
|
||||
encStr := `
|
||||
FgEDAAAABQAAAABgAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAADgIQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAA==
|
||||
`
|
||||
|
||||
enc, err := base64.StdEncoding.DecodeString(encStr)
|
||||
require_NoError(t, err)
|
||||
|
||||
ss, _, err := Decode(enc)
|
||||
require_NoError(t, err)
|
||||
|
||||
require_True(t, ss.Size() == len(seqs))
|
||||
for _, seq := range seqs {
|
||||
require_True(t, ss.Exists(seq))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeqSetDecodeNodeCountOverflow(t *testing.T) {
|
||||
// A replicated stream state carries an avl-encoded delete map decoded from
|
||||
// a peer. The node count is read as a uint32 into an int; on 32-bit builds
|
||||
// a value above MaxInt32 turns negative (and a smaller value can overflow
|
||||
// the nn*perNode multiply), defeating the length check so the decode then
|
||||
// panics in make([]node, nn) or reads past the buffer. Both encodings must
|
||||
// reject such counts on every architecture instead.
|
||||
le := binary.LittleEndian
|
||||
for _, ver := range []uint8{1, 2} {
|
||||
buf := make([]byte, minLen)
|
||||
buf[0], buf[1] = magic, ver
|
||||
le.PutUint32(buf[2:], 0xFFFFFFFF) // node count, negative as a 32-bit int
|
||||
if _, _, err := Decode(buf); err != ErrBadEncoding {
|
||||
t.Fatalf("v%d: expected ErrBadEncoding for oversized node count, got %v", ver, err)
|
||||
}
|
||||
|
||||
// Positive on 32-bit but the multiply still overflows for a short buffer.
|
||||
le.PutUint32(buf[2:], 0x00800000)
|
||||
if _, _, err := Decode(buf); err != ErrBadEncoding {
|
||||
t.Fatalf("v%d: expected ErrBadEncoding for overflowing node count, got %v", ver, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A well-formed single-node v2 buffer must still decode.
|
||||
valid := make([]byte, minLen+(numBuckets+1)*8+2)
|
||||
valid[0], valid[1] = magic, version
|
||||
le.PutUint32(valid[2:], 1)
|
||||
if _, _, err := Decode(valid); err != nil {
|
||||
t.Fatalf("valid single-node buffer should decode, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeqSetInsertExistingNodeFastPath(t *testing.T) {
|
||||
var ss SequenceSet
|
||||
|
||||
// Build a multi-node tree with only even sequences set.
|
||||
for i := uint64(0); i < 8*numEntries; i += 2 {
|
||||
ss.Insert(i)
|
||||
}
|
||||
nodes, size := ss.Nodes(), ss.Size()
|
||||
|
||||
// Inserting into existing nodes must not change the tree structure.
|
||||
for i := uint64(1); i < 8*numEntries; i += 2 {
|
||||
ss.Insert(i)
|
||||
require_True(t, ss.Exists(i))
|
||||
}
|
||||
require_True(t, ss.Nodes() == nodes)
|
||||
require_True(t, ss.Size() == size+4*numEntries)
|
||||
|
||||
// Duplicate inserts must not change the size.
|
||||
for i := uint64(0); i < 8*numEntries; i++ {
|
||||
ss.Insert(i)
|
||||
}
|
||||
require_True(t, ss.Size() == 8*numEntries)
|
||||
|
||||
// Heights and balance must still be intact.
|
||||
ss.root.nodeIter(func(n *node) {
|
||||
if n.h != maxH(n)+1 {
|
||||
t.Fatalf("Node height is wrong: %+v", n)
|
||||
}
|
||||
})
|
||||
if bf := balanceF(ss.root); bf > 1 || bf < -1 {
|
||||
t.Fatalf("Unbalanced tree")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeqSetInsertUnalignedBase(t *testing.T) {
|
||||
var ss SequenceSet
|
||||
|
||||
// SetInitialMin can create a node whose base is not aligned to numEntries.
|
||||
require_NoError(t, ss.SetInitialMin(1000))
|
||||
for _, seq := range []uint64{1000, 1001, 1000 + numEntries - 1} {
|
||||
ss.Insert(seq)
|
||||
require_True(t, ss.Exists(seq))
|
||||
}
|
||||
require_True(t, ss.Nodes() == 1)
|
||||
require_True(t, ss.Size() == 3)
|
||||
|
||||
// Outside the node's range still creates a new node.
|
||||
ss.Insert(1000 + numEntries)
|
||||
require_True(t, ss.Exists(1000+numEntries))
|
||||
require_True(t, ss.Nodes() == 2)
|
||||
require_True(t, ss.Size() == 4)
|
||||
}
|
||||
|
||||
func TestSeqSetDeleteNotPresent(t *testing.T) {
|
||||
var ss SequenceSet
|
||||
|
||||
seqs := []uint64{22, 222, 2222, 222_222}
|
||||
for _, seq := range seqs {
|
||||
ss.Insert(seq)
|
||||
}
|
||||
nodes, size := ss.Nodes(), ss.Size()
|
||||
|
||||
// Deleting sequences that are not set, both inside and outside
|
||||
// existing node ranges, must be a no-op and report false.
|
||||
for _, seq := range []uint64{0, 23, 2223, 5000, 100_000, 999_999} {
|
||||
require_True(t, !ss.Delete(seq))
|
||||
}
|
||||
require_True(t, ss.Nodes() == nodes)
|
||||
require_True(t, ss.Size() == size)
|
||||
for _, seq := range seqs {
|
||||
require_True(t, ss.Exists(seq))
|
||||
}
|
||||
|
||||
// Real deletes still work and empty the set.
|
||||
for _, seq := range seqs {
|
||||
require_True(t, ss.Delete(seq))
|
||||
}
|
||||
require_True(t, ss.root == nil)
|
||||
}
|
||||
|
||||
func TestSeqSetRangeSparse(t *testing.T) {
|
||||
// Cover bucket boundaries, node boundaries and large gaps.
|
||||
seqs := []uint64{0, 1, 63, 64, 127, 128, 2047, 2048, 4095, 4096, 100_000, 1_000_000}
|
||||
var ss SequenceSet
|
||||
for _, seq := range seqs {
|
||||
ss.Insert(seq)
|
||||
}
|
||||
|
||||
var got []uint64
|
||||
ss.Range(func(n uint64) bool {
|
||||
got = append(got, n)
|
||||
return true
|
||||
})
|
||||
require_True(t, len(got) == len(seqs))
|
||||
for i, seq := range seqs {
|
||||
require_True(t, got[i] == seq)
|
||||
}
|
||||
|
||||
// Terminating mid-bucket should stop the iteration immediately.
|
||||
got = got[:0]
|
||||
ss.Range(func(n uint64) bool {
|
||||
got = append(got, n)
|
||||
return n != 64
|
||||
})
|
||||
require_True(t, len(got) == 4)
|
||||
require_True(t, got[3] == 64)
|
||||
}
|
||||
|
||||
func BenchmarkSeqSetInsertDense(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
var ss SequenceSet
|
||||
for s := uint64(0); s < 200_000; s++ {
|
||||
ss.Insert(s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSeqSetInsertSparse(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
var ss SequenceSet
|
||||
for s := uint64(0); s < 200_000; s++ {
|
||||
ss.Insert(s * 7)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSeqSetRangeDense(b *testing.B) {
|
||||
var ss SequenceSet
|
||||
for s := uint64(0); s < 200_000; s++ {
|
||||
ss.Insert(s)
|
||||
}
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
var sum uint64
|
||||
ss.Range(func(n uint64) bool { sum += n; return true })
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSeqSetRangeSparse(b *testing.B) {
|
||||
var ss SequenceSet
|
||||
for s := uint64(0); s < 10_000; s++ {
|
||||
ss.Insert(s * 211) // ~10 bits per 2048-entry node
|
||||
}
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
var sum uint64
|
||||
ss.Range(func(n uint64) bool { sum += n; return true })
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSeqSetDeleteNotPresent(b *testing.B) {
|
||||
var ss SequenceSet
|
||||
for s := uint64(0); s < 200_000; s += 2 {
|
||||
ss.Insert(s)
|
||||
}
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
for s := uint64(1); s < 200_000; s += 2 {
|
||||
ss.Delete(s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func require_NoError(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatalf("require no error, but got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func require_True(t *testing.T, b bool) {
|
||||
t.Helper()
|
||||
if !b {
|
||||
t.Fatalf("require true")
|
||||
}
|
||||
}
|
||||
|
||||
func require_False(t *testing.T, b bool) {
|
||||
t.Helper()
|
||||
if b {
|
||||
t.Fatalf("require false")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user