commit f8d3d9f0cf4f7b3ec8d2c7d6592360a8090eb990 Author: upstream-mirror Date: Tue Sep 15 22:27:16 2026 +0500 snapshot of https://github.com/blevesearch/bleve.git (master, 2026-09-15; blobs > 131072B omitted) diff --git a/.github/workflows/cover.yml b/.github/workflows/cover.yml new file mode 100644 index 0000000..9d58cf9 --- /dev/null +++ b/.github/workflows/cover.yml @@ -0,0 +1,30 @@ +on: + push: + branches: + - master + pull_request: +name: Coverage +jobs: + coverage: + runs-on: ubuntu-latest + steps: + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: '1.25.x' + - name: Checkout code + uses: actions/checkout@v2 + - name: Clean environment + run: | + go clean -cache -testcache -modcache + rm -f profile.cov + - name: Test + run: | + go test -coverprofile=profile.cov ./... + - name: Remove non-GO entries from coverage profile + run: | + grep -E 'mode|\.go' profile.cov > profile_go.cov + - name: Send coverage + uses: shogo82148/actions-goveralls@v1 + with: + path-to-profile: profile_go.cov diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..beb7b87 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,24 @@ +on: + push: + branches: + - master + pull_request: +name: Tests +jobs: + test: + strategy: + matrix: + go-version: [1.24.x, 1.25.x, 1.26.x] + platform: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.platform }} + steps: + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + - name: Checkout code + uses: actions/checkout@v2 + - name: Test + run: | + go version + go test -race ./... diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7512de7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +#* +*.sublime-* +*~ +.#* +.project +.settings +**/.idea/ +**/*.iml +.DS_Store +query_string.y.go.tmp +/analysis/token_filters/cld2/cld2-read-only +/analysis/token_filters/cld2/libcld2_full.a +/cmd/bleve/bleve +vendor/** +!vendor/manifest +/y.output +/search/query/y.output +*.test +tags +go.sum diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..1cc8ecc --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,16 @@ +# Contributing to Bleve + +We look forward to your contributions, but ask that you first review these guidelines. + +## Sign the CLA + +As Bleve is a Couchbase project we require contributors accept the [Couchbase Contributor License Agreement](http://review.couchbase.org/static/individual_agreement.html). To sign this agreement log into the Couchbase [code review tool](http://review.couchbase.org/). The Bleve project does not use this code review tool but it is still used to track acceptance of the contributor license agreements. + +## Submitting a Pull Request + +All types of contributions are welcome, but please keep the following in mind: + +- If you're planning a large change, you should really discuss it in a github issue or on the google group first. This helps avoid duplicate effort and spending time on something that may not be merged. +- Existing tests should continue to pass, new tests for the contribution are nice to have. +- All code should have gone through `go fmt` +- All code should pass `go vet` diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7a4a3ea --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..c0e85e5 --- /dev/null +++ b/README.md @@ -0,0 +1,124 @@ +# ![bleve](docs/bleve.png) bleve + +[![Tests](https://github.com/blevesearch/bleve/actions/workflows/tests.yml/badge.svg?branch=master&event=push)](https://github.com/blevesearch/bleve/actions/workflows/tests.yml?query=event%3Apush+branch%3Amaster) +[![Coverage Status](https://coveralls.io/repos/github/blevesearch/bleve/badge.svg)](https://coveralls.io/github/blevesearch/bleve) +[![Go Reference](https://pkg.go.dev/badge/github.com/blevesearch/bleve/v2.svg)](https://pkg.go.dev/github.com/blevesearch/bleve/v2) +[![Join the chat](https://badges.gitter.im/join_chat.svg)](https://app.gitter.im/#/room/#blevesearch_bleve:gitter.im) +[![Go Report Card](https://goreportcard.com/badge/github.com/blevesearch/bleve/v2)](https://goreportcard.com/report/github.com/blevesearch/bleve/v2) +[![Sourcegraph](https://sourcegraph.com/github.com/blevesearch/bleve/-/badge.svg)](https://sourcegraph.com/github.com/blevesearch/bleve?badge) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +A modern indexing + search library in GO + +## Features + +* Index any GO data structure or JSON +* Intelligent defaults backed up by powerful configuration ([scorch](https://github.com/blevesearch/bleve/blob/master/index/scorch/README.md)) +* Supported field types: + * `text`, `number`, `datetime`, `boolean`, `geopoint`, `geoshape`, `IP`, `vector` +* Supported query types: + * `term`, `phrase`, `match`, `match_phrase`, `prefix`, `regexp`, `wildcard`, `fuzzy` + * term range, numeric range, date range, boolean field + * compound queries: `conjuncts`, `disjuncts`, boolean (`must`/`should`/`must_not`) + * [query string syntax](http://www.blevesearch.com/docs/Query-String-Query/) + * [geo spatial search](https://github.com/blevesearch/bleve/blob/master/geo/README.md) + * approximate k-nearest neighbors via [vector search](https://github.com/blevesearch/bleve/blob/master/docs/vectors.md) + * [synonym search](https://github.com/blevesearch/bleve/blob/master/docs/synonyms.md) + * [hierarchical nested search](https://github.com/blevesearch/bleve/blob/master/docs/hierarchy.md) +* [tf-idf](https://github.com/blevesearch/bleve/blob/master/docs/scoring.md#tf-idf) / [bm25](https://github.com/blevesearch/bleve/blob/master/docs/scoring.md#bm25) scoring models +* Hybrid search: exact + semantic + * Supports [RRF (Reciprocal Rank Fusion) and RSF (Relative Score Fusion)](docs/score_fusion.md) +* [Result pagination](https://github.com/blevesearch/bleve/blob/master/docs/pagination.md) +* Query time boosting +* Search result match highlighting with document fragments +* Aggregations/faceting support: + * terms facet + * numeric range facet + * date range facet + +## Indexing + +```go +message := struct { + Id string + From string + Body string +}{ + Id: "example", + From: "xyz@couchbase.com", + Body: "bleve indexing is easy", +} + +mapping := bleve.NewIndexMapping() +index, err := bleve.New("example.bleve", mapping) +if err != nil { + panic(err) +} +index.Index(message.Id, message) +``` + +## Querying + +```go +index, _ := bleve.Open("example.bleve") +query := bleve.NewQueryStringQuery("bleve") +searchRequest := bleve.NewSearchRequest(query) +searchResult, _ := index.Search(searchRequest) +``` + +## Command Line Interface + +To install the CLI for the latest release of bleve, run: + +```bash +go install github.com/blevesearch/bleve/v2/cmd/bleve@latest +``` + +```text +$ bleve --help +Bleve is a command-line tool to interact with a bleve index. + +Usage: + bleve [command] + +Available Commands: + bulk bulk loads from newline delimited JSON files + check checks the contents of the index + count counts the number documents in the index + create creates a new index + dictionary prints the term dictionary for the specified field in the index + dump dumps the contents of the index + fields lists the fields in this index + help Help about any command + index adds the files to the index + mapping prints the mapping used for this index + query queries the index + registry registry lists the bleve components compiled into this executable + scorch command-line tool to interact with a scorch index + +Flags: + -h, --help help for bleve + +Use "bleve [command] --help" for more information about a command. +``` + +## Text Analysis + +Bleve includes general-purpose analyzers (customizable) as well as pre-built text analyzers for the following languages: + +Arabic (ar), Bulgarian (bg), Catalan (ca), Chinese-Japanese-Korean (cjk), Kurdish (ckb), Danish (da), German (de), Greek (el), English (en), Spanish - Castilian (es), Basque (eu), Persian (fa), Finnish (fi), French (fr), Gaelic (ga), Spanish - Galician (gl), Hindi (hi), Croatian (hr), Hungarian (hu), Armenian (hy), Indonesian (id, in), Italian (it), Dutch (nl), Norwegian (no), Polish (pl), Portuguese (pt), Romanian (ro), Russian (ru), Swedish (sv), Turkish (tr) + +## Text Analysis Wizard + +[bleveanalysis.couchbase.com](https://bleveanalysis.couchbase.com) + +## Discussion/Issues + +Discuss usage/development of bleve and/or report issues here: + +* [Github issues](https://github.com/blevesearch/bleve/issues) +* [Google group](https://groups.google.com/forum/#!forum/bleve) + +## License + +Apache License Version 2.0 diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..6f194a3 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,16 @@ +# Security Policy + +## Supported Versions + +We support the latest release (for example, bleve v2.5.x). + +## Reporting a Vulnerability + +All security issues for this project should be reported via email to [security@couchbase.com](mailto:security@couchbase.com) and [fts-team@couchbase.com](mailto:fts-team@couchbase.com). + +This mail will be delivered to the owners of this project. + +- To ensure your report is NOT marked as spam, please include the word "security/vulnerability" along with the project name (blevesearch/bleve) in the subject of the email. +- Please be as descriptive as possible while explaining the issue, and a testcase highlighting the issue is always welcome. + +Your email will be acknowledged at the soonest possible. diff --git a/analysis/analyzer/custom/custom.go b/analysis/analyzer/custom/custom.go new file mode 100644 index 0000000..9040e02 --- /dev/null +++ b/analysis/analyzer/custom/custom.go @@ -0,0 +1,148 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package custom + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "custom" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + + var err error + var charFilters []analysis.CharFilter + charFiltersValue, ok := config["char_filters"] + if ok { + switch charFiltersValue := charFiltersValue.(type) { + case []string: + charFilters, err = getCharFilters(charFiltersValue, cache) + if err != nil { + return nil, err + } + case []interface{}: + charFiltersNames, err := convertInterfaceSliceToStringSlice(charFiltersValue, "char filter") + if err != nil { + return nil, err + } + charFilters, err = getCharFilters(charFiltersNames, cache) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unsupported type for char_filters, must be slice") + } + } + + var tokenizerName string + tokenizerValue, ok := config["tokenizer"] + if ok { + tokenizerName, ok = tokenizerValue.(string) + if !ok { + return nil, fmt.Errorf("must specify tokenizer as string") + } + } else { + return nil, fmt.Errorf("must specify tokenizer") + } + + tokenizer, err := cache.TokenizerNamed(tokenizerName) + if err != nil { + return nil, err + } + + var tokenFilters []analysis.TokenFilter + tokenFiltersValue, ok := config["token_filters"] + if ok { + switch tokenFiltersValue := tokenFiltersValue.(type) { + case []string: + tokenFilters, err = getTokenFilters(tokenFiltersValue, cache) + if err != nil { + return nil, err + } + case []interface{}: + tokenFiltersNames, err := convertInterfaceSliceToStringSlice(tokenFiltersValue, "token filter") + if err != nil { + return nil, err + } + tokenFilters, err = getTokenFilters(tokenFiltersNames, cache) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unsupported type for token_filters, must be slice") + } + } + + rv := analysis.DefaultAnalyzer{ + Tokenizer: tokenizer, + } + if charFilters != nil { + rv.CharFilters = charFilters + } + if tokenFilters != nil { + rv.TokenFilters = tokenFilters + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(Name, AnalyzerConstructor) + if err != nil { + panic(err) + } +} + +func getCharFilters(charFilterNames []string, cache *registry.Cache) ([]analysis.CharFilter, error) { + charFilters := make([]analysis.CharFilter, len(charFilterNames)) + for i, charFilterName := range charFilterNames { + charFilter, err := cache.CharFilterNamed(charFilterName) + if err != nil { + return nil, err + } + charFilters[i] = charFilter + } + + return charFilters, nil +} + +func getTokenFilters(tokenFilterNames []string, cache *registry.Cache) ([]analysis.TokenFilter, error) { + tokenFilters := make([]analysis.TokenFilter, len(tokenFilterNames)) + for i, tokenFilterName := range tokenFilterNames { + tokenFilter, err := cache.TokenFilterNamed(tokenFilterName) + if err != nil { + return nil, err + } + tokenFilters[i] = tokenFilter + } + + return tokenFilters, nil +} + +func convertInterfaceSliceToStringSlice(interfaceSlice []interface{}, objType string) ([]string, error) { + stringSlice := make([]string, len(interfaceSlice)) + for i, interfaceObj := range interfaceSlice { + stringObj, ok := interfaceObj.(string) + if ok { + stringSlice[i] = stringObj + } else { + return nil, fmt.Errorf("%s name must be a string", objType) + } + } + + return stringSlice, nil +} diff --git a/analysis/analyzer/keyword/keyword.go b/analysis/analyzer/keyword/keyword.go new file mode 100644 index 0000000..6eb052e --- /dev/null +++ b/analysis/analyzer/keyword/keyword.go @@ -0,0 +1,41 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyword + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/single" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "keyword" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + keywordTokenizer, err := cache.TokenizerNamed(single.Name) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: keywordTokenizer, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(Name, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/analyzer/simple/simple.go b/analysis/analyzer/simple/simple.go new file mode 100644 index 0000000..2a9834b --- /dev/null +++ b/analysis/analyzer/simple/simple.go @@ -0,0 +1,49 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package simple + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/letter" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "simple" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + tokenizer, err := cache.TokenizerNamed(letter.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: tokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(Name, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/analyzer/standard/standard.go b/analysis/analyzer/standard/standard.go new file mode 100644 index 0000000..fa752be --- /dev/null +++ b/analysis/analyzer/standard/standard.go @@ -0,0 +1,55 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package standard + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/lang/en" + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "standard" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + tokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopEnFilter, err := cache.TokenFilterNamed(en.StopName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: tokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + stopEnFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(Name, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/analyzer/web/web.go b/analysis/analyzer/web/web.go new file mode 100644 index 0000000..778586e --- /dev/null +++ b/analysis/analyzer/web/web.go @@ -0,0 +1,55 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package web + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/lang/en" + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/web" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "web" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + tokenizer, err := cache.TokenizerNamed(web.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopEnFilter, err := cache.TokenFilterNamed(en.StopName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: tokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + stopEnFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(Name, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/benchmark_test.go b/analysis/benchmark_test.go new file mode 100644 index 0000000..c5b6647 --- /dev/null +++ b/analysis/benchmark_test.go @@ -0,0 +1,117 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package analysis_test + +import ( + index "github.com/blevesearch/bleve_index_api" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" + "github.com/blevesearch/bleve/v2/registry" +) + +func BenchmarkAnalysis(b *testing.B) { + for i := 0; i < b.N; i++ { + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(standard.Name) + if err != nil { + b.Fatal(err) + } + + ts := analyzer.Analyze(bleveWikiArticle) + freqs := analysis.TokenFrequency(ts, nil, index.IncludeTermVectors) + if len(freqs) != 511 { + b.Errorf("expected %d freqs, got %d", 511, len(freqs)) + } + } +} + +var bleveWikiArticle = []byte(`Boiling liquid expanding vapor explosion +From Wikipedia, the free encyclopedia +See also: Boiler explosion and Steam explosion + +Flames subsequent to a flammable liquid BLEVE from a tanker. BLEVEs do not necessarily involve fire. + +This article's tone or style may not reflect the encyclopedic tone used on Wikipedia. See Wikipedia's guide to writing better articles for suggestions. (July 2013) +A boiling liquid expanding vapor explosion (BLEVE, /ˈblɛviː/ blev-ee) is an explosion caused by the rupture of a vessel containing a pressurized liquid above its boiling point.[1] +Contents [hide] +1 Mechanism +1.1 Water example +1.2 BLEVEs without chemical reactions +2 Fires +3 Incidents +4 Safety measures +5 See also +6 References +7 External links +Mechanism[edit] + +This section needs additional citations for verification. Please help improve this article by adding citations to reliable sources. Unsourced material may be challenged and removed. (July 2013) +There are three characteristics of liquids which are relevant to the discussion of a BLEVE: +If a liquid in a sealed container is boiled, the pressure inside the container increases. As the liquid changes to a gas it expands - this expansion in a vented container would cause the gas and liquid to take up more space. In a sealed container the gas and liquid are not able to take up more space and so the pressure rises. Pressurized vessels containing liquids can reach an equilibrium where the liquid stops boiling and the pressure stops rising. This occurs when no more heat is being added to the system (either because it has reached ambient temperature or has had a heat source removed). +The boiling temperature of a liquid is dependent on pressure - high pressures will yield high boiling temperatures, and low pressures will yield low boiling temperatures. A common simple experiment is to place a cup of water in a vacuum chamber, and then reduce the pressure in the chamber until the water boils. By reducing the pressure the water will boil even at room temperature. This works both ways - if the pressure is increased beyond normal atmospheric pressures, the boiling of hot water could be suppressed far beyond normal temperatures. The cooling system of a modern internal combustion engine is a real-world example. +When a liquid boils it turns into a gas. The resulting gas takes up far more space than the liquid did. +Typically, a BLEVE starts with a container of liquid which is held above its normal, atmospheric-pressure boiling temperature. Many substances normally stored as liquids, such as CO2, propane, and other similar industrial gases have boiling temperatures, at atmospheric pressure, far below room temperature. In the case of water, a BLEVE could occur if a pressurized chamber of water is heated far beyond the standard 100 °C (212 °F). That container, because the boiling water pressurizes it, is capable of holding liquid water at very high temperatures. +If the pressurized vessel, containing liquid at high temperature (which may be room temperature, depending on the substance) ruptures, the pressure which prevents the liquid from boiling is lost. If the rupture is catastrophic, where the vessel is immediately incapable of holding any pressure at all, then there suddenly exists a large mass of liquid which is at very high temperature and very low pressure. This causes the entire volume of liquid to instantaneously boil, which in turn causes an extremely rapid expansion. Depending on temperatures, pressures and the substance involved, that expansion may be so rapid that it can be classified as an explosion, fully capable of inflicting severe damage on its surroundings. +Water example[edit] +Imagine, for example, a tank of pressurized liquid water held at 204.4 °C (400 °F). This tank would normally be pressurized to 1.7 MPa (250 psi) above atmospheric ("gauge") pressure. If the tank containing the water were to rupture, there would for a slight moment exist a volume of liquid water which would be +at atmospheric pressure, and +204.4 °C (400 °F). +At atmospheric pressure the boiling point of water is 100 °C (212 °F) - liquid water at atmospheric pressure cannot exist at temperatures higher than 100 °C (212 °F). At that moment, the water would boil and turn to vapour explosively, and the 204.4 °C (400 °F) liquid water turned to gas would take up a lot more volume than it did as liquid, causing a vapour explosion. Such explosions can happen when the superheated water of a steam engine escapes through a crack in a boiler, causing a boiler explosion. +BLEVEs without chemical reactions[edit] +It is important to note that a BLEVE need not be a chemical explosion—nor does there need to be a fire—however if a flammable substance is subject to a BLEVE it may also be subject to intense heating, either from an external source of heat which may have caused the vessel to rupture in the first place or from an internal source of localized heating such as skin friction. This heating can cause a flammable substance to ignite, adding a secondary explosion caused by the primary BLEVE. While blast effects of any BLEVE can be devastating, a flammable substance such as propane can add significantly to the danger. +Bleve explosion.svg +While the term BLEVE is most often used to describe the results of a container of flammable liquid rupturing due to fire, a BLEVE can occur even with a non-flammable substance such as water,[2] liquid nitrogen,[3] liquid helium or other refrigerants or cryogens, and therefore is not usually considered a type of chemical explosion. +Fires[edit] +BLEVEs can be caused by an external fire near the storage vessel causing heating of the contents and pressure build-up. While tanks are often designed to withstand great pressure, constant heating can cause the metal to weaken and eventually fail. If the tank is being heated in an area where there is no liquid, it may rupture faster without the liquid to absorb the heat. Gas containers are usually equipped with relief valves that vent off excess pressure, but the tank can still fail if the pressure is not released quickly enough.[1] Relief valves are sized to release pressure fast enough to prevent the pressure from increasing beyond the strength of the vessel, but not so fast as to be the cause of an explosion. An appropriately sized relief valve will allow the liquid inside to boil slowly, maintaining a constant pressure in the vessel until all the liquid has boiled and the vessel empties. +If the substance involved is flammable, it is likely that the resulting cloud of the substance will ignite after the BLEVE has occurred, forming a fireball and possibly a fuel-air explosion, also termed a vapor cloud explosion (VCE). If the materials are toxic, a large area will be contaminated.[4] +Incidents[edit] +The term "BLEVE" was coined by three researchers at Factory Mutual, in the analysis of an accident there in 1957 involving a chemical reactor vessel.[5] +In August 1959 the Kansas City Fire Department suffered its largest ever loss of life in the line of duty, when a 25,000 gallon (95,000 litre) gas tank exploded during a fire on Southwest Boulevard killing five firefighters. This was the first time BLEVE was used to describe a burning fuel tank.[citation needed] +Later incidents included the Cheapside Street Whisky Bond Fire in Glasgow, Scotland in 1960; Feyzin, France in 1966; Crescent City, Illinois in 1970; Kingman, Arizona in 1973; a liquid nitrogen tank rupture[6] at Air Products and Chemicals and Mobay Chemical Company at New Martinsville, West Virginia on January 31, 1978 [1];Texas City, Texas in 1978; Murdock, Illinois in 1983; San Juan Ixhuatepec, Mexico City in 1984; and Toronto, Ontario in 2008. +Safety measures[edit] +[icon] This section requires expansion. (July 2013) +Some fire mitigation measures are listed under liquefied petroleum gas. +See also[edit] +Boiler explosion +Expansion ratio +Explosive boiling or phase explosion +Rapid phase transition +Viareggio train derailment +2008 Toronto explosions +Gas carriers +Los Alfaques Disaster +Lac-Mégantic derailment +References[edit] +^ Jump up to: a b Kletz, Trevor (March 1990). Critical Aspects of Safety and Loss Prevention. London: Butterworth–Heinemann. pp. 43–45. ISBN 0-408-04429-2. +Jump up ^ "Temperature Pressure Relief Valves on Water Heaters: test, inspect, replace, repair guide". Inspect-ny.com. Retrieved 2011-07-12. +Jump up ^ Liquid nitrogen BLEVE demo +Jump up ^ "Chemical Process Safety" (PDF). Retrieved 2011-07-12. +Jump up ^ David F. Peterson, BLEVE: Facts, Risk Factors, and Fallacies, Fire Engineering magazine (2002). +Jump up ^ "STATE EX REL. VAPOR CORP. v. NARICK". Supreme Court of Appeals of West Virginia. 1984-07-12. Retrieved 2014-03-16. +External links[edit] + Look up boiling liquid expanding vapor explosion in Wiktionary, the free dictionary. + Wikimedia Commons has media related to BLEVE. +BLEVE Demo on YouTube — video of a controlled BLEVE demo +huge explosions on YouTube — video of propane and isobutane BLEVEs from a train derailment at Murdock, Illinois (3 September 1983) +Propane BLEVE on YouTube — video of BLEVE from the Toronto propane depot fire +Moscow Ring Road Accident on YouTube - Dozens of LPG tank BLEVEs after a road accident in Moscow +Kingman, AZ BLEVE — An account of the 5 July 1973 explosion in Kingman, with photographs +Propane Tank Explosions — Description of circumstances required to cause a propane tank BLEVE. +Analysis of BLEVE Events at DOE Sites - Details physics and mathematics of BLEVEs. +HID - SAFETY REPORT ASSESSMENT GUIDE: Whisky Maturation Warehouses - The liquor is aged in wooden barrels that can suffer BLEVE. +Categories: ExplosivesFirefightingFireTypes of fireGas technologiesIndustrial fires and explosions`) diff --git a/analysis/char/asciifolding/asciifolding.go b/analysis/char/asciifolding/asciifolding.go new file mode 100644 index 0000000..9e21c61 --- /dev/null +++ b/analysis/char/asciifolding/asciifolding.go @@ -0,0 +1,3572 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// converted to Go from Lucene's AsciiFoldingFilter +// https://lucene.apache.org/core/4_0_0/analyzers-common/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.html + +package asciifolding + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "asciifolding" + +type AsciiFoldingFilter struct{} + +func New() *AsciiFoldingFilter { + return &AsciiFoldingFilter{} +} + +func (s *AsciiFoldingFilter) Filter(input []byte) []byte { + if len(input) == 0 { + return input + } + + in := []rune(string(input)) + length := len(in) + + // Worst-case length required if all runes fold to 4 runes + out := make([]rune, length, length*4) + + out = foldToASCII(in, 0, out, 0, length) + return []byte(string(out)) +} + +func AsciiFoldingFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.CharFilter, error) { + return New(), nil +} + +func init() { + err := registry.RegisterCharFilter(Name, AsciiFoldingFilterConstructor) + if err != nil { + panic(err) + } +} + +// Converts characters above ASCII to their ASCII equivalents. +// For example, accents are removed from accented characters. +func foldToASCII(input []rune, inputPos int, output []rune, outputPos int, length int) []rune { + end := inputPos + length + for pos := inputPos; pos < end; pos++ { + c := input[pos] + + // Quick test: if it's not in range then just keep current character + if c < '\u0080' { + output[outputPos] = c + outputPos++ + } else { + switch c { + case '\u00C0': // À [LATIN CAPITAL LETTER A WITH GRAVE] + fallthrough + case '\u00C1': // Á [LATIN CAPITAL LETTER A WITH ACUTE] + fallthrough + case '\u00C2': //  [LATIN CAPITAL LETTER A WITH CIRCUMFLEX] + fallthrough + case '\u00C3': // à [LATIN CAPITAL LETTER A WITH TILDE] + fallthrough + case '\u00C4': // Ä [LATIN CAPITAL LETTER A WITH DIAERESIS] + fallthrough + case '\u00C5': // Å [LATIN CAPITAL LETTER A WITH RING ABOVE] + fallthrough + case '\u0100': // Ā [LATIN CAPITAL LETTER A WITH MACRON] + fallthrough + case '\u0102': // Ă [LATIN CAPITAL LETTER A WITH BREVE] + fallthrough + case '\u0104': // Ą [LATIN CAPITAL LETTER A WITH OGONEK] + fallthrough + case '\u018F': // Ə http://en.wikipedia.org/wiki/Schwa [LATIN CAPITAL LETTER SCHWA] + fallthrough + case '\u01CD': // Ǎ [LATIN CAPITAL LETTER A WITH CARON] + fallthrough + case '\u01DE': // Ǟ [LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON] + fallthrough + case '\u01E0': // Ǡ [LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON] + fallthrough + case '\u01FA': // Ǻ [LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE] + fallthrough + case '\u0200': // Ȁ [LATIN CAPITAL LETTER A WITH DOUBLE GRAVE] + fallthrough + case '\u0202': // Ȃ [LATIN CAPITAL LETTER A WITH INVERTED BREVE] + fallthrough + case '\u0226': // Ȧ [LATIN CAPITAL LETTER A WITH DOT ABOVE] + fallthrough + case '\u023A': // Ⱥ [LATIN CAPITAL LETTER A WITH STROKE] + fallthrough + case '\u1D00': // ᴀ [LATIN LETTER SMALL CAPITAL A] + fallthrough + case '\u1E00': // Ḁ [LATIN CAPITAL LETTER A WITH RING BELOW] + fallthrough + case '\u1EA0': // Ạ [LATIN CAPITAL LETTER A WITH DOT BELOW] + fallthrough + case '\u1EA2': // Ả [LATIN CAPITAL LETTER A WITH HOOK ABOVE] + fallthrough + case '\u1EA4': // Ấ [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE] + fallthrough + case '\u1EA6': // Ầ [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE] + fallthrough + case '\u1EA8': // Ẩ [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE] + fallthrough + case '\u1EAA': // Ẫ [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE] + fallthrough + case '\u1EAC': // Ậ [LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW] + fallthrough + case '\u1EAE': // Ắ [LATIN CAPITAL LETTER A WITH BREVE AND ACUTE] + fallthrough + case '\u1EB0': // Ằ [LATIN CAPITAL LETTER A WITH BREVE AND GRAVE] + fallthrough + case '\u1EB2': // Ẳ [LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE] + fallthrough + case '\u1EB4': // Ẵ [LATIN CAPITAL LETTER A WITH BREVE AND TILDE] + fallthrough + case '\u24B6': // Ⓐ [CIRCLED LATIN CAPITAL LETTER A] + fallthrough + case '\uFF21': // Ａ [FULLWIDTH LATIN CAPITAL LETTER A] + fallthrough + case '\u1EB6': // Ặ [LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW] + output[outputPos] = 'A' + outputPos++ + + case '\u00E0': // à [LATIN SMALL LETTER A WITH GRAVE] + fallthrough + case '\u00E1': // á [LATIN SMALL LETTER A WITH ACUTE] + fallthrough + case '\u00E2': // â [LATIN SMALL LETTER A WITH CIRCUMFLEX] + fallthrough + case '\u00E3': // ã [LATIN SMALL LETTER A WITH TILDE] + fallthrough + case '\u00E4': // ä [LATIN SMALL LETTER A WITH DIAERESIS] + fallthrough + case '\u00E5': // å [LATIN SMALL LETTER A WITH RING ABOVE] + fallthrough + case '\u0101': // ā [LATIN SMALL LETTER A WITH MACRON] + fallthrough + case '\u0103': // ă [LATIN SMALL LETTER A WITH BREVE] + fallthrough + case '\u0105': // ą [LATIN SMALL LETTER A WITH OGONEK] + fallthrough + case '\u01CE': // ǎ [LATIN SMALL LETTER A WITH CARON] + fallthrough + case '\u01DF': // ǟ [LATIN SMALL LETTER A WITH DIAERESIS AND MACRON] + fallthrough + case '\u01E1': // ǡ [LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON] + fallthrough + case '\u01FB': // ǻ [LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE] + fallthrough + case '\u0201': // ȁ [LATIN SMALL LETTER A WITH DOUBLE GRAVE] + fallthrough + case '\u0203': // ȃ [LATIN SMALL LETTER A WITH INVERTED BREVE] + fallthrough + case '\u0227': // ȧ [LATIN SMALL LETTER A WITH DOT ABOVE] + fallthrough + case '\u0250': // ɐ [LATIN SMALL LETTER TURNED A] + fallthrough + case '\u0259': // ə [LATIN SMALL LETTER SCHWA] + fallthrough + case '\u025A': // ɚ [LATIN SMALL LETTER SCHWA WITH HOOK] + fallthrough + case '\u1D8F': // ᶏ [LATIN SMALL LETTER A WITH RETROFLEX HOOK] + fallthrough + case '\u1D95': // ᶕ [LATIN SMALL LETTER SCHWA WITH RETROFLEX HOOK] + fallthrough + case '\u1E01': // ạ [LATIN SMALL LETTER A WITH RING BELOW] + fallthrough + case '\u1E9A': // ả [LATIN SMALL LETTER A WITH RIGHT HALF RING] + fallthrough + case '\u1EA1': // ạ [LATIN SMALL LETTER A WITH DOT BELOW] + fallthrough + case '\u1EA3': // ả [LATIN SMALL LETTER A WITH HOOK ABOVE] + fallthrough + case '\u1EA5': // ấ [LATIN SMALL LETTER A WITH CIRCUMFLEX AND ACUTE] + fallthrough + case '\u1EA7': // ầ [LATIN SMALL LETTER A WITH CIRCUMFLEX AND GRAVE] + fallthrough + case '\u1EA9': // ẩ [LATIN SMALL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE] + fallthrough + case '\u1EAB': // ẫ [LATIN SMALL LETTER A WITH CIRCUMFLEX AND TILDE] + fallthrough + case '\u1EAD': // ậ [LATIN SMALL LETTER A WITH CIRCUMFLEX AND DOT BELOW] + fallthrough + case '\u1EAF': // ắ [LATIN SMALL LETTER A WITH BREVE AND ACUTE] + fallthrough + case '\u1EB1': // ằ [LATIN SMALL LETTER A WITH BREVE AND GRAVE] + fallthrough + case '\u1EB3': // ẳ [LATIN SMALL LETTER A WITH BREVE AND HOOK ABOVE] + fallthrough + case '\u1EB5': // ẵ [LATIN SMALL LETTER A WITH BREVE AND TILDE] + fallthrough + case '\u1EB7': // ặ [LATIN SMALL LETTER A WITH BREVE AND DOT BELOW] + fallthrough + case '\u2090': // ₐ [LATIN SUBSCRIPT SMALL LETTER A] + fallthrough + case '\u2094': // ₔ [LATIN SUBSCRIPT SMALL LETTER SCHWA] + fallthrough + case '\u24D0': // ⓐ [CIRCLED LATIN SMALL LETTER A] + fallthrough + case '\u2C65': // ⱥ [LATIN SMALL LETTER A WITH STROKE] + fallthrough + case '\u2C6F': // Ɐ [LATIN CAPITAL LETTER TURNED A] + fallthrough + case '\uFF41': // ａ [FULLWIDTH LATIN SMALL LETTER A] + output[outputPos] = 'a' + outputPos++ + + case '\uA732': // Ꜳ [LATIN CAPITAL LETTER AA] + output = output[:(len(output) + 1)] + output[outputPos] = 'A' + outputPos++ + output[outputPos] = 'A' + outputPos++ + + case '\u00C6': // Æ [LATIN CAPITAL LETTER AE] + fallthrough + case '\u01E2': // Ǣ [LATIN CAPITAL LETTER AE WITH MACRON] + fallthrough + case '\u01FC': // Ǽ [LATIN CAPITAL LETTER AE WITH ACUTE] + fallthrough + case '\u1D01': // ᴁ [LATIN LETTER SMALL CAPITAL AE] + output = output[:(len(output) + 1)] + output[outputPos] = 'A' + outputPos++ + output[outputPos] = 'E' + outputPos++ + + case '\uA734': // Ꜵ [LATIN CAPITAL LETTER AO] + output = output[:(len(output) + 1)] + output[outputPos] = 'A' + outputPos++ + output[outputPos] = 'O' + outputPos++ + + case '\uA736': // Ꜷ [LATIN CAPITAL LETTER AU] + output = output[:(len(output) + 1)] + output[outputPos] = 'A' + outputPos++ + output[outputPos] = 'U' + outputPos++ + + case '\uA738': // Ꜹ [LATIN CAPITAL LETTER AV] + fallthrough + case '\uA73A': // Ꜻ [LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR] + output = output[:(len(output) + 1)] + output[outputPos] = 'A' + outputPos++ + output[outputPos] = 'V' + outputPos++ + + case '\uA73C': // Ꜽ [LATIN CAPITAL LETTER AY] + output = output[:(len(output) + 1)] + output[outputPos] = 'A' + outputPos++ + output[outputPos] = 'Y' + outputPos++ + + case '\u249C': // ⒜ [PARENTHESIZED LATIN SMALL LETTER A] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'a' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\uA733': // ꜳ [LATIN SMALL LETTER AA] + output = output[:(len(output) + 1)] + output[outputPos] = 'a' + outputPos++ + output[outputPos] = 'a' + outputPos++ + + case '\u00E6': // æ [LATIN SMALL LETTER AE] + fallthrough + case '\u01E3': // ǣ [LATIN SMALL LETTER AE WITH MACRON] + fallthrough + case '\u01FD': // ǽ [LATIN SMALL LETTER AE WITH ACUTE] + fallthrough + case '\u1D02': // ᴂ [LATIN SMALL LETTER TURNED AE] + output = output[:(len(output) + 1)] + output[outputPos] = 'a' + outputPos++ + output[outputPos] = 'e' + outputPos++ + + case '\uA735': // ꜵ [LATIN SMALL LETTER AO] + output = output[:(len(output) + 1)] + output[outputPos] = 'a' + outputPos++ + output[outputPos] = 'o' + outputPos++ + + case '\uA737': // ꜷ [LATIN SMALL LETTER AU] + output = output[:(len(output) + 1)] + output[outputPos] = 'a' + outputPos++ + output[outputPos] = 'u' + outputPos++ + + case '\uA739': // ꜹ [LATIN SMALL LETTER AV] + fallthrough + case '\uA73B': // ꜻ [LATIN SMALL LETTER AV WITH HORIZONTAL BAR] + output = output[:(len(output) + 1)] + output[outputPos] = 'a' + outputPos++ + output[outputPos] = 'v' + outputPos++ + + case '\uA73D': // ꜽ [LATIN SMALL LETTER AY] + output = output[:(len(output) + 1)] + output[outputPos] = 'a' + outputPos++ + output[outputPos] = 'y' + outputPos++ + + case '\u0181': // Ɓ [LATIN CAPITAL LETTER B WITH HOOK] + fallthrough + case '\u0182': // Ƃ [LATIN CAPITAL LETTER B WITH TOPBAR] + fallthrough + case '\u0243': // Ƀ [LATIN CAPITAL LETTER B WITH STROKE] + fallthrough + case '\u0299': // ʙ [LATIN LETTER SMALL CAPITAL B] + fallthrough + case '\u1D03': // ᴃ [LATIN LETTER SMALL CAPITAL BARRED B] + fallthrough + case '\u1E02': // Ḃ [LATIN CAPITAL LETTER B WITH DOT ABOVE] + fallthrough + case '\u1E04': // Ḅ [LATIN CAPITAL LETTER B WITH DOT BELOW] + fallthrough + case '\u1E06': // Ḇ [LATIN CAPITAL LETTER B WITH LINE BELOW] + fallthrough + case '\u24B7': // Ⓑ [CIRCLED LATIN CAPITAL LETTER B] + fallthrough + case '\uFF22': // Ｂ [FULLWIDTH LATIN CAPITAL LETTER B] + output[outputPos] = 'B' + outputPos++ + + case '\u0180': // ƀ [LATIN SMALL LETTER B WITH STROKE] + fallthrough + case '\u0183': // ƃ [LATIN SMALL LETTER B WITH TOPBAR] + fallthrough + case '\u0253': // ɓ [LATIN SMALL LETTER B WITH HOOK] + fallthrough + case '\u1D6C': // ᵬ [LATIN SMALL LETTER B WITH MIDDLE TILDE] + fallthrough + case '\u1D80': // ᶀ [LATIN SMALL LETTER B WITH PALATAL HOOK] + fallthrough + case '\u1E03': // ḃ [LATIN SMALL LETTER B WITH DOT ABOVE] + fallthrough + case '\u1E05': // ḅ [LATIN SMALL LETTER B WITH DOT BELOW] + fallthrough + case '\u1E07': // ḇ [LATIN SMALL LETTER B WITH LINE BELOW] + fallthrough + case '\u24D1': // ⓑ [CIRCLED LATIN SMALL LETTER B] + fallthrough + case '\uFF42': // ｂ [FULLWIDTH LATIN SMALL LETTER B] + output[outputPos] = 'b' + outputPos++ + + case '\u249D': // ⒝ [PARENTHESIZED LATIN SMALL LETTER B] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'b' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u00C7': // Ç [LATIN CAPITAL LETTER C WITH CEDILLA] + fallthrough + case '\u0106': // Ć [LATIN CAPITAL LETTER C WITH ACUTE] + fallthrough + case '\u0108': // Ĉ [LATIN CAPITAL LETTER C WITH CIRCUMFLEX] + fallthrough + case '\u010A': // Ċ [LATIN CAPITAL LETTER C WITH DOT ABOVE] + fallthrough + case '\u010C': // Č [LATIN CAPITAL LETTER C WITH CARON] + fallthrough + case '\u0187': // Ƈ [LATIN CAPITAL LETTER C WITH HOOK] + fallthrough + case '\u023B': // Ȼ [LATIN CAPITAL LETTER C WITH STROKE] + fallthrough + case '\u0297': // ʗ [LATIN LETTER STRETCHED C] + fallthrough + case '\u1D04': // ᴄ [LATIN LETTER SMALL CAPITAL C] + fallthrough + case '\u1E08': // Ḉ [LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE] + fallthrough + case '\u24B8': // Ⓒ [CIRCLED LATIN CAPITAL LETTER C] + fallthrough + case '\uFF23': // Ｃ [FULLWIDTH LATIN CAPITAL LETTER C] + output[outputPos] = 'C' + outputPos++ + + case '\u00E7': // ç [LATIN SMALL LETTER C WITH CEDILLA] + fallthrough + case '\u0107': // ć [LATIN SMALL LETTER C WITH ACUTE] + fallthrough + case '\u0109': // ĉ [LATIN SMALL LETTER C WITH CIRCUMFLEX] + fallthrough + case '\u010B': // ċ [LATIN SMALL LETTER C WITH DOT ABOVE] + fallthrough + case '\u010D': // č [LATIN SMALL LETTER C WITH CARON] + fallthrough + case '\u0188': // ƈ [LATIN SMALL LETTER C WITH HOOK] + fallthrough + case '\u023C': // ȼ [LATIN SMALL LETTER C WITH STROKE] + fallthrough + case '\u0255': // ɕ [LATIN SMALL LETTER C WITH CURL] + fallthrough + case '\u1E09': // ḉ [LATIN SMALL LETTER C WITH CEDILLA AND ACUTE] + fallthrough + case '\u2184': // ↄ [LATIN SMALL LETTER REVERSED C] + fallthrough + case '\u24D2': // ⓒ [CIRCLED LATIN SMALL LETTER C] + fallthrough + case '\uA73E': // Ꜿ [LATIN CAPITAL LETTER REVERSED C WITH DOT] + fallthrough + case '\uA73F': // ꜿ [LATIN SMALL LETTER REVERSED C WITH DOT] + fallthrough + case '\uFF43': // ｃ [FULLWIDTH LATIN SMALL LETTER C] + output[outputPos] = 'c' + outputPos++ + + case '\u249E': // ⒞ [PARENTHESIZED LATIN SMALL LETTER C] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'c' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u00D0': // Ð [LATIN CAPITAL LETTER ETH] + fallthrough + case '\u010E': // Ď [LATIN CAPITAL LETTER D WITH CARON] + fallthrough + case '\u0110': // Đ [LATIN CAPITAL LETTER D WITH STROKE] + fallthrough + case '\u0189': // Ɖ [LATIN CAPITAL LETTER AFRICAN D] + fallthrough + case '\u018A': // Ɗ [LATIN CAPITAL LETTER D WITH HOOK] + fallthrough + case '\u018B': // Ƌ [LATIN CAPITAL LETTER D WITH TOPBAR] + fallthrough + case '\u1D05': // ᴅ [LATIN LETTER SMALL CAPITAL D] + fallthrough + case '\u1D06': // ᴆ [LATIN LETTER SMALL CAPITAL ETH] + fallthrough + case '\u1E0A': // Ḋ [LATIN CAPITAL LETTER D WITH DOT ABOVE] + fallthrough + case '\u1E0C': // Ḍ [LATIN CAPITAL LETTER D WITH DOT BELOW] + fallthrough + case '\u1E0E': // Ḏ [LATIN CAPITAL LETTER D WITH LINE BELOW] + fallthrough + case '\u1E10': // Ḑ [LATIN CAPITAL LETTER D WITH CEDILLA] + fallthrough + case '\u1E12': // Ḓ [LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW] + fallthrough + case '\u24B9': // Ⓓ [CIRCLED LATIN CAPITAL LETTER D] + fallthrough + case '\uA779': // Ꝺ [LATIN CAPITAL LETTER INSULAR D] + fallthrough + case '\uFF24': // Ｄ [FULLWIDTH LATIN CAPITAL LETTER D] + output[outputPos] = 'D' + outputPos++ + + case '\u00F0': // ð [LATIN SMALL LETTER ETH] + fallthrough + case '\u010F': // ď [LATIN SMALL LETTER D WITH CARON] + fallthrough + case '\u0111': // đ [LATIN SMALL LETTER D WITH STROKE] + fallthrough + case '\u018C': // ƌ [LATIN SMALL LETTER D WITH TOPBAR] + fallthrough + case '\u0221': // ȡ [LATIN SMALL LETTER D WITH CURL] + fallthrough + case '\u0256': // ɖ [LATIN SMALL LETTER D WITH TAIL] + fallthrough + case '\u0257': // ɗ [LATIN SMALL LETTER D WITH HOOK] + fallthrough + case '\u1D6D': // ᵭ [LATIN SMALL LETTER D WITH MIDDLE TILDE] + fallthrough + case '\u1D81': // ᶁ [LATIN SMALL LETTER D WITH PALATAL HOOK] + fallthrough + case '\u1D91': // ᶑ [LATIN SMALL LETTER D WITH HOOK AND TAIL] + fallthrough + case '\u1E0B': // ḋ [LATIN SMALL LETTER D WITH DOT ABOVE] + fallthrough + case '\u1E0D': // ḍ [LATIN SMALL LETTER D WITH DOT BELOW] + fallthrough + case '\u1E0F': // ḏ [LATIN SMALL LETTER D WITH LINE BELOW] + fallthrough + case '\u1E11': // ḑ [LATIN SMALL LETTER D WITH CEDILLA] + fallthrough + case '\u1E13': // ḓ [LATIN SMALL LETTER D WITH CIRCUMFLEX BELOW] + fallthrough + case '\u24D3': // ⓓ [CIRCLED LATIN SMALL LETTER D] + fallthrough + case '\uA77A': // ꝺ [LATIN SMALL LETTER INSULAR D] + fallthrough + case '\uFF44': // ｄ [FULLWIDTH LATIN SMALL LETTER D] + output[outputPos] = 'd' + outputPos++ + + case '\u01C4': // DŽ [LATIN CAPITAL LETTER DZ WITH CARON] + fallthrough + case '\u01F1': // DZ [LATIN CAPITAL LETTER DZ] + output = output[:(len(output) + 1)] + output[outputPos] = 'D' + outputPos++ + output[outputPos] = 'Z' + outputPos++ + + case '\u01C5': // Dž [LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON] + fallthrough + case '\u01F2': // Dz [LATIN CAPITAL LETTER D WITH SMALL LETTER Z] + output = output[:(len(output) + 1)] + output[outputPos] = 'D' + outputPos++ + output[outputPos] = 'z' + outputPos++ + + case '\u249F': // ⒟ [PARENTHESIZED LATIN SMALL LETTER D] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'd' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u0238': // ȸ [LATIN SMALL LETTER DB DIGRAPH] + output = output[:(len(output) + 1)] + output[outputPos] = 'd' + outputPos++ + output[outputPos] = 'b' + outputPos++ + + case '\u01C6': // dž [LATIN SMALL LETTER DZ WITH CARON] + fallthrough + case '\u01F3': // dz [LATIN SMALL LETTER DZ] + fallthrough + case '\u02A3': // ʣ [LATIN SMALL LETTER DZ DIGRAPH] + fallthrough + case '\u02A5': // ʥ [LATIN SMALL LETTER DZ DIGRAPH WITH CURL] + output = output[:(len(output) + 1)] + output[outputPos] = 'd' + outputPos++ + output[outputPos] = 'z' + outputPos++ + + case '\u00C8': // È [LATIN CAPITAL LETTER E WITH GRAVE] + fallthrough + case '\u00C9': // É [LATIN CAPITAL LETTER E WITH ACUTE] + fallthrough + case '\u00CA': // Ê [LATIN CAPITAL LETTER E WITH CIRCUMFLEX] + fallthrough + case '\u00CB': // Ë [LATIN CAPITAL LETTER E WITH DIAERESIS] + fallthrough + case '\u0112': // Ē [LATIN CAPITAL LETTER E WITH MACRON] + fallthrough + case '\u0114': // Ĕ [LATIN CAPITAL LETTER E WITH BREVE] + fallthrough + case '\u0116': // Ė [LATIN CAPITAL LETTER E WITH DOT ABOVE] + fallthrough + case '\u0118': // Ę [LATIN CAPITAL LETTER E WITH OGONEK] + fallthrough + case '\u011A': // Ě [LATIN CAPITAL LETTER E WITH CARON] + fallthrough + case '\u018E': // Ǝ [LATIN CAPITAL LETTER REVERSED E] + fallthrough + case '\u0190': // Ɛ [LATIN CAPITAL LETTER OPEN E] + fallthrough + case '\u0204': // Ȅ [LATIN CAPITAL LETTER E WITH DOUBLE GRAVE] + fallthrough + case '\u0206': // Ȇ [LATIN CAPITAL LETTER E WITH INVERTED BREVE] + fallthrough + case '\u0228': // Ȩ [LATIN CAPITAL LETTER E WITH CEDILLA] + fallthrough + case '\u0246': // Ɇ [LATIN CAPITAL LETTER E WITH STROKE] + fallthrough + case '\u1D07': // ᴇ [LATIN LETTER SMALL CAPITAL E] + fallthrough + case '\u1E14': // Ḕ [LATIN CAPITAL LETTER E WITH MACRON AND GRAVE] + fallthrough + case '\u1E16': // Ḗ [LATIN CAPITAL LETTER E WITH MACRON AND ACUTE] + fallthrough + case '\u1E18': // Ḙ [LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW] + fallthrough + case '\u1E1A': // Ḛ [LATIN CAPITAL LETTER E WITH TILDE BELOW] + fallthrough + case '\u1E1C': // Ḝ [LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE] + fallthrough + case '\u1EB8': // Ẹ [LATIN CAPITAL LETTER E WITH DOT BELOW] + fallthrough + case '\u1EBA': // Ẻ [LATIN CAPITAL LETTER E WITH HOOK ABOVE] + fallthrough + case '\u1EBC': // Ẽ [LATIN CAPITAL LETTER E WITH TILDE] + fallthrough + case '\u1EBE': // Ế [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE] + fallthrough + case '\u1EC0': // Ề [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE] + fallthrough + case '\u1EC2': // Ể [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE] + fallthrough + case '\u1EC4': // Ễ [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE] + fallthrough + case '\u1EC6': // Ệ [LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW] + fallthrough + case '\u24BA': // Ⓔ [CIRCLED LATIN CAPITAL LETTER E] + fallthrough + case '\u2C7B': // ⱻ [LATIN LETTER SMALL CAPITAL TURNED E] + fallthrough + case '\uFF25': // Ｅ [FULLWIDTH LATIN CAPITAL LETTER E] + output[outputPos] = 'E' + outputPos++ + + case '\u00E8': // è [LATIN SMALL LETTER E WITH GRAVE] + fallthrough + case '\u00E9': // é [LATIN SMALL LETTER E WITH ACUTE] + fallthrough + case '\u00EA': // ê [LATIN SMALL LETTER E WITH CIRCUMFLEX] + fallthrough + case '\u00EB': // ë [LATIN SMALL LETTER E WITH DIAERESIS] + fallthrough + case '\u0113': // ē [LATIN SMALL LETTER E WITH MACRON] + fallthrough + case '\u0115': // ĕ [LATIN SMALL LETTER E WITH BREVE] + fallthrough + case '\u0117': // ė [LATIN SMALL LETTER E WITH DOT ABOVE] + fallthrough + case '\u0119': // ę [LATIN SMALL LETTER E WITH OGONEK] + fallthrough + case '\u011B': // ě [LATIN SMALL LETTER E WITH CARON] + fallthrough + case '\u01DD': // ǝ [LATIN SMALL LETTER TURNED E] + fallthrough + case '\u0205': // ȅ [LATIN SMALL LETTER E WITH DOUBLE GRAVE] + fallthrough + case '\u0207': // ȇ [LATIN SMALL LETTER E WITH INVERTED BREVE] + fallthrough + case '\u0229': // ȩ [LATIN SMALL LETTER E WITH CEDILLA] + fallthrough + case '\u0247': // ɇ [LATIN SMALL LETTER E WITH STROKE] + fallthrough + case '\u0258': // ɘ [LATIN SMALL LETTER REVERSED E] + fallthrough + case '\u025B': // ɛ [LATIN SMALL LETTER OPEN E] + fallthrough + case '\u025C': // ɜ [LATIN SMALL LETTER REVERSED OPEN E] + fallthrough + case '\u025D': // ɝ [LATIN SMALL LETTER REVERSED OPEN E WITH HOOK] + fallthrough + case '\u025E': // ɞ [LATIN SMALL LETTER CLOSED REVERSED OPEN E] + fallthrough + case '\u029A': // ʚ [LATIN SMALL LETTER CLOSED OPEN E] + fallthrough + case '\u1D08': // ᴈ [LATIN SMALL LETTER TURNED OPEN E] + fallthrough + case '\u1D92': // ᶒ [LATIN SMALL LETTER E WITH RETROFLEX HOOK] + fallthrough + case '\u1D93': // ᶓ [LATIN SMALL LETTER OPEN E WITH RETROFLEX HOOK] + fallthrough + case '\u1D94': // ᶔ [LATIN SMALL LETTER REVERSED OPEN E WITH RETROFLEX HOOK] + fallthrough + case '\u1E15': // ḕ [LATIN SMALL LETTER E WITH MACRON AND GRAVE] + fallthrough + case '\u1E17': // ḗ [LATIN SMALL LETTER E WITH MACRON AND ACUTE] + fallthrough + case '\u1E19': // ḙ [LATIN SMALL LETTER E WITH CIRCUMFLEX BELOW] + fallthrough + case '\u1E1B': // ḛ [LATIN SMALL LETTER E WITH TILDE BELOW] + fallthrough + case '\u1E1D': // ḝ [LATIN SMALL LETTER E WITH CEDILLA AND BREVE] + fallthrough + case '\u1EB9': // ẹ [LATIN SMALL LETTER E WITH DOT BELOW] + fallthrough + case '\u1EBB': // ẻ [LATIN SMALL LETTER E WITH HOOK ABOVE] + fallthrough + case '\u1EBD': // ẽ [LATIN SMALL LETTER E WITH TILDE] + fallthrough + case '\u1EBF': // ế [LATIN SMALL LETTER E WITH CIRCUMFLEX AND ACUTE] + fallthrough + case '\u1EC1': // ề [LATIN SMALL LETTER E WITH CIRCUMFLEX AND GRAVE] + fallthrough + case '\u1EC3': // ể [LATIN SMALL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE] + fallthrough + case '\u1EC5': // ễ [LATIN SMALL LETTER E WITH CIRCUMFLEX AND TILDE] + fallthrough + case '\u1EC7': // ệ [LATIN SMALL LETTER E WITH CIRCUMFLEX AND DOT BELOW] + fallthrough + case '\u2091': // ₑ [LATIN SUBSCRIPT SMALL LETTER E] + fallthrough + case '\u24D4': // ⓔ [CIRCLED LATIN SMALL LETTER E] + fallthrough + case '\u2C78': // ⱸ [LATIN SMALL LETTER E WITH NOTCH] + fallthrough + case '\uFF45': // ｅ [FULLWIDTH LATIN SMALL LETTER E] + output[outputPos] = 'e' + outputPos++ + + case '\u24A0': // ⒠ [PARENTHESIZED LATIN SMALL LETTER E] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'e' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u0191': // Ƒ [LATIN CAPITAL LETTER F WITH HOOK] + fallthrough + case '\u1E1E': // Ḟ [LATIN CAPITAL LETTER F WITH DOT ABOVE] + fallthrough + case '\u24BB': // Ⓕ [CIRCLED LATIN CAPITAL LETTER F] + fallthrough + case '\uA730': // ꜰ [LATIN LETTER SMALL CAPITAL F] + fallthrough + case '\uA77B': // Ꝼ [LATIN CAPITAL LETTER INSULAR F] + fallthrough + case '\uA7FB': // ꟻ [LATIN EPIGRAPHIC LETTER REVERSED F] + fallthrough + case '\uFF26': // Ｆ [FULLWIDTH LATIN CAPITAL LETTER F] + output[outputPos] = 'F' + outputPos++ + + case '\u0192': // ƒ [LATIN SMALL LETTER F WITH HOOK] + fallthrough + case '\u1D6E': // ᵮ [LATIN SMALL LETTER F WITH MIDDLE TILDE] + fallthrough + case '\u1D82': // ᶂ [LATIN SMALL LETTER F WITH PALATAL HOOK] + fallthrough + case '\u1E1F': // ḟ [LATIN SMALL LETTER F WITH DOT ABOVE] + fallthrough + case '\u1E9B': // ẛ [LATIN SMALL LETTER LONG S WITH DOT ABOVE] + fallthrough + case '\u24D5': // ⓕ [CIRCLED LATIN SMALL LETTER F] + fallthrough + case '\uA77C': // ꝼ [LATIN SMALL LETTER INSULAR F] + fallthrough + case '\uFF46': // ｆ [FULLWIDTH LATIN SMALL LETTER F] + output[outputPos] = 'f' + outputPos++ + + case '\u24A1': // ⒡ [PARENTHESIZED LATIN SMALL LETTER F] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'f' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\uFB00': // ff [LATIN SMALL LIGATURE FF] + output = output[:(len(output) + 1)] + output[outputPos] = 'f' + outputPos++ + output[outputPos] = 'f' + outputPos++ + + case '\uFB03': // ffi [LATIN SMALL LIGATURE FFI] + output = output[:(len(output) + 2)] + output[outputPos] = 'f' + outputPos++ + output[outputPos] = 'f' + outputPos++ + output[outputPos] = 'i' + outputPos++ + + case '\uFB04': // ffl [LATIN SMALL LIGATURE FFL] + output = output[:(len(output) + 2)] + output[outputPos] = 'f' + outputPos++ + output[outputPos] = 'f' + outputPos++ + output[outputPos] = 'l' + outputPos++ + + case '\uFB01': // fi [LATIN SMALL LIGATURE FI] + output = output[:(len(output) + 1)] + output[outputPos] = 'f' + outputPos++ + output[outputPos] = 'i' + outputPos++ + + case '\uFB02': // fl [LATIN SMALL LIGATURE FL] + output = output[:(len(output) + 1)] + output[outputPos] = 'f' + outputPos++ + output[outputPos] = 'l' + outputPos++ + + case '\u011C': // Ĝ [LATIN CAPITAL LETTER G WITH CIRCUMFLEX] + fallthrough + case '\u011E': // Ğ [LATIN CAPITAL LETTER G WITH BREVE] + fallthrough + case '\u0120': // Ġ [LATIN CAPITAL LETTER G WITH DOT ABOVE] + fallthrough + case '\u0122': // Ģ [LATIN CAPITAL LETTER G WITH CEDILLA] + fallthrough + case '\u0193': // Ɠ [LATIN CAPITAL LETTER G WITH HOOK] + fallthrough + case '\u01E4': // Ǥ [LATIN CAPITAL LETTER G WITH STROKE] + fallthrough + case '\u01E5': // ǥ [LATIN SMALL LETTER G WITH STROKE] + fallthrough + case '\u01E6': // Ǧ [LATIN CAPITAL LETTER G WITH CARON] + fallthrough + case '\u01E7': // ǧ [LATIN SMALL LETTER G WITH CARON] + fallthrough + case '\u01F4': // Ǵ [LATIN CAPITAL LETTER G WITH ACUTE] + fallthrough + case '\u0262': // ɢ [LATIN LETTER SMALL CAPITAL G] + fallthrough + case '\u029B': // ʛ [LATIN LETTER SMALL CAPITAL G WITH HOOK] + fallthrough + case '\u1E20': // Ḡ [LATIN CAPITAL LETTER G WITH MACRON] + fallthrough + case '\u24BC': // Ⓖ [CIRCLED LATIN CAPITAL LETTER G] + fallthrough + case '\uA77D': // Ᵹ [LATIN CAPITAL LETTER INSULAR G] + fallthrough + case '\uA77E': // Ꝿ [LATIN CAPITAL LETTER TURNED INSULAR G] + fallthrough + case '\uFF27': // Ｇ [FULLWIDTH LATIN CAPITAL LETTER G] + output[outputPos] = 'G' + outputPos++ + + case '\u011D': // ĝ [LATIN SMALL LETTER G WITH CIRCUMFLEX] + fallthrough + case '\u011F': // ğ [LATIN SMALL LETTER G WITH BREVE] + fallthrough + case '\u0121': // ġ [LATIN SMALL LETTER G WITH DOT ABOVE] + fallthrough + case '\u0123': // ģ [LATIN SMALL LETTER G WITH CEDILLA] + fallthrough + case '\u01F5': // ǵ [LATIN SMALL LETTER G WITH ACUTE] + fallthrough + case '\u0260': // ɠ [LATIN SMALL LETTER G WITH HOOK] + fallthrough + case '\u0261': // ɡ [LATIN SMALL LETTER SCRIPT G] + fallthrough + case '\u1D77': // ᵷ [LATIN SMALL LETTER TURNED G] + fallthrough + case '\u1D79': // ᵹ [LATIN SMALL LETTER INSULAR G] + fallthrough + case '\u1D83': // ᶃ [LATIN SMALL LETTER G WITH PALATAL HOOK] + fallthrough + case '\u1E21': // ḡ [LATIN SMALL LETTER G WITH MACRON] + fallthrough + case '\u24D6': // ⓖ [CIRCLED LATIN SMALL LETTER G] + fallthrough + case '\uA77F': // ꝿ [LATIN SMALL LETTER TURNED INSULAR G] + fallthrough + case '\uFF47': // ｇ [FULLWIDTH LATIN SMALL LETTER G] + output[outputPos] = 'g' + outputPos++ + + case '\u24A2': // ⒢ [PARENTHESIZED LATIN SMALL LETTER G] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'g' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u0124': // Ĥ [LATIN CAPITAL LETTER H WITH CIRCUMFLEX] + fallthrough + case '\u0126': // Ħ [LATIN CAPITAL LETTER H WITH STROKE] + fallthrough + case '\u021E': // Ȟ [LATIN CAPITAL LETTER H WITH CARON] + fallthrough + case '\u029C': // ʜ [LATIN LETTER SMALL CAPITAL H] + fallthrough + case '\u1E22': // Ḣ [LATIN CAPITAL LETTER H WITH DOT ABOVE] + fallthrough + case '\u1E24': // Ḥ [LATIN CAPITAL LETTER H WITH DOT BELOW] + fallthrough + case '\u1E26': // Ḧ [LATIN CAPITAL LETTER H WITH DIAERESIS] + fallthrough + case '\u1E28': // Ḩ [LATIN CAPITAL LETTER H WITH CEDILLA] + fallthrough + case '\u1E2A': // Ḫ [LATIN CAPITAL LETTER H WITH BREVE BELOW] + fallthrough + case '\u24BD': // Ⓗ [CIRCLED LATIN CAPITAL LETTER H] + fallthrough + case '\u2C67': // Ⱨ [LATIN CAPITAL LETTER H WITH DESCENDER] + fallthrough + case '\u2C75': // Ⱶ [LATIN CAPITAL LETTER HALF H] + fallthrough + case '\uFF28': // Ｈ [FULLWIDTH LATIN CAPITAL LETTER H] + output[outputPos] = 'H' + outputPos++ + + case '\u0125': // ĥ [LATIN SMALL LETTER H WITH CIRCUMFLEX] + fallthrough + case '\u0127': // ħ [LATIN SMALL LETTER H WITH STROKE] + fallthrough + case '\u021F': // ȟ [LATIN SMALL LETTER H WITH CARON] + fallthrough + case '\u0265': // ɥ [LATIN SMALL LETTER TURNED H] + fallthrough + case '\u0266': // ɦ [LATIN SMALL LETTER H WITH HOOK] + fallthrough + case '\u02AE': // ʮ [LATIN SMALL LETTER TURNED H WITH FISHHOOK] + fallthrough + case '\u02AF': // ʯ [LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL] + fallthrough + case '\u1E23': // ḣ [LATIN SMALL LETTER H WITH DOT ABOVE] + fallthrough + case '\u1E25': // ḥ [LATIN SMALL LETTER H WITH DOT BELOW] + fallthrough + case '\u1E27': // ḧ [LATIN SMALL LETTER H WITH DIAERESIS] + fallthrough + case '\u1E29': // ḩ [LATIN SMALL LETTER H WITH CEDILLA] + fallthrough + case '\u1E2B': // ḫ [LATIN SMALL LETTER H WITH BREVE BELOW] + fallthrough + case '\u1E96': // ẖ [LATIN SMALL LETTER H WITH LINE BELOW] + fallthrough + case '\u24D7': // ⓗ [CIRCLED LATIN SMALL LETTER H] + fallthrough + case '\u2C68': // ⱨ [LATIN SMALL LETTER H WITH DESCENDER] + fallthrough + case '\u2C76': // ⱶ [LATIN SMALL LETTER HALF H] + fallthrough + case '\uFF48': // ｈ [FULLWIDTH LATIN SMALL LETTER H] + output[outputPos] = 'h' + outputPos++ + + case '\u01F6': // Ƕ http://en.wikipedia.org/wiki/Hwair [LATIN CAPITAL LETTER HWAIR] + output = output[:(len(output) + 1)] + output[outputPos] = 'H' + outputPos++ + output[outputPos] = 'V' + outputPos++ + + case '\u24A3': // ⒣ [PARENTHESIZED LATIN SMALL LETTER H] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'h' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u0195': // ƕ [LATIN SMALL LETTER HV] + output = output[:(len(output) + 1)] + output[outputPos] = 'h' + outputPos++ + output[outputPos] = 'v' + outputPos++ + + case '\u00CC': // Ì [LATIN CAPITAL LETTER I WITH GRAVE] + fallthrough + case '\u00CD': // Í [LATIN CAPITAL LETTER I WITH ACUTE] + fallthrough + case '\u00CE': // Î [LATIN CAPITAL LETTER I WITH CIRCUMFLEX] + fallthrough + case '\u00CF': // Ï [LATIN CAPITAL LETTER I WITH DIAERESIS] + fallthrough + case '\u0128': // Ĩ [LATIN CAPITAL LETTER I WITH TILDE] + fallthrough + case '\u012A': // Ī [LATIN CAPITAL LETTER I WITH MACRON] + fallthrough + case '\u012C': // Ĭ [LATIN CAPITAL LETTER I WITH BREVE] + fallthrough + case '\u012E': // Į [LATIN CAPITAL LETTER I WITH OGONEK] + fallthrough + case '\u0130': // İ [LATIN CAPITAL LETTER I WITH DOT ABOVE] + fallthrough + case '\u0196': // Ɩ [LATIN CAPITAL LETTER IOTA] + fallthrough + case '\u0197': // Ɨ [LATIN CAPITAL LETTER I WITH STROKE] + fallthrough + case '\u01CF': // Ǐ [LATIN CAPITAL LETTER I WITH CARON] + fallthrough + case '\u0208': // Ȉ [LATIN CAPITAL LETTER I WITH DOUBLE GRAVE] + fallthrough + case '\u020A': // Ȋ [LATIN CAPITAL LETTER I WITH INVERTED BREVE] + fallthrough + case '\u026A': // ɪ [LATIN LETTER SMALL CAPITAL I] + fallthrough + case '\u1D7B': // ᵻ [LATIN SMALL CAPITAL LETTER I WITH STROKE] + fallthrough + case '\u1E2C': // Ḭ [LATIN CAPITAL LETTER I WITH TILDE BELOW] + fallthrough + case '\u1E2E': // Ḯ [LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE] + fallthrough + case '\u1EC8': // Ỉ [LATIN CAPITAL LETTER I WITH HOOK ABOVE] + fallthrough + case '\u1ECA': // Ị [LATIN CAPITAL LETTER I WITH DOT BELOW] + fallthrough + case '\u24BE': // Ⓘ [CIRCLED LATIN CAPITAL LETTER I] + fallthrough + case '\uA7FE': // ꟾ [LATIN EPIGRAPHIC LETTER I LONGA] + fallthrough + case '\uFF29': // Ｉ [FULLWIDTH LATIN CAPITAL LETTER I] + output[outputPos] = 'I' + outputPos++ + + case '\u00EC': // ì [LATIN SMALL LETTER I WITH GRAVE] + fallthrough + case '\u00ED': // í [LATIN SMALL LETTER I WITH ACUTE] + fallthrough + case '\u00EE': // î [LATIN SMALL LETTER I WITH CIRCUMFLEX] + fallthrough + case '\u00EF': // ï [LATIN SMALL LETTER I WITH DIAERESIS] + fallthrough + case '\u0129': // ĩ [LATIN SMALL LETTER I WITH TILDE] + fallthrough + case '\u012B': // ī [LATIN SMALL LETTER I WITH MACRON] + fallthrough + case '\u012D': // ĭ [LATIN SMALL LETTER I WITH BREVE] + fallthrough + case '\u012F': // į [LATIN SMALL LETTER I WITH OGONEK] + fallthrough + case '\u0131': // ı [LATIN SMALL LETTER DOTLESS I] + fallthrough + case '\u01D0': // ǐ [LATIN SMALL LETTER I WITH CARON] + fallthrough + case '\u0209': // ȉ [LATIN SMALL LETTER I WITH DOUBLE GRAVE] + fallthrough + case '\u020B': // ȋ [LATIN SMALL LETTER I WITH INVERTED BREVE] + fallthrough + case '\u0268': // ɨ [LATIN SMALL LETTER I WITH STROKE] + fallthrough + case '\u1D09': // ᴉ [LATIN SMALL LETTER TURNED I] + fallthrough + case '\u1D62': // ᵢ [LATIN SUBSCRIPT SMALL LETTER I] + fallthrough + case '\u1D7C': // ᵼ [LATIN SMALL LETTER IOTA WITH STROKE] + fallthrough + case '\u1D96': // ᶖ [LATIN SMALL LETTER I WITH RETROFLEX HOOK] + fallthrough + case '\u1E2D': // ḭ [LATIN SMALL LETTER I WITH TILDE BELOW] + fallthrough + case '\u1E2F': // ḯ [LATIN SMALL LETTER I WITH DIAERESIS AND ACUTE] + fallthrough + case '\u1EC9': // ỉ [LATIN SMALL LETTER I WITH HOOK ABOVE] + fallthrough + case '\u1ECB': // ị [LATIN SMALL LETTER I WITH DOT BELOW] + fallthrough + case '\u2071': // ⁱ [SUPERSCRIPT LATIN SMALL LETTER I] + fallthrough + case '\u24D8': // ⓘ [CIRCLED LATIN SMALL LETTER I] + fallthrough + case '\uFF49': // ｉ [FULLWIDTH LATIN SMALL LETTER I] + output[outputPos] = 'i' + outputPos++ + + case '\u0132': // IJ [LATIN CAPITAL LIGATURE IJ] + output = output[:(len(output) + 1)] + output[outputPos] = 'I' + outputPos++ + output[outputPos] = 'J' + outputPos++ + + case '\u24A4': // ⒤ [PARENTHESIZED LATIN SMALL LETTER I] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'i' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u0133': // ij [LATIN SMALL LIGATURE IJ] + output = output[:(len(output) + 1)] + output[outputPos] = 'i' + outputPos++ + output[outputPos] = 'j' + outputPos++ + + case '\u0134': // Ĵ [LATIN CAPITAL LETTER J WITH CIRCUMFLEX] + fallthrough + case '\u0248': // Ɉ [LATIN CAPITAL LETTER J WITH STROKE] + fallthrough + case '\u1D0A': // ᴊ [LATIN LETTER SMALL CAPITAL J] + fallthrough + case '\u24BF': // Ⓙ [CIRCLED LATIN CAPITAL LETTER J] + fallthrough + case '\uFF2A': // Ｊ [FULLWIDTH LATIN CAPITAL LETTER J] + output[outputPos] = 'J' + outputPos++ + + case '\u0135': // ĵ [LATIN SMALL LETTER J WITH CIRCUMFLEX] + fallthrough + case '\u01F0': // ǰ [LATIN SMALL LETTER J WITH CARON] + fallthrough + case '\u0237': // ȷ [LATIN SMALL LETTER DOTLESS J] + fallthrough + case '\u0249': // ɉ [LATIN SMALL LETTER J WITH STROKE] + fallthrough + case '\u025F': // ɟ [LATIN SMALL LETTER DOTLESS J WITH STROKE] + fallthrough + case '\u0284': // ʄ [LATIN SMALL LETTER DOTLESS J WITH STROKE AND HOOK] + fallthrough + case '\u029D': // ʝ [LATIN SMALL LETTER J WITH CROSSED-TAIL] + fallthrough + case '\u24D9': // ⓙ [CIRCLED LATIN SMALL LETTER J] + fallthrough + case '\u2C7C': // ⱼ [LATIN SUBSCRIPT SMALL LETTER J] + fallthrough + case '\uFF4A': // ｊ [FULLWIDTH LATIN SMALL LETTER J] + output[outputPos] = 'j' + outputPos++ + + case '\u24A5': // ⒥ [PARENTHESIZED LATIN SMALL LETTER J] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'j' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u0136': // Ķ [LATIN CAPITAL LETTER K WITH CEDILLA] + fallthrough + case '\u0198': // Ƙ [LATIN CAPITAL LETTER K WITH HOOK] + fallthrough + case '\u01E8': // Ǩ [LATIN CAPITAL LETTER K WITH CARON] + fallthrough + case '\u1D0B': // ᴋ [LATIN LETTER SMALL CAPITAL K] + fallthrough + case '\u1E30': // Ḱ [LATIN CAPITAL LETTER K WITH ACUTE] + fallthrough + case '\u1E32': // Ḳ [LATIN CAPITAL LETTER K WITH DOT BELOW] + fallthrough + case '\u1E34': // Ḵ [LATIN CAPITAL LETTER K WITH LINE BELOW] + fallthrough + case '\u24C0': // Ⓚ [CIRCLED LATIN CAPITAL LETTER K] + fallthrough + case '\u2C69': // Ⱪ [LATIN CAPITAL LETTER K WITH DESCENDER] + fallthrough + case '\uA740': // Ꝁ [LATIN CAPITAL LETTER K WITH STROKE] + fallthrough + case '\uA742': // Ꝃ [LATIN CAPITAL LETTER K WITH DIAGONAL STROKE] + fallthrough + case '\uA744': // Ꝅ [LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE] + fallthrough + case '\uFF2B': // Ｋ [FULLWIDTH LATIN CAPITAL LETTER K] + output[outputPos] = 'K' + outputPos++ + + case '\u0137': // ķ [LATIN SMALL LETTER K WITH CEDILLA] + fallthrough + case '\u0199': // ƙ [LATIN SMALL LETTER K WITH HOOK] + fallthrough + case '\u01E9': // ǩ [LATIN SMALL LETTER K WITH CARON] + fallthrough + case '\u029E': // ʞ [LATIN SMALL LETTER TURNED K] + fallthrough + case '\u1D84': // ᶄ [LATIN SMALL LETTER K WITH PALATAL HOOK] + fallthrough + case '\u1E31': // ḱ [LATIN SMALL LETTER K WITH ACUTE] + fallthrough + case '\u1E33': // ḳ [LATIN SMALL LETTER K WITH DOT BELOW] + fallthrough + case '\u1E35': // ḵ [LATIN SMALL LETTER K WITH LINE BELOW] + fallthrough + case '\u24DA': // ⓚ [CIRCLED LATIN SMALL LETTER K] + fallthrough + case '\u2C6A': // ⱪ [LATIN SMALL LETTER K WITH DESCENDER] + fallthrough + case '\uA741': // ꝁ [LATIN SMALL LETTER K WITH STROKE] + fallthrough + case '\uA743': // ꝃ [LATIN SMALL LETTER K WITH DIAGONAL STROKE] + fallthrough + case '\uA745': // ꝅ [LATIN SMALL LETTER K WITH STROKE AND DIAGONAL STROKE] + fallthrough + case '\uFF4B': // ｋ [FULLWIDTH LATIN SMALL LETTER K] + output[outputPos] = 'k' + outputPos++ + + case '\u24A6': // ⒦ [PARENTHESIZED LATIN SMALL LETTER K] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'k' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u0139': // Ĺ [LATIN CAPITAL LETTER L WITH ACUTE] + fallthrough + case '\u013B': // Ļ [LATIN CAPITAL LETTER L WITH CEDILLA] + fallthrough + case '\u013D': // Ľ [LATIN CAPITAL LETTER L WITH CARON] + fallthrough + case '\u013F': // Ŀ [LATIN CAPITAL LETTER L WITH MIDDLE DOT] + fallthrough + case '\u0141': // Ł [LATIN CAPITAL LETTER L WITH STROKE] + fallthrough + case '\u023D': // Ƚ [LATIN CAPITAL LETTER L WITH BAR] + fallthrough + case '\u029F': // ʟ [LATIN LETTER SMALL CAPITAL L] + fallthrough + case '\u1D0C': // ᴌ [LATIN LETTER SMALL CAPITAL L WITH STROKE] + fallthrough + case '\u1E36': // Ḷ [LATIN CAPITAL LETTER L WITH DOT BELOW] + fallthrough + case '\u1E38': // Ḹ [LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON] + fallthrough + case '\u1E3A': // Ḻ [LATIN CAPITAL LETTER L WITH LINE BELOW] + fallthrough + case '\u1E3C': // Ḽ [LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW] + fallthrough + case '\u24C1': // Ⓛ [CIRCLED LATIN CAPITAL LETTER L] + fallthrough + case '\u2C60': // Ⱡ [LATIN CAPITAL LETTER L WITH DOUBLE BAR] + fallthrough + case '\u2C62': // Ɫ [LATIN CAPITAL LETTER L WITH MIDDLE TILDE] + fallthrough + case '\uA746': // Ꝇ [LATIN CAPITAL LETTER BROKEN L] + fallthrough + case '\uA748': // Ꝉ [LATIN CAPITAL LETTER L WITH HIGH STROKE] + fallthrough + case '\uA780': // Ꞁ [LATIN CAPITAL LETTER TURNED L] + fallthrough + case '\uFF2C': // Ｌ [FULLWIDTH LATIN CAPITAL LETTER L] + output[outputPos] = 'L' + outputPos++ + + case '\u013A': // ĺ [LATIN SMALL LETTER L WITH ACUTE] + fallthrough + case '\u013C': // ļ [LATIN SMALL LETTER L WITH CEDILLA] + fallthrough + case '\u013E': // ľ [LATIN SMALL LETTER L WITH CARON] + fallthrough + case '\u0140': // ŀ [LATIN SMALL LETTER L WITH MIDDLE DOT] + fallthrough + case '\u0142': // ł [LATIN SMALL LETTER L WITH STROKE] + fallthrough + case '\u019A': // ƚ [LATIN SMALL LETTER L WITH BAR] + fallthrough + case '\u0234': // ȴ [LATIN SMALL LETTER L WITH CURL] + fallthrough + case '\u026B': // ɫ [LATIN SMALL LETTER L WITH MIDDLE TILDE] + fallthrough + case '\u026C': // ɬ [LATIN SMALL LETTER L WITH BELT] + fallthrough + case '\u026D': // ɭ [LATIN SMALL LETTER L WITH RETROFLEX HOOK] + fallthrough + case '\u1D85': // ᶅ [LATIN SMALL LETTER L WITH PALATAL HOOK] + fallthrough + case '\u1E37': // ḷ [LATIN SMALL LETTER L WITH DOT BELOW] + fallthrough + case '\u1E39': // ḹ [LATIN SMALL LETTER L WITH DOT BELOW AND MACRON] + fallthrough + case '\u1E3B': // ḻ [LATIN SMALL LETTER L WITH LINE BELOW] + fallthrough + case '\u1E3D': // ḽ [LATIN SMALL LETTER L WITH CIRCUMFLEX BELOW] + fallthrough + case '\u24DB': // ⓛ [CIRCLED LATIN SMALL LETTER L] + fallthrough + case '\u2C61': // ⱡ [LATIN SMALL LETTER L WITH DOUBLE BAR] + fallthrough + case '\uA747': // ꝇ [LATIN SMALL LETTER BROKEN L] + fallthrough + case '\uA749': // ꝉ [LATIN SMALL LETTER L WITH HIGH STROKE] + fallthrough + case '\uA781': // ꞁ [LATIN SMALL LETTER TURNED L] + fallthrough + case '\uFF4C': // ｌ [FULLWIDTH LATIN SMALL LETTER L] + output[outputPos] = 'l' + outputPos++ + + case '\u01C7': // LJ [LATIN CAPITAL LETTER LJ] + output = output[:(len(output) + 1)] + output[outputPos] = 'L' + outputPos++ + output[outputPos] = 'J' + outputPos++ + + case '\u1EFA': // Ỻ [LATIN CAPITAL LETTER MIDDLE-WELSH LL] + output = output[:(len(output) + 1)] + output[outputPos] = 'L' + outputPos++ + output[outputPos] = 'L' + outputPos++ + + case '\u01C8': // Lj [LATIN CAPITAL LETTER L WITH SMALL LETTER J] + output = output[:(len(output) + 1)] + output[outputPos] = 'L' + outputPos++ + output[outputPos] = 'j' + outputPos++ + + case '\u24A7': // ⒧ [PARENTHESIZED LATIN SMALL LETTER L] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'l' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u01C9': // lj [LATIN SMALL LETTER LJ] + output = output[:(len(output) + 1)] + output[outputPos] = 'l' + outputPos++ + output[outputPos] = 'j' + outputPos++ + + case '\u1EFB': // ỻ [LATIN SMALL LETTER MIDDLE-WELSH LL] + output = output[:(len(output) + 1)] + output[outputPos] = 'l' + outputPos++ + output[outputPos] = 'l' + outputPos++ + + case '\u02AA': // ʪ [LATIN SMALL LETTER LS DIGRAPH] + output = output[:(len(output) + 1)] + output[outputPos] = 'l' + outputPos++ + output[outputPos] = 's' + outputPos++ + + case '\u02AB': // ʫ [LATIN SMALL LETTER LZ DIGRAPH] + output = output[:(len(output) + 1)] + output[outputPos] = 'l' + outputPos++ + output[outputPos] = 'z' + outputPos++ + + case '\u019C': // Ɯ [LATIN CAPITAL LETTER TURNED M] + fallthrough + case '\u1D0D': // ᴍ [LATIN LETTER SMALL CAPITAL M] + fallthrough + case '\u1E3E': // Ḿ [LATIN CAPITAL LETTER M WITH ACUTE] + fallthrough + case '\u1E40': // Ṁ [LATIN CAPITAL LETTER M WITH DOT ABOVE] + fallthrough + case '\u1E42': // Ṃ [LATIN CAPITAL LETTER M WITH DOT BELOW] + fallthrough + case '\u24C2': // Ⓜ [CIRCLED LATIN CAPITAL LETTER M] + fallthrough + case '\u2C6E': // Ɱ [LATIN CAPITAL LETTER M WITH HOOK] + fallthrough + case '\uA7FD': // ꟽ [LATIN EPIGRAPHIC LETTER INVERTED M] + fallthrough + case '\uA7FF': // ꟿ [LATIN EPIGRAPHIC LETTER ARCHAIC M] + fallthrough + case '\uFF2D': // Ｍ [FULLWIDTH LATIN CAPITAL LETTER M] + output[outputPos] = 'M' + outputPos++ + + case '\u026F': // ɯ [LATIN SMALL LETTER TURNED M] + fallthrough + case '\u0270': // ɰ [LATIN SMALL LETTER TURNED M WITH LONG LEG] + fallthrough + case '\u0271': // ɱ [LATIN SMALL LETTER M WITH HOOK] + fallthrough + case '\u1D6F': // ᵯ [LATIN SMALL LETTER M WITH MIDDLE TILDE] + fallthrough + case '\u1D86': // ᶆ [LATIN SMALL LETTER M WITH PALATAL HOOK] + fallthrough + case '\u1E3F': // ḿ [LATIN SMALL LETTER M WITH ACUTE] + fallthrough + case '\u1E41': // ṁ [LATIN SMALL LETTER M WITH DOT ABOVE] + fallthrough + case '\u1E43': // ṃ [LATIN SMALL LETTER M WITH DOT BELOW] + fallthrough + case '\u24DC': // ⓜ [CIRCLED LATIN SMALL LETTER M] + fallthrough + case '\uFF4D': // ｍ [FULLWIDTH LATIN SMALL LETTER M] + output[outputPos] = 'm' + outputPos++ + + case '\u24A8': // ⒨ [PARENTHESIZED LATIN SMALL LETTER M] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'm' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u00D1': // Ñ [LATIN CAPITAL LETTER N WITH TILDE] + fallthrough + case '\u0143': // Ń [LATIN CAPITAL LETTER N WITH ACUTE] + fallthrough + case '\u0145': // Ņ [LATIN CAPITAL LETTER N WITH CEDILLA] + fallthrough + case '\u0147': // Ň [LATIN CAPITAL LETTER N WITH CARON] + fallthrough + case '\u014A': // Ŋ http://en.wikipedia.org/wiki/Eng_(letter) [LATIN CAPITAL LETTER ENG] + fallthrough + case '\u019D': // Ɲ [LATIN CAPITAL LETTER N WITH LEFT HOOK] + fallthrough + case '\u01F8': // Ǹ [LATIN CAPITAL LETTER N WITH GRAVE] + fallthrough + case '\u0220': // Ƞ [LATIN CAPITAL LETTER N WITH LONG RIGHT LEG] + fallthrough + case '\u0274': // ɴ [LATIN LETTER SMALL CAPITAL N] + fallthrough + case '\u1D0E': // ᴎ [LATIN LETTER SMALL CAPITAL REVERSED N] + fallthrough + case '\u1E44': // Ṅ [LATIN CAPITAL LETTER N WITH DOT ABOVE] + fallthrough + case '\u1E46': // Ṇ [LATIN CAPITAL LETTER N WITH DOT BELOW] + fallthrough + case '\u1E48': // Ṉ [LATIN CAPITAL LETTER N WITH LINE BELOW] + fallthrough + case '\u1E4A': // Ṋ [LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW] + fallthrough + case '\u24C3': // Ⓝ [CIRCLED LATIN CAPITAL LETTER N] + fallthrough + case '\uFF2E': // Ｎ [FULLWIDTH LATIN CAPITAL LETTER N] + output[outputPos] = 'N' + outputPos++ + + case '\u00F1': // ñ [LATIN SMALL LETTER N WITH TILDE] + fallthrough + case '\u0144': // ń [LATIN SMALL LETTER N WITH ACUTE] + fallthrough + case '\u0146': // ņ [LATIN SMALL LETTER N WITH CEDILLA] + fallthrough + case '\u0148': // ň [LATIN SMALL LETTER N WITH CARON] + fallthrough + case '\u0149': // ʼn [LATIN SMALL LETTER N PRECEDED BY APOSTROPHE] + fallthrough + case '\u014B': // ŋ http://en.wikipedia.org/wiki/Eng_(letter) [LATIN SMALL LETTER ENG] + fallthrough + case '\u019E': // ƞ [LATIN SMALL LETTER N WITH LONG RIGHT LEG] + fallthrough + case '\u01F9': // ǹ [LATIN SMALL LETTER N WITH GRAVE] + fallthrough + case '\u0235': // ȵ [LATIN SMALL LETTER N WITH CURL] + fallthrough + case '\u0272': // ɲ [LATIN SMALL LETTER N WITH LEFT HOOK] + fallthrough + case '\u0273': // ɳ [LATIN SMALL LETTER N WITH RETROFLEX HOOK] + fallthrough + case '\u1D70': // ᵰ [LATIN SMALL LETTER N WITH MIDDLE TILDE] + fallthrough + case '\u1D87': // ᶇ [LATIN SMALL LETTER N WITH PALATAL HOOK] + fallthrough + case '\u1E45': // ṅ [LATIN SMALL LETTER N WITH DOT ABOVE] + fallthrough + case '\u1E47': // ṇ [LATIN SMALL LETTER N WITH DOT BELOW] + fallthrough + case '\u1E49': // ṉ [LATIN SMALL LETTER N WITH LINE BELOW] + fallthrough + case '\u1E4B': // ṋ [LATIN SMALL LETTER N WITH CIRCUMFLEX BELOW] + fallthrough + case '\u207F': // ⁿ [SUPERSCRIPT LATIN SMALL LETTER N] + fallthrough + case '\u24DD': // ⓝ [CIRCLED LATIN SMALL LETTER N] + fallthrough + case '\uFF4E': // ｎ [FULLWIDTH LATIN SMALL LETTER N] + output[outputPos] = 'n' + outputPos++ + + case '\u01CA': // NJ [LATIN CAPITAL LETTER NJ] + output = output[:(len(output) + 1)] + output[outputPos] = 'N' + outputPos++ + output[outputPos] = 'J' + outputPos++ + + case '\u01CB': // Nj [LATIN CAPITAL LETTER N WITH SMALL LETTER J] + output = output[:(len(output) + 1)] + output[outputPos] = 'N' + outputPos++ + output[outputPos] = 'j' + outputPos++ + + case '\u24A9': // ⒩ [PARENTHESIZED LATIN SMALL LETTER N] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'n' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u01CC': // nj [LATIN SMALL LETTER NJ] + output = output[:(len(output) + 1)] + output[outputPos] = 'n' + outputPos++ + output[outputPos] = 'j' + outputPos++ + + case '\u00D2': // Ò [LATIN CAPITAL LETTER O WITH GRAVE] + fallthrough + case '\u00D3': // Ó [LATIN CAPITAL LETTER O WITH ACUTE] + fallthrough + case '\u00D4': // Ô [LATIN CAPITAL LETTER O WITH CIRCUMFLEX] + fallthrough + case '\u00D5': // Õ [LATIN CAPITAL LETTER O WITH TILDE] + fallthrough + case '\u00D6': // Ö [LATIN CAPITAL LETTER O WITH DIAERESIS] + fallthrough + case '\u00D8': // Ø [LATIN CAPITAL LETTER O WITH STROKE] + fallthrough + case '\u014C': // Ō [LATIN CAPITAL LETTER O WITH MACRON] + fallthrough + case '\u014E': // Ŏ [LATIN CAPITAL LETTER O WITH BREVE] + fallthrough + case '\u0150': // Ő [LATIN CAPITAL LETTER O WITH DOUBLE ACUTE] + fallthrough + case '\u0186': // Ɔ [LATIN CAPITAL LETTER OPEN O] + fallthrough + case '\u019F': // Ɵ [LATIN CAPITAL LETTER O WITH MIDDLE TILDE] + fallthrough + case '\u01A0': // Ơ [LATIN CAPITAL LETTER O WITH HORN] + fallthrough + case '\u01D1': // Ǒ [LATIN CAPITAL LETTER O WITH CARON] + fallthrough + case '\u01EA': // Ǫ [LATIN CAPITAL LETTER O WITH OGONEK] + fallthrough + case '\u01EC': // Ǭ [LATIN CAPITAL LETTER O WITH OGONEK AND MACRON] + fallthrough + case '\u01FE': // Ǿ [LATIN CAPITAL LETTER O WITH STROKE AND ACUTE] + fallthrough + case '\u020C': // Ȍ [LATIN CAPITAL LETTER O WITH DOUBLE GRAVE] + fallthrough + case '\u020E': // Ȏ [LATIN CAPITAL LETTER O WITH INVERTED BREVE] + fallthrough + case '\u022A': // Ȫ [LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON] + fallthrough + case '\u022C': // Ȭ [LATIN CAPITAL LETTER O WITH TILDE AND MACRON] + fallthrough + case '\u022E': // Ȯ [LATIN CAPITAL LETTER O WITH DOT ABOVE] + fallthrough + case '\u0230': // Ȱ [LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON] + fallthrough + case '\u1D0F': // ᴏ [LATIN LETTER SMALL CAPITAL O] + fallthrough + case '\u1D10': // ᴐ [LATIN LETTER SMALL CAPITAL OPEN O] + fallthrough + case '\u1E4C': // Ṍ [LATIN CAPITAL LETTER O WITH TILDE AND ACUTE] + fallthrough + case '\u1E4E': // Ṏ [LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS] + fallthrough + case '\u1E50': // Ṑ [LATIN CAPITAL LETTER O WITH MACRON AND GRAVE] + fallthrough + case '\u1E52': // Ṓ [LATIN CAPITAL LETTER O WITH MACRON AND ACUTE] + fallthrough + case '\u1ECC': // Ọ [LATIN CAPITAL LETTER O WITH DOT BELOW] + fallthrough + case '\u1ECE': // Ỏ [LATIN CAPITAL LETTER O WITH HOOK ABOVE] + fallthrough + case '\u1ED0': // Ố [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE] + fallthrough + case '\u1ED2': // Ồ [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE] + fallthrough + case '\u1ED4': // Ổ [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE] + fallthrough + case '\u1ED6': // Ỗ [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE] + fallthrough + case '\u1ED8': // Ộ [LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW] + fallthrough + case '\u1EDA': // Ớ [LATIN CAPITAL LETTER O WITH HORN AND ACUTE] + fallthrough + case '\u1EDC': // Ờ [LATIN CAPITAL LETTER O WITH HORN AND GRAVE] + fallthrough + case '\u1EDE': // Ở [LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE] + fallthrough + case '\u1EE0': // Ỡ [LATIN CAPITAL LETTER O WITH HORN AND TILDE] + fallthrough + case '\u1EE2': // Ợ [LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW] + fallthrough + case '\u24C4': // Ⓞ [CIRCLED LATIN CAPITAL LETTER O] + fallthrough + case '\uA74A': // Ꝋ [LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY] + fallthrough + case '\uA74C': // Ꝍ [LATIN CAPITAL LETTER O WITH LOOP] + fallthrough + case '\uFF2F': // Ｏ [FULLWIDTH LATIN CAPITAL LETTER O] + output[outputPos] = 'O' + outputPos++ + + case '\u00F2': // ò [LATIN SMALL LETTER O WITH GRAVE] + fallthrough + case '\u00F3': // ó [LATIN SMALL LETTER O WITH ACUTE] + fallthrough + case '\u00F4': // ô [LATIN SMALL LETTER O WITH CIRCUMFLEX] + fallthrough + case '\u00F5': // õ [LATIN SMALL LETTER O WITH TILDE] + fallthrough + case '\u00F6': // ö [LATIN SMALL LETTER O WITH DIAERESIS] + fallthrough + case '\u00F8': // ø [LATIN SMALL LETTER O WITH STROKE] + fallthrough + case '\u014D': // ō [LATIN SMALL LETTER O WITH MACRON] + fallthrough + case '\u014F': // ŏ [LATIN SMALL LETTER O WITH BREVE] + fallthrough + case '\u0151': // ő [LATIN SMALL LETTER O WITH DOUBLE ACUTE] + fallthrough + case '\u01A1': // ơ [LATIN SMALL LETTER O WITH HORN] + fallthrough + case '\u01D2': // ǒ [LATIN SMALL LETTER O WITH CARON] + fallthrough + case '\u01EB': // ǫ [LATIN SMALL LETTER O WITH OGONEK] + fallthrough + case '\u01ED': // ǭ [LATIN SMALL LETTER O WITH OGONEK AND MACRON] + fallthrough + case '\u01FF': // ǿ [LATIN SMALL LETTER O WITH STROKE AND ACUTE] + fallthrough + case '\u020D': // ȍ [LATIN SMALL LETTER O WITH DOUBLE GRAVE] + fallthrough + case '\u020F': // ȏ [LATIN SMALL LETTER O WITH INVERTED BREVE] + fallthrough + case '\u022B': // ȫ [LATIN SMALL LETTER O WITH DIAERESIS AND MACRON] + fallthrough + case '\u022D': // ȭ [LATIN SMALL LETTER O WITH TILDE AND MACRON] + fallthrough + case '\u022F': // ȯ [LATIN SMALL LETTER O WITH DOT ABOVE] + fallthrough + case '\u0231': // ȱ [LATIN SMALL LETTER O WITH DOT ABOVE AND MACRON] + fallthrough + case '\u0254': // ɔ [LATIN SMALL LETTER OPEN O] + fallthrough + case '\u0275': // ɵ [LATIN SMALL LETTER BARRED O] + fallthrough + case '\u1D16': // ᴖ [LATIN SMALL LETTER TOP HALF O] + fallthrough + case '\u1D17': // ᴗ [LATIN SMALL LETTER BOTTOM HALF O] + fallthrough + case '\u1D97': // ᶗ [LATIN SMALL LETTER OPEN O WITH RETROFLEX HOOK] + fallthrough + case '\u1E4D': // ṍ [LATIN SMALL LETTER O WITH TILDE AND ACUTE] + fallthrough + case '\u1E4F': // ṏ [LATIN SMALL LETTER O WITH TILDE AND DIAERESIS] + fallthrough + case '\u1E51': // ṑ [LATIN SMALL LETTER O WITH MACRON AND GRAVE] + fallthrough + case '\u1E53': // ṓ [LATIN SMALL LETTER O WITH MACRON AND ACUTE] + fallthrough + case '\u1ECD': // ọ [LATIN SMALL LETTER O WITH DOT BELOW] + fallthrough + case '\u1ECF': // ỏ [LATIN SMALL LETTER O WITH HOOK ABOVE] + fallthrough + case '\u1ED1': // ố [LATIN SMALL LETTER O WITH CIRCUMFLEX AND ACUTE] + fallthrough + case '\u1ED3': // ồ [LATIN SMALL LETTER O WITH CIRCUMFLEX AND GRAVE] + fallthrough + case '\u1ED5': // ổ [LATIN SMALL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE] + fallthrough + case '\u1ED7': // ỗ [LATIN SMALL LETTER O WITH CIRCUMFLEX AND TILDE] + fallthrough + case '\u1ED9': // ộ [LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW] + fallthrough + case '\u1EDB': // ớ [LATIN SMALL LETTER O WITH HORN AND ACUTE] + fallthrough + case '\u1EDD': // ờ [LATIN SMALL LETTER O WITH HORN AND GRAVE] + fallthrough + case '\u1EDF': // ở [LATIN SMALL LETTER O WITH HORN AND HOOK ABOVE] + fallthrough + case '\u1EE1': // ỡ [LATIN SMALL LETTER O WITH HORN AND TILDE] + fallthrough + case '\u1EE3': // ợ [LATIN SMALL LETTER O WITH HORN AND DOT BELOW] + fallthrough + case '\u2092': // ₒ [LATIN SUBSCRIPT SMALL LETTER O] + fallthrough + case '\u24DE': // ⓞ [CIRCLED LATIN SMALL LETTER O] + fallthrough + case '\u2C7A': // ⱺ [LATIN SMALL LETTER O WITH LOW RING INSIDE] + fallthrough + case '\uA74B': // ꝋ [LATIN SMALL LETTER O WITH LONG STROKE OVERLAY] + fallthrough + case '\uA74D': // ꝍ [LATIN SMALL LETTER O WITH LOOP] + fallthrough + case '\uFF4F': // ｏ [FULLWIDTH LATIN SMALL LETTER O] + output[outputPos] = 'o' + outputPos++ + + case '\u0152': // Œ [LATIN CAPITAL LIGATURE OE] + fallthrough + case '\u0276': // ɶ [LATIN LETTER SMALL CAPITAL OE] + output = output[:(len(output) + 1)] + output[outputPos] = 'O' + outputPos++ + output[outputPos] = 'E' + outputPos++ + + case '\uA74E': // Ꝏ [LATIN CAPITAL LETTER OO] + output = output[:(len(output) + 1)] + output[outputPos] = 'O' + outputPos++ + output[outputPos] = 'O' + outputPos++ + + case '\u0222': // Ȣ http://en.wikipedia.org/wiki/OU [LATIN CAPITAL LETTER OU] + fallthrough + case '\u1D15': // ᴕ [LATIN LETTER SMALL CAPITAL OU] + output = output[:(len(output) + 1)] + output[outputPos] = 'O' + outputPos++ + output[outputPos] = 'U' + outputPos++ + + case '\u24AA': // ⒪ [PARENTHESIZED LATIN SMALL LETTER O] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'o' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u0153': // œ [LATIN SMALL LIGATURE OE] + fallthrough + case '\u1D14': // ᴔ [LATIN SMALL LETTER TURNED OE] + output = output[:(len(output) + 1)] + output[outputPos] = 'o' + outputPos++ + output[outputPos] = 'e' + outputPos++ + + case '\uA74F': // ꝏ [LATIN SMALL LETTER OO] + output = output[:(len(output) + 1)] + output[outputPos] = 'o' + outputPos++ + output[outputPos] = 'o' + outputPos++ + + case '\u0223': // ȣ http://en.wikipedia.org/wiki/OU [LATIN SMALL LETTER OU] + output = output[:(len(output) + 1)] + output[outputPos] = 'o' + outputPos++ + output[outputPos] = 'u' + outputPos++ + + case '\u01A4': // Ƥ [LATIN CAPITAL LETTER P WITH HOOK] + fallthrough + case '\u1D18': // ᴘ [LATIN LETTER SMALL CAPITAL P] + fallthrough + case '\u1E54': // Ṕ [LATIN CAPITAL LETTER P WITH ACUTE] + fallthrough + case '\u1E56': // Ṗ [LATIN CAPITAL LETTER P WITH DOT ABOVE] + fallthrough + case '\u24C5': // Ⓟ [CIRCLED LATIN CAPITAL LETTER P] + fallthrough + case '\u2C63': // Ᵽ [LATIN CAPITAL LETTER P WITH STROKE] + fallthrough + case '\uA750': // Ꝑ [LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER] + fallthrough + case '\uA752': // Ꝓ [LATIN CAPITAL LETTER P WITH FLOURISH] + fallthrough + case '\uA754': // Ꝕ [LATIN CAPITAL LETTER P WITH SQUIRREL TAIL] + fallthrough + case '\uFF30': // Ｐ [FULLWIDTH LATIN CAPITAL LETTER P] + output[outputPos] = 'P' + outputPos++ + + case '\u01A5': // ƥ [LATIN SMALL LETTER P WITH HOOK] + fallthrough + case '\u1D71': // ᵱ [LATIN SMALL LETTER P WITH MIDDLE TILDE] + fallthrough + case '\u1D7D': // ᵽ [LATIN SMALL LETTER P WITH STROKE] + fallthrough + case '\u1D88': // ᶈ [LATIN SMALL LETTER P WITH PALATAL HOOK] + fallthrough + case '\u1E55': // ṕ [LATIN SMALL LETTER P WITH ACUTE] + fallthrough + case '\u1E57': // ṗ [LATIN SMALL LETTER P WITH DOT ABOVE] + fallthrough + case '\u24DF': // ⓟ [CIRCLED LATIN SMALL LETTER P] + fallthrough + case '\uA751': // ꝑ [LATIN SMALL LETTER P WITH STROKE THROUGH DESCENDER] + fallthrough + case '\uA753': // ꝓ [LATIN SMALL LETTER P WITH FLOURISH] + fallthrough + case '\uA755': // ꝕ [LATIN SMALL LETTER P WITH SQUIRREL TAIL] + fallthrough + case '\uA7FC': // ꟼ [LATIN EPIGRAPHIC LETTER REVERSED P] + fallthrough + case '\uFF50': // ｐ [FULLWIDTH LATIN SMALL LETTER P] + output[outputPos] = 'p' + outputPos++ + + case '\u24AB': // ⒫ [PARENTHESIZED LATIN SMALL LETTER P] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'p' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u024A': // Ɋ [LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL] + fallthrough + case '\u24C6': // Ⓠ [CIRCLED LATIN CAPITAL LETTER Q] + fallthrough + case '\uA756': // Ꝗ [LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER] + fallthrough + case '\uA758': // Ꝙ [LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE] + fallthrough + case '\uFF31': // Ｑ [FULLWIDTH LATIN CAPITAL LETTER Q] + output[outputPos] = 'Q' + outputPos++ + + case '\u0138': // ĸ http://en.wikipedia.org/wiki/Kra_(letter) [LATIN SMALL LETTER KRA] + fallthrough + case '\u024B': // ɋ [LATIN SMALL LETTER Q WITH HOOK TAIL] + fallthrough + case '\u02A0': // ʠ [LATIN SMALL LETTER Q WITH HOOK] + fallthrough + case '\u24E0': // ⓠ [CIRCLED LATIN SMALL LETTER Q] + fallthrough + case '\uA757': // ꝗ [LATIN SMALL LETTER Q WITH STROKE THROUGH DESCENDER] + fallthrough + case '\uA759': // ꝙ [LATIN SMALL LETTER Q WITH DIAGONAL STROKE] + fallthrough + case '\uFF51': // ｑ [FULLWIDTH LATIN SMALL LETTER Q] + output[outputPos] = 'q' + outputPos++ + + case '\u24AC': // ⒬ [PARENTHESIZED LATIN SMALL LETTER Q] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'q' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u0239': // ȹ [LATIN SMALL LETTER QP DIGRAPH] + output = output[:(len(output) + 1)] + output[outputPos] = 'q' + outputPos++ + output[outputPos] = 'p' + outputPos++ + + case '\u0154': // Ŕ [LATIN CAPITAL LETTER R WITH ACUTE] + fallthrough + case '\u0156': // Ŗ [LATIN CAPITAL LETTER R WITH CEDILLA] + fallthrough + case '\u0158': // Ř [LATIN CAPITAL LETTER R WITH CARON] + fallthrough + case '\u0210': // Ȓ [LATIN CAPITAL LETTER R WITH DOUBLE GRAVE] + fallthrough + case '\u0212': // Ȓ [LATIN CAPITAL LETTER R WITH INVERTED BREVE] + fallthrough + case '\u024C': // Ɍ [LATIN CAPITAL LETTER R WITH STROKE] + fallthrough + case '\u0280': // ʀ [LATIN LETTER SMALL CAPITAL R] + fallthrough + case '\u0281': // ʁ [LATIN LETTER SMALL CAPITAL INVERTED R] + fallthrough + case '\u1D19': // ᴙ [LATIN LETTER SMALL CAPITAL REVERSED R] + fallthrough + case '\u1D1A': // ᴚ [LATIN LETTER SMALL CAPITAL TURNED R] + fallthrough + case '\u1E58': // Ṙ [LATIN CAPITAL LETTER R WITH DOT ABOVE] + fallthrough + case '\u1E5A': // Ṛ [LATIN CAPITAL LETTER R WITH DOT BELOW] + fallthrough + case '\u1E5C': // Ṝ [LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON] + fallthrough + case '\u1E5E': // Ṟ [LATIN CAPITAL LETTER R WITH LINE BELOW] + fallthrough + case '\u24C7': // Ⓡ [CIRCLED LATIN CAPITAL LETTER R] + fallthrough + case '\u2C64': // Ɽ [LATIN CAPITAL LETTER R WITH TAIL] + fallthrough + case '\uA75A': // Ꝛ [LATIN CAPITAL LETTER R ROTUNDA] + fallthrough + case '\uA782': // Ꞃ [LATIN CAPITAL LETTER INSULAR R] + fallthrough + case '\uFF32': // Ｒ [FULLWIDTH LATIN CAPITAL LETTER R] + output[outputPos] = 'R' + outputPos++ + + case '\u0155': // ŕ [LATIN SMALL LETTER R WITH ACUTE] + fallthrough + case '\u0157': // ŗ [LATIN SMALL LETTER R WITH CEDILLA] + fallthrough + case '\u0159': // ř [LATIN SMALL LETTER R WITH CARON] + fallthrough + case '\u0211': // ȑ [LATIN SMALL LETTER R WITH DOUBLE GRAVE] + fallthrough + case '\u0213': // ȓ [LATIN SMALL LETTER R WITH INVERTED BREVE] + fallthrough + case '\u024D': // ɍ [LATIN SMALL LETTER R WITH STROKE] + fallthrough + case '\u027C': // ɼ [LATIN SMALL LETTER R WITH LONG LEG] + fallthrough + case '\u027D': // ɽ [LATIN SMALL LETTER R WITH TAIL] + fallthrough + case '\u027E': // ɾ [LATIN SMALL LETTER R WITH FISHHOOK] + fallthrough + case '\u027F': // ɿ [LATIN SMALL LETTER REVERSED R WITH FISHHOOK] + fallthrough + case '\u1D63': // ᵣ [LATIN SUBSCRIPT SMALL LETTER R] + fallthrough + case '\u1D72': // ᵲ [LATIN SMALL LETTER R WITH MIDDLE TILDE] + fallthrough + case '\u1D73': // ᵳ [LATIN SMALL LETTER R WITH FISHHOOK AND MIDDLE TILDE] + fallthrough + case '\u1D89': // ᶉ [LATIN SMALL LETTER R WITH PALATAL HOOK] + fallthrough + case '\u1E59': // ṙ [LATIN SMALL LETTER R WITH DOT ABOVE] + fallthrough + case '\u1E5B': // ṛ [LATIN SMALL LETTER R WITH DOT BELOW] + fallthrough + case '\u1E5D': // ṝ [LATIN SMALL LETTER R WITH DOT BELOW AND MACRON] + fallthrough + case '\u1E5F': // ṟ [LATIN SMALL LETTER R WITH LINE BELOW] + fallthrough + case '\u24E1': // ⓡ [CIRCLED LATIN SMALL LETTER R] + fallthrough + case '\uA75B': // ꝛ [LATIN SMALL LETTER R ROTUNDA] + fallthrough + case '\uA783': // ꞃ [LATIN SMALL LETTER INSULAR R] + fallthrough + case '\uFF52': // ｒ [FULLWIDTH LATIN SMALL LETTER R] + output[outputPos] = 'r' + outputPos++ + + case '\u24AD': // ⒭ [PARENTHESIZED LATIN SMALL LETTER R] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'r' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u015A': // Ś [LATIN CAPITAL LETTER S WITH ACUTE] + fallthrough + case '\u015C': // Ŝ [LATIN CAPITAL LETTER S WITH CIRCUMFLEX] + fallthrough + case '\u015E': // Ş [LATIN CAPITAL LETTER S WITH CEDILLA] + fallthrough + case '\u0160': // Š [LATIN CAPITAL LETTER S WITH CARON] + fallthrough + case '\u0218': // Ș [LATIN CAPITAL LETTER S WITH COMMA BELOW] + fallthrough + case '\u1E60': // Ṡ [LATIN CAPITAL LETTER S WITH DOT ABOVE] + fallthrough + case '\u1E62': // Ṣ [LATIN CAPITAL LETTER S WITH DOT BELOW] + fallthrough + case '\u1E64': // Ṥ [LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE] + fallthrough + case '\u1E66': // Ṧ [LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE] + fallthrough + case '\u1E68': // Ṩ [LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE] + fallthrough + case '\u24C8': // Ⓢ [CIRCLED LATIN CAPITAL LETTER S] + fallthrough + case '\uA731': // ꜱ [LATIN LETTER SMALL CAPITAL S] + fallthrough + case '\uA785': // ꞅ [LATIN SMALL LETTER INSULAR S] + fallthrough + case '\uFF33': // Ｓ [FULLWIDTH LATIN CAPITAL LETTER S] + output[outputPos] = 'S' + outputPos++ + + case '\u015B': // ś [LATIN SMALL LETTER S WITH ACUTE] + fallthrough + case '\u015D': // ŝ [LATIN SMALL LETTER S WITH CIRCUMFLEX] + fallthrough + case '\u015F': // ş [LATIN SMALL LETTER S WITH CEDILLA] + fallthrough + case '\u0161': // š [LATIN SMALL LETTER S WITH CARON] + fallthrough + case '\u017F': // ſ http://en.wikipedia.org/wiki/Long_S [LATIN SMALL LETTER LONG S] + fallthrough + case '\u0219': // ș [LATIN SMALL LETTER S WITH COMMA BELOW] + fallthrough + case '\u023F': // ȿ [LATIN SMALL LETTER S WITH SWASH TAIL] + fallthrough + case '\u0282': // ʂ [LATIN SMALL LETTER S WITH HOOK] + fallthrough + case '\u1D74': // ᵴ [LATIN SMALL LETTER S WITH MIDDLE TILDE] + fallthrough + case '\u1D8A': // ᶊ [LATIN SMALL LETTER S WITH PALATAL HOOK] + fallthrough + case '\u1E61': // ṡ [LATIN SMALL LETTER S WITH DOT ABOVE] + fallthrough + case '\u1E63': // ṣ [LATIN SMALL LETTER S WITH DOT BELOW] + fallthrough + case '\u1E65': // ṥ [LATIN SMALL LETTER S WITH ACUTE AND DOT ABOVE] + fallthrough + case '\u1E67': // ṧ [LATIN SMALL LETTER S WITH CARON AND DOT ABOVE] + fallthrough + case '\u1E69': // ṩ [LATIN SMALL LETTER S WITH DOT BELOW AND DOT ABOVE] + fallthrough + case '\u1E9C': // ẜ [LATIN SMALL LETTER LONG S WITH DIAGONAL STROKE] + fallthrough + case '\u1E9D': // ẝ [LATIN SMALL LETTER LONG S WITH HIGH STROKE] + fallthrough + case '\u24E2': // ⓢ [CIRCLED LATIN SMALL LETTER S] + fallthrough + case '\uA784': // Ꞅ [LATIN CAPITAL LETTER INSULAR S] + fallthrough + case '\uFF53': // ｓ [FULLWIDTH LATIN SMALL LETTER S] + output[outputPos] = 's' + outputPos++ + + case '\u1E9E': // ẞ [LATIN CAPITAL LETTER SHARP S] + output = output[:(len(output) + 1)] + output[outputPos] = 'S' + outputPos++ + output[outputPos] = 'S' + outputPos++ + + case '\u24AE': // ⒮ [PARENTHESIZED LATIN SMALL LETTER S] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 's' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u00DF': // ß [LATIN SMALL LETTER SHARP S] + output = output[:(len(output) + 1)] + output[outputPos] = 's' + outputPos++ + output[outputPos] = 's' + outputPos++ + + case '\uFB06': // st [LATIN SMALL LIGATURE ST] + output = output[:(len(output) + 1)] + output[outputPos] = 's' + outputPos++ + output[outputPos] = 't' + outputPos++ + + case '\u0162': // Ţ [LATIN CAPITAL LETTER T WITH CEDILLA] + fallthrough + case '\u0164': // Ť [LATIN CAPITAL LETTER T WITH CARON] + fallthrough + case '\u0166': // Ŧ [LATIN CAPITAL LETTER T WITH STROKE] + fallthrough + case '\u01AC': // Ƭ [LATIN CAPITAL LETTER T WITH HOOK] + fallthrough + case '\u01AE': // Ʈ [LATIN CAPITAL LETTER T WITH RETROFLEX HOOK] + fallthrough + case '\u021A': // Ț [LATIN CAPITAL LETTER T WITH COMMA BELOW] + fallthrough + case '\u023E': // Ⱦ [LATIN CAPITAL LETTER T WITH DIAGONAL STROKE] + fallthrough + case '\u1D1B': // ᴛ [LATIN LETTER SMALL CAPITAL T] + fallthrough + case '\u1E6A': // Ṫ [LATIN CAPITAL LETTER T WITH DOT ABOVE] + fallthrough + case '\u1E6C': // Ṭ [LATIN CAPITAL LETTER T WITH DOT BELOW] + fallthrough + case '\u1E6E': // Ṯ [LATIN CAPITAL LETTER T WITH LINE BELOW] + fallthrough + case '\u1E70': // Ṱ [LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW] + fallthrough + case '\u24C9': // Ⓣ [CIRCLED LATIN CAPITAL LETTER T] + fallthrough + case '\uA786': // Ꞇ [LATIN CAPITAL LETTER INSULAR T] + fallthrough + case '\uFF34': // Ｔ [FULLWIDTH LATIN CAPITAL LETTER T] + output[outputPos] = 'T' + outputPos++ + + case '\u0163': // ţ [LATIN SMALL LETTER T WITH CEDILLA] + fallthrough + case '\u0165': // ť [LATIN SMALL LETTER T WITH CARON] + fallthrough + case '\u0167': // ŧ [LATIN SMALL LETTER T WITH STROKE] + fallthrough + case '\u01AB': // ƫ [LATIN SMALL LETTER T WITH PALATAL HOOK] + fallthrough + case '\u01AD': // ƭ [LATIN SMALL LETTER T WITH HOOK] + fallthrough + case '\u021B': // ț [LATIN SMALL LETTER T WITH COMMA BELOW] + fallthrough + case '\u0236': // ȶ [LATIN SMALL LETTER T WITH CURL] + fallthrough + case '\u0287': // ʇ [LATIN SMALL LETTER TURNED T] + fallthrough + case '\u0288': // ʈ [LATIN SMALL LETTER T WITH RETROFLEX HOOK] + fallthrough + case '\u1D75': // ᵵ [LATIN SMALL LETTER T WITH MIDDLE TILDE] + fallthrough + case '\u1E6B': // ṫ [LATIN SMALL LETTER T WITH DOT ABOVE] + fallthrough + case '\u1E6D': // ṭ [LATIN SMALL LETTER T WITH DOT BELOW] + fallthrough + case '\u1E6F': // ṯ [LATIN SMALL LETTER T WITH LINE BELOW] + fallthrough + case '\u1E71': // ṱ [LATIN SMALL LETTER T WITH CIRCUMFLEX BELOW] + fallthrough + case '\u1E97': // ẗ [LATIN SMALL LETTER T WITH DIAERESIS] + fallthrough + case '\u24E3': // ⓣ [CIRCLED LATIN SMALL LETTER T] + fallthrough + case '\u2C66': // ⱦ [LATIN SMALL LETTER T WITH DIAGONAL STROKE] + fallthrough + case '\uFF54': // ｔ [FULLWIDTH LATIN SMALL LETTER T] + output[outputPos] = 't' + outputPos++ + + case '\u00DE': // Þ [LATIN CAPITAL LETTER THORN] + fallthrough + case '\uA766': // Ꝧ [LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER] + output = output[:(len(output) + 1)] + output[outputPos] = 'T' + outputPos++ + output[outputPos] = 'H' + outputPos++ + + case '\uA728': // Ꜩ [LATIN CAPITAL LETTER TZ] + output = output[:(len(output) + 1)] + output[outputPos] = 'T' + outputPos++ + output[outputPos] = 'Z' + outputPos++ + + case '\u24AF': // ⒯ [PARENTHESIZED LATIN SMALL LETTER T] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 't' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u02A8': // ʨ [LATIN SMALL LETTER TC DIGRAPH WITH CURL] + output = output[:(len(output) + 1)] + output[outputPos] = 't' + outputPos++ + output[outputPos] = 'c' + outputPos++ + + case '\u00FE': // þ [LATIN SMALL LETTER THORN] + fallthrough + case '\u1D7A': // ᵺ [LATIN SMALL LETTER TH WITH STRIKETHROUGH] + fallthrough + case '\uA767': // ꝧ [LATIN SMALL LETTER THORN WITH STROKE THROUGH DESCENDER] + output = output[:(len(output) + 1)] + output[outputPos] = 't' + outputPos++ + output[outputPos] = 'h' + outputPos++ + + case '\u02A6': // ʦ [LATIN SMALL LETTER TS DIGRAPH] + output = output[:(len(output) + 1)] + output[outputPos] = 't' + outputPos++ + output[outputPos] = 's' + outputPos++ + + case '\uA729': // ꜩ [LATIN SMALL LETTER TZ] + output = output[:(len(output) + 1)] + output[outputPos] = 't' + outputPos++ + output[outputPos] = 'z' + outputPos++ + + case '\u00D9': // Ù [LATIN CAPITAL LETTER U WITH GRAVE] + fallthrough + case '\u00DA': // Ú [LATIN CAPITAL LETTER U WITH ACUTE] + fallthrough + case '\u00DB': // Û [LATIN CAPITAL LETTER U WITH CIRCUMFLEX] + fallthrough + case '\u00DC': // Ü [LATIN CAPITAL LETTER U WITH DIAERESIS] + fallthrough + case '\u0168': // Ũ [LATIN CAPITAL LETTER U WITH TILDE] + fallthrough + case '\u016A': // Ū [LATIN CAPITAL LETTER U WITH MACRON] + fallthrough + case '\u016C': // Ŭ [LATIN CAPITAL LETTER U WITH BREVE] + fallthrough + case '\u016E': // Ů [LATIN CAPITAL LETTER U WITH RING ABOVE] + fallthrough + case '\u0170': // Ű [LATIN CAPITAL LETTER U WITH DOUBLE ACUTE] + fallthrough + case '\u0172': // Ų [LATIN CAPITAL LETTER U WITH OGONEK] + fallthrough + case '\u01AF': // Ư [LATIN CAPITAL LETTER U WITH HORN] + fallthrough + case '\u01D3': // Ǔ [LATIN CAPITAL LETTER U WITH CARON] + fallthrough + case '\u01D5': // Ǖ [LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON] + fallthrough + case '\u01D7': // Ǘ [LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE] + fallthrough + case '\u01D9': // Ǚ [LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON] + fallthrough + case '\u01DB': // Ǜ [LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE] + fallthrough + case '\u0214': // Ȕ [LATIN CAPITAL LETTER U WITH DOUBLE GRAVE] + fallthrough + case '\u0216': // Ȗ [LATIN CAPITAL LETTER U WITH INVERTED BREVE] + fallthrough + case '\u0244': // Ʉ [LATIN CAPITAL LETTER U BAR] + fallthrough + case '\u1D1C': // ᴜ [LATIN LETTER SMALL CAPITAL U] + fallthrough + case '\u1D7E': // ᵾ [LATIN SMALL CAPITAL LETTER U WITH STROKE] + fallthrough + case '\u1E72': // Ṳ [LATIN CAPITAL LETTER U WITH DIAERESIS BELOW] + fallthrough + case '\u1E74': // Ṵ [LATIN CAPITAL LETTER U WITH TILDE BELOW] + fallthrough + case '\u1E76': // Ṷ [LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW] + fallthrough + case '\u1E78': // Ṹ [LATIN CAPITAL LETTER U WITH TILDE AND ACUTE] + fallthrough + case '\u1E7A': // Ṻ [LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS] + fallthrough + case '\u1EE4': // Ụ [LATIN CAPITAL LETTER U WITH DOT BELOW] + fallthrough + case '\u1EE6': // Ủ [LATIN CAPITAL LETTER U WITH HOOK ABOVE] + fallthrough + case '\u1EE8': // Ứ [LATIN CAPITAL LETTER U WITH HORN AND ACUTE] + fallthrough + case '\u1EEA': // Ừ [LATIN CAPITAL LETTER U WITH HORN AND GRAVE] + fallthrough + case '\u1EEC': // Ử [LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE] + fallthrough + case '\u1EEE': // Ữ [LATIN CAPITAL LETTER U WITH HORN AND TILDE] + fallthrough + case '\u1EF0': // Ự [LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW] + fallthrough + case '\u24CA': // Ⓤ [CIRCLED LATIN CAPITAL LETTER U] + fallthrough + case '\uFF35': // Ｕ [FULLWIDTH LATIN CAPITAL LETTER U] + output[outputPos] = 'U' + outputPos++ + + case '\u00F9': // ù [LATIN SMALL LETTER U WITH GRAVE] + fallthrough + case '\u00FA': // ú [LATIN SMALL LETTER U WITH ACUTE] + fallthrough + case '\u00FB': // û [LATIN SMALL LETTER U WITH CIRCUMFLEX] + fallthrough + case '\u00FC': // ü [LATIN SMALL LETTER U WITH DIAERESIS] + fallthrough + case '\u0169': // ũ [LATIN SMALL LETTER U WITH TILDE] + fallthrough + case '\u016B': // ū [LATIN SMALL LETTER U WITH MACRON] + fallthrough + case '\u016D': // ŭ [LATIN SMALL LETTER U WITH BREVE] + fallthrough + case '\u016F': // ů [LATIN SMALL LETTER U WITH RING ABOVE] + fallthrough + case '\u0171': // ű [LATIN SMALL LETTER U WITH DOUBLE ACUTE] + fallthrough + case '\u0173': // ų [LATIN SMALL LETTER U WITH OGONEK] + fallthrough + case '\u01B0': // ư [LATIN SMALL LETTER U WITH HORN] + fallthrough + case '\u01D4': // ǔ [LATIN SMALL LETTER U WITH CARON] + fallthrough + case '\u01D6': // ǖ [LATIN SMALL LETTER U WITH DIAERESIS AND MACRON] + fallthrough + case '\u01D8': // ǘ [LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE] + fallthrough + case '\u01DA': // ǚ [LATIN SMALL LETTER U WITH DIAERESIS AND CARON] + fallthrough + case '\u01DC': // ǜ [LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE] + fallthrough + case '\u0215': // ȕ [LATIN SMALL LETTER U WITH DOUBLE GRAVE] + fallthrough + case '\u0217': // ȗ [LATIN SMALL LETTER U WITH INVERTED BREVE] + fallthrough + case '\u0289': // ʉ [LATIN SMALL LETTER U BAR] + fallthrough + case '\u1D64': // ᵤ [LATIN SUBSCRIPT SMALL LETTER U] + fallthrough + case '\u1D99': // ᶙ [LATIN SMALL LETTER U WITH RETROFLEX HOOK] + fallthrough + case '\u1E73': // ṳ [LATIN SMALL LETTER U WITH DIAERESIS BELOW] + fallthrough + case '\u1E75': // ṵ [LATIN SMALL LETTER U WITH TILDE BELOW] + fallthrough + case '\u1E77': // ṷ [LATIN SMALL LETTER U WITH CIRCUMFLEX BELOW] + fallthrough + case '\u1E79': // ṹ [LATIN SMALL LETTER U WITH TILDE AND ACUTE] + fallthrough + case '\u1E7B': // ṻ [LATIN SMALL LETTER U WITH MACRON AND DIAERESIS] + fallthrough + case '\u1EE5': // ụ [LATIN SMALL LETTER U WITH DOT BELOW] + fallthrough + case '\u1EE7': // ủ [LATIN SMALL LETTER U WITH HOOK ABOVE] + fallthrough + case '\u1EE9': // ứ [LATIN SMALL LETTER U WITH HORN AND ACUTE] + fallthrough + case '\u1EEB': // ừ [LATIN SMALL LETTER U WITH HORN AND GRAVE] + fallthrough + case '\u1EED': // ử [LATIN SMALL LETTER U WITH HORN AND HOOK ABOVE] + fallthrough + case '\u1EEF': // ữ [LATIN SMALL LETTER U WITH HORN AND TILDE] + fallthrough + case '\u1EF1': // ự [LATIN SMALL LETTER U WITH HORN AND DOT BELOW] + fallthrough + case '\u24E4': // ⓤ [CIRCLED LATIN SMALL LETTER U] + fallthrough + case '\uFF55': // ｕ [FULLWIDTH LATIN SMALL LETTER U] + output[outputPos] = 'u' + outputPos++ + + case '\u24B0': // ⒰ [PARENTHESIZED LATIN SMALL LETTER U] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'u' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u1D6B': // ᵫ [LATIN SMALL LETTER UE] + output = output[:(len(output) + 1)] + output[outputPos] = 'u' + outputPos++ + output[outputPos] = 'e' + outputPos++ + + case '\u01B2': // Ʋ [LATIN CAPITAL LETTER V WITH HOOK] + fallthrough + case '\u0245': // Ʌ [LATIN CAPITAL LETTER TURNED V] + fallthrough + case '\u1D20': // ᴠ [LATIN LETTER SMALL CAPITAL V] + fallthrough + case '\u1E7C': // Ṽ [LATIN CAPITAL LETTER V WITH TILDE] + fallthrough + case '\u1E7E': // Ṿ [LATIN CAPITAL LETTER V WITH DOT BELOW] + fallthrough + case '\u1EFC': // Ỽ [LATIN CAPITAL LETTER MIDDLE-WELSH V] + fallthrough + case '\u24CB': // Ⓥ [CIRCLED LATIN CAPITAL LETTER V] + fallthrough + case '\uA75E': // Ꝟ [LATIN CAPITAL LETTER V WITH DIAGONAL STROKE] + fallthrough + case '\uA768': // Ꝩ [LATIN CAPITAL LETTER VEND] + fallthrough + case '\uFF36': // Ｖ [FULLWIDTH LATIN CAPITAL LETTER V] + output[outputPos] = 'V' + outputPos++ + + case '\u028B': // ʋ [LATIN SMALL LETTER V WITH HOOK] + fallthrough + case '\u028C': // ʌ [LATIN SMALL LETTER TURNED V] + fallthrough + case '\u1D65': // ᵥ [LATIN SUBSCRIPT SMALL LETTER V] + fallthrough + case '\u1D8C': // ᶌ [LATIN SMALL LETTER V WITH PALATAL HOOK] + fallthrough + case '\u1E7D': // ṽ [LATIN SMALL LETTER V WITH TILDE] + fallthrough + case '\u1E7F': // ṿ [LATIN SMALL LETTER V WITH DOT BELOW] + fallthrough + case '\u24E5': // ⓥ [CIRCLED LATIN SMALL LETTER V] + fallthrough + case '\u2C71': // ⱱ [LATIN SMALL LETTER V WITH RIGHT HOOK] + fallthrough + case '\u2C74': // ⱴ [LATIN SMALL LETTER V WITH CURL] + fallthrough + case '\uA75F': // ꝟ [LATIN SMALL LETTER V WITH DIAGONAL STROKE] + fallthrough + case '\uFF56': // ｖ [FULLWIDTH LATIN SMALL LETTER V] + output[outputPos] = 'v' + outputPos++ + + case '\uA760': // Ꝡ [LATIN CAPITAL LETTER VY] + output = output[:(len(output) + 1)] + output[outputPos] = 'V' + outputPos++ + output[outputPos] = 'Y' + outputPos++ + + case '\u24B1': // ⒱ [PARENTHESIZED LATIN SMALL LETTER V] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'v' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\uA761': // ꝡ [LATIN SMALL LETTER VY] + output = output[:(len(output) + 1)] + output[outputPos] = 'v' + outputPos++ + output[outputPos] = 'y' + outputPos++ + + case '\u0174': // Ŵ [LATIN CAPITAL LETTER W WITH CIRCUMFLEX] + fallthrough + case '\u01F7': // Ƿ http://en.wikipedia.org/wiki/Wynn [LATIN CAPITAL LETTER WYNN] + fallthrough + case '\u1D21': // ᴡ [LATIN LETTER SMALL CAPITAL W] + fallthrough + case '\u1E80': // Ẁ [LATIN CAPITAL LETTER W WITH GRAVE] + fallthrough + case '\u1E82': // Ẃ [LATIN CAPITAL LETTER W WITH ACUTE] + fallthrough + case '\u1E84': // Ẅ [LATIN CAPITAL LETTER W WITH DIAERESIS] + fallthrough + case '\u1E86': // Ẇ [LATIN CAPITAL LETTER W WITH DOT ABOVE] + fallthrough + case '\u1E88': // Ẉ [LATIN CAPITAL LETTER W WITH DOT BELOW] + fallthrough + case '\u24CC': // Ⓦ [CIRCLED LATIN CAPITAL LETTER W] + fallthrough + case '\u2C72': // Ⱳ [LATIN CAPITAL LETTER W WITH HOOK] + fallthrough + case '\uFF37': // Ｗ [FULLWIDTH LATIN CAPITAL LETTER W] + output[outputPos] = 'W' + outputPos++ + + case '\u0175': // ŵ [LATIN SMALL LETTER W WITH CIRCUMFLEX] + fallthrough + case '\u01BF': // ƿ http://en.wikipedia.org/wiki/Wynn [LATIN LETTER WYNN] + fallthrough + case '\u028D': // ʍ [LATIN SMALL LETTER TURNED W] + fallthrough + case '\u1E81': // ẁ [LATIN SMALL LETTER W WITH GRAVE] + fallthrough + case '\u1E83': // ẃ [LATIN SMALL LETTER W WITH ACUTE] + fallthrough + case '\u1E85': // ẅ [LATIN SMALL LETTER W WITH DIAERESIS] + fallthrough + case '\u1E87': // ẇ [LATIN SMALL LETTER W WITH DOT ABOVE] + fallthrough + case '\u1E89': // ẉ [LATIN SMALL LETTER W WITH DOT BELOW] + fallthrough + case '\u1E98': // ẘ [LATIN SMALL LETTER W WITH RING ABOVE] + fallthrough + case '\u24E6': // ⓦ [CIRCLED LATIN SMALL LETTER W] + fallthrough + case '\u2C73': // ⱳ [LATIN SMALL LETTER W WITH HOOK] + fallthrough + case '\uFF57': // ｗ [FULLWIDTH LATIN SMALL LETTER W] + output[outputPos] = 'w' + outputPos++ + + case '\u24B2': // ⒲ [PARENTHESIZED LATIN SMALL LETTER W] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'w' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u1E8A': // Ẋ [LATIN CAPITAL LETTER X WITH DOT ABOVE] + fallthrough + case '\u1E8C': // Ẍ [LATIN CAPITAL LETTER X WITH DIAERESIS] + fallthrough + case '\u24CD': // Ⓧ [CIRCLED LATIN CAPITAL LETTER X] + fallthrough + case '\uFF38': // Ｘ [FULLWIDTH LATIN CAPITAL LETTER X] + output[outputPos] = 'X' + outputPos++ + + case '\u1D8D': // ᶍ [LATIN SMALL LETTER X WITH PALATAL HOOK] + fallthrough + case '\u1E8B': // ẋ [LATIN SMALL LETTER X WITH DOT ABOVE] + fallthrough + case '\u1E8D': // ẍ [LATIN SMALL LETTER X WITH DIAERESIS] + fallthrough + case '\u2093': // ₓ [LATIN SUBSCRIPT SMALL LETTER X] + fallthrough + case '\u24E7': // ⓧ [CIRCLED LATIN SMALL LETTER X] + fallthrough + case '\uFF58': // ｘ [FULLWIDTH LATIN SMALL LETTER X] + output[outputPos] = 'x' + outputPos++ + + case '\u24B3': // ⒳ [PARENTHESIZED LATIN SMALL LETTER X] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'x' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u00DD': // Ý [LATIN CAPITAL LETTER Y WITH ACUTE] + fallthrough + case '\u0176': // Ŷ [LATIN CAPITAL LETTER Y WITH CIRCUMFLEX] + fallthrough + case '\u0178': // Ÿ [LATIN CAPITAL LETTER Y WITH DIAERESIS] + fallthrough + case '\u01B3': // Ƴ [LATIN CAPITAL LETTER Y WITH HOOK] + fallthrough + case '\u0232': // Ȳ [LATIN CAPITAL LETTER Y WITH MACRON] + fallthrough + case '\u024E': // Ɏ [LATIN CAPITAL LETTER Y WITH STROKE] + fallthrough + case '\u028F': // ʏ [LATIN LETTER SMALL CAPITAL Y] + fallthrough + case '\u1E8E': // Ẏ [LATIN CAPITAL LETTER Y WITH DOT ABOVE] + fallthrough + case '\u1EF2': // Ỳ [LATIN CAPITAL LETTER Y WITH GRAVE] + fallthrough + case '\u1EF4': // Ỵ [LATIN CAPITAL LETTER Y WITH DOT BELOW] + fallthrough + case '\u1EF6': // Ỷ [LATIN CAPITAL LETTER Y WITH HOOK ABOVE] + fallthrough + case '\u1EF8': // Ỹ [LATIN CAPITAL LETTER Y WITH TILDE] + fallthrough + case '\u1EFE': // Ỿ [LATIN CAPITAL LETTER Y WITH LOOP] + fallthrough + case '\u24CE': // Ⓨ [CIRCLED LATIN CAPITAL LETTER Y] + fallthrough + case '\uFF39': // Ｙ [FULLWIDTH LATIN CAPITAL LETTER Y] + output[outputPos] = 'Y' + outputPos++ + + case '\u00FD': // ý [LATIN SMALL LETTER Y WITH ACUTE] + fallthrough + case '\u00FF': // ÿ [LATIN SMALL LETTER Y WITH DIAERESIS] + fallthrough + case '\u0177': // ŷ [LATIN SMALL LETTER Y WITH CIRCUMFLEX] + fallthrough + case '\u01B4': // ƴ [LATIN SMALL LETTER Y WITH HOOK] + fallthrough + case '\u0233': // ȳ [LATIN SMALL LETTER Y WITH MACRON] + fallthrough + case '\u024F': // ɏ [LATIN SMALL LETTER Y WITH STROKE] + fallthrough + case '\u028E': // ʎ [LATIN SMALL LETTER TURNED Y] + fallthrough + case '\u1E8F': // ẏ [LATIN SMALL LETTER Y WITH DOT ABOVE] + fallthrough + case '\u1E99': // ẙ [LATIN SMALL LETTER Y WITH RING ABOVE] + fallthrough + case '\u1EF3': // ỳ [LATIN SMALL LETTER Y WITH GRAVE] + fallthrough + case '\u1EF5': // ỵ [LATIN SMALL LETTER Y WITH DOT BELOW] + fallthrough + case '\u1EF7': // ỷ [LATIN SMALL LETTER Y WITH HOOK ABOVE] + fallthrough + case '\u1EF9': // ỹ [LATIN SMALL LETTER Y WITH TILDE] + fallthrough + case '\u1EFF': // ỿ [LATIN SMALL LETTER Y WITH LOOP] + fallthrough + case '\u24E8': // ⓨ [CIRCLED LATIN SMALL LETTER Y] + fallthrough + case '\uFF59': // ｙ [FULLWIDTH LATIN SMALL LETTER Y] + output[outputPos] = 'y' + outputPos++ + + case '\u24B4': // ⒴ [PARENTHESIZED LATIN SMALL LETTER Y] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'y' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u0179': // Ź [LATIN CAPITAL LETTER Z WITH ACUTE] + fallthrough + case '\u017B': // Ż [LATIN CAPITAL LETTER Z WITH DOT ABOVE] + fallthrough + case '\u017D': // Ž [LATIN CAPITAL LETTER Z WITH CARON] + fallthrough + case '\u01B5': // Ƶ [LATIN CAPITAL LETTER Z WITH STROKE] + fallthrough + case '\u021C': // Ȝ http://en.wikipedia.org/wiki/Yogh [LATIN CAPITAL LETTER YOGH] + fallthrough + case '\u0224': // Ȥ [LATIN CAPITAL LETTER Z WITH HOOK] + fallthrough + case '\u1D22': // ᴢ [LATIN LETTER SMALL CAPITAL Z] + fallthrough + case '\u1E90': // Ẑ [LATIN CAPITAL LETTER Z WITH CIRCUMFLEX] + fallthrough + case '\u1E92': // Ẓ [LATIN CAPITAL LETTER Z WITH DOT BELOW] + fallthrough + case '\u1E94': // Ẕ [LATIN CAPITAL LETTER Z WITH LINE BELOW] + fallthrough + case '\u24CF': // Ⓩ [CIRCLED LATIN CAPITAL LETTER Z] + fallthrough + case '\u2C6B': // Ⱬ [LATIN CAPITAL LETTER Z WITH DESCENDER] + fallthrough + case '\uA762': // Ꝣ [LATIN CAPITAL LETTER VISIGOTHIC Z] + fallthrough + case '\uFF3A': // Ｚ [FULLWIDTH LATIN CAPITAL LETTER Z] + output[outputPos] = 'Z' + outputPos++ + + case '\u017A': // ź [LATIN SMALL LETTER Z WITH ACUTE] + fallthrough + case '\u017C': // ż [LATIN SMALL LETTER Z WITH DOT ABOVE] + fallthrough + case '\u017E': // ž [LATIN SMALL LETTER Z WITH CARON] + fallthrough + case '\u01B6': // ƶ [LATIN SMALL LETTER Z WITH STROKE] + fallthrough + case '\u021D': // ȝ http://en.wikipedia.org/wiki/Yogh [LATIN SMALL LETTER YOGH] + fallthrough + case '\u0225': // ȥ [LATIN SMALL LETTER Z WITH HOOK] + fallthrough + case '\u0240': // ɀ [LATIN SMALL LETTER Z WITH SWASH TAIL] + fallthrough + case '\u0290': // ʐ [LATIN SMALL LETTER Z WITH RETROFLEX HOOK] + fallthrough + case '\u0291': // ʑ [LATIN SMALL LETTER Z WITH CURL] + fallthrough + case '\u1D76': // ᵶ [LATIN SMALL LETTER Z WITH MIDDLE TILDE] + fallthrough + case '\u1D8E': // ᶎ [LATIN SMALL LETTER Z WITH PALATAL HOOK] + fallthrough + case '\u1E91': // ẑ [LATIN SMALL LETTER Z WITH CIRCUMFLEX] + fallthrough + case '\u1E93': // ẓ [LATIN SMALL LETTER Z WITH DOT BELOW] + fallthrough + case '\u1E95': // ẕ [LATIN SMALL LETTER Z WITH LINE BELOW] + fallthrough + case '\u24E9': // ⓩ [CIRCLED LATIN SMALL LETTER Z] + fallthrough + case '\u2C6C': // ⱬ [LATIN SMALL LETTER Z WITH DESCENDER] + fallthrough + case '\uA763': // ꝣ [LATIN SMALL LETTER VISIGOTHIC Z] + fallthrough + case '\uFF5A': // ｚ [FULLWIDTH LATIN SMALL LETTER Z] + output[outputPos] = 'z' + outputPos++ + + case '\u24B5': // ⒵ [PARENTHESIZED LATIN SMALL LETTER Z] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = 'z' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u2070': // ⁰ [SUPERSCRIPT ZERO] + fallthrough + case '\u2080': // ₀ [SUBSCRIPT ZERO] + fallthrough + case '\u24EA': // ⓪ [CIRCLED DIGIT ZERO] + fallthrough + case '\u24FF': // ⓿ [NEGATIVE CIRCLED DIGIT ZERO] + fallthrough + case '\uFF10': // ０ [FULLWIDTH DIGIT ZERO] + output[outputPos] = '0' + outputPos++ + + case '\u00B9': // ¹ [SUPERSCRIPT ONE] + fallthrough + case '\u2081': // ₁ [SUBSCRIPT ONE] + fallthrough + case '\u2460': // ① [CIRCLED DIGIT ONE] + fallthrough + case '\u24F5': // ⓵ [DOUBLE CIRCLED DIGIT ONE] + fallthrough + case '\u2776': // ❶ [DINGBAT NEGATIVE CIRCLED DIGIT ONE] + fallthrough + case '\u2780': // ➀ [DINGBAT CIRCLED SANS-SERIF DIGIT ONE] + fallthrough + case '\u278A': // ➊ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT ONE] + fallthrough + case '\uFF11': // １ [FULLWIDTH DIGIT ONE] + output[outputPos] = '1' + outputPos++ + + case '\u2488': // ⒈ [DIGIT ONE FULL STOP] + output = output[:(len(output) + 1)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u2474': // ⑴ [PARENTHESIZED DIGIT ONE] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '1' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u00B2': // ² [SUPERSCRIPT TWO] + fallthrough + case '\u2082': // ₂ [SUBSCRIPT TWO] + fallthrough + case '\u2461': // ② [CIRCLED DIGIT TWO] + fallthrough + case '\u24F6': // ⓶ [DOUBLE CIRCLED DIGIT TWO] + fallthrough + case '\u2777': // ❷ [DINGBAT NEGATIVE CIRCLED DIGIT TWO] + fallthrough + case '\u2781': // ➁ [DINGBAT CIRCLED SANS-SERIF DIGIT TWO] + fallthrough + case '\u278B': // ➋ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT TWO] + fallthrough + case '\uFF12': // ２ [FULLWIDTH DIGIT TWO] + output[outputPos] = '2' + outputPos++ + + case '\u2489': // ⒉ [DIGIT TWO FULL STOP] + output = output[:(len(output) + 1)] + output[outputPos] = '2' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u2475': // ⑵ [PARENTHESIZED DIGIT TWO] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '2' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u00B3': // ³ [SUPERSCRIPT THREE] + fallthrough + case '\u2083': // ₃ [SUBSCRIPT THREE] + fallthrough + case '\u2462': // ③ [CIRCLED DIGIT THREE] + fallthrough + case '\u24F7': // ⓷ [DOUBLE CIRCLED DIGIT THREE] + fallthrough + case '\u2778': // ❸ [DINGBAT NEGATIVE CIRCLED DIGIT THREE] + fallthrough + case '\u2782': // ➂ [DINGBAT CIRCLED SANS-SERIF DIGIT THREE] + fallthrough + case '\u278C': // ➌ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT THREE] + fallthrough + case '\uFF13': // ３ [FULLWIDTH DIGIT THREE] + output[outputPos] = '3' + outputPos++ + + case '\u248A': // ⒊ [DIGIT THREE FULL STOP] + output = output[:(len(output) + 1)] + output[outputPos] = '3' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u2476': // ⑶ [PARENTHESIZED DIGIT THREE] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '3' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u2074': // ⁴ [SUPERSCRIPT FOUR] + fallthrough + case '\u2084': // ₄ [SUBSCRIPT FOUR] + fallthrough + case '\u2463': // ④ [CIRCLED DIGIT FOUR] + fallthrough + case '\u24F8': // ⓸ [DOUBLE CIRCLED DIGIT FOUR] + fallthrough + case '\u2779': // ❹ [DINGBAT NEGATIVE CIRCLED DIGIT FOUR] + fallthrough + case '\u2783': // ➃ [DINGBAT CIRCLED SANS-SERIF DIGIT FOUR] + fallthrough + case '\u278D': // ➍ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT FOUR] + fallthrough + case '\uFF14': // ４ [FULLWIDTH DIGIT FOUR] + output[outputPos] = '4' + outputPos++ + + case '\u248B': // ⒋ [DIGIT FOUR FULL STOP] + output = output[:(len(output) + 1)] + output[outputPos] = '4' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u2477': // ⑷ [PARENTHESIZED DIGIT FOUR] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '4' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u2075': // ⁵ [SUPERSCRIPT FIVE] + fallthrough + case '\u2085': // ₅ [SUBSCRIPT FIVE] + fallthrough + case '\u2464': // ⑤ [CIRCLED DIGIT FIVE] + fallthrough + case '\u24F9': // ⓹ [DOUBLE CIRCLED DIGIT FIVE] + fallthrough + case '\u277A': // ❺ [DINGBAT NEGATIVE CIRCLED DIGIT FIVE] + fallthrough + case '\u2784': // ➄ [DINGBAT CIRCLED SANS-SERIF DIGIT FIVE] + fallthrough + case '\u278E': // ➎ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT FIVE] + fallthrough + case '\uFF15': // ５ [FULLWIDTH DIGIT FIVE] + output[outputPos] = '5' + outputPos++ + + case '\u248C': // ⒌ [DIGIT FIVE FULL STOP] + output = output[:(len(output) + 1)] + output[outputPos] = '5' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u2478': // ⑸ [PARENTHESIZED DIGIT FIVE] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '5' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u2076': // ⁶ [SUPERSCRIPT SIX] + fallthrough + case '\u2086': // ₆ [SUBSCRIPT SIX] + fallthrough + case '\u2465': // ⑥ [CIRCLED DIGIT SIX] + fallthrough + case '\u24FA': // ⓺ [DOUBLE CIRCLED DIGIT SIX] + fallthrough + case '\u277B': // ❻ [DINGBAT NEGATIVE CIRCLED DIGIT SIX] + fallthrough + case '\u2785': // ➅ [DINGBAT CIRCLED SANS-SERIF DIGIT SIX] + fallthrough + case '\u278F': // ➏ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT SIX] + fallthrough + case '\uFF16': // ６ [FULLWIDTH DIGIT SIX] + output[outputPos] = '6' + outputPos++ + + case '\u248D': // ⒍ [DIGIT SIX FULL STOP] + output = output[:(len(output) + 1)] + output[outputPos] = '6' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u2479': // ⑹ [PARENTHESIZED DIGIT SIX] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '6' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u2077': // ⁷ [SUPERSCRIPT SEVEN] + fallthrough + case '\u2087': // ₇ [SUBSCRIPT SEVEN] + fallthrough + case '\u2466': // ⑦ [CIRCLED DIGIT SEVEN] + fallthrough + case '\u24FB': // ⓻ [DOUBLE CIRCLED DIGIT SEVEN] + fallthrough + case '\u277C': // ❼ [DINGBAT NEGATIVE CIRCLED DIGIT SEVEN] + fallthrough + case '\u2786': // ➆ [DINGBAT CIRCLED SANS-SERIF DIGIT SEVEN] + fallthrough + case '\u2790': // ➐ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT SEVEN] + fallthrough + case '\uFF17': // ７ [FULLWIDTH DIGIT SEVEN] + output[outputPos] = '7' + outputPos++ + + case '\u248E': // ⒎ [DIGIT SEVEN FULL STOP] + output = output[:(len(output) + 1)] + output[outputPos] = '7' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u247A': // ⑺ [PARENTHESIZED DIGIT SEVEN] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '7' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u2078': // ⁸ [SUPERSCRIPT EIGHT] + fallthrough + case '\u2088': // ₈ [SUBSCRIPT EIGHT] + fallthrough + case '\u2467': // ⑧ [CIRCLED DIGIT EIGHT] + fallthrough + case '\u24FC': // ⓼ [DOUBLE CIRCLED DIGIT EIGHT] + fallthrough + case '\u277D': // ❽ [DINGBAT NEGATIVE CIRCLED DIGIT EIGHT] + fallthrough + case '\u2787': // ➇ [DINGBAT CIRCLED SANS-SERIF DIGIT EIGHT] + fallthrough + case '\u2791': // ➑ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT EIGHT] + fallthrough + case '\uFF18': // ８ [FULLWIDTH DIGIT EIGHT] + output[outputPos] = '8' + outputPos++ + + case '\u248F': // ⒏ [DIGIT EIGHT FULL STOP] + output = output[:(len(output) + 1)] + output[outputPos] = '8' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u247B': // ⑻ [PARENTHESIZED DIGIT EIGHT] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '8' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u2079': // ⁹ [SUPERSCRIPT NINE] + fallthrough + case '\u2089': // ₉ [SUBSCRIPT NINE] + fallthrough + case '\u2468': // ⑨ [CIRCLED DIGIT NINE] + fallthrough + case '\u24FD': // ⓽ [DOUBLE CIRCLED DIGIT NINE] + fallthrough + case '\u277E': // ❾ [DINGBAT NEGATIVE CIRCLED DIGIT NINE] + fallthrough + case '\u2788': // ➈ [DINGBAT CIRCLED SANS-SERIF DIGIT NINE] + fallthrough + case '\u2792': // ➒ [DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT NINE] + fallthrough + case '\uFF19': // ９ [FULLWIDTH DIGIT NINE] + output[outputPos] = '9' + outputPos++ + + case '\u2490': // ⒐ [DIGIT NINE FULL STOP] + output = output[:(len(output) + 1)] + output[outputPos] = '9' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u247C': // ⑼ [PARENTHESIZED DIGIT NINE] + output = output[:(len(output) + 2)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '9' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u2469': // ⑩ [CIRCLED NUMBER TEN] + fallthrough + case '\u24FE': // ⓾ [DOUBLE CIRCLED NUMBER TEN] + fallthrough + case '\u277F': // ❿ [DINGBAT NEGATIVE CIRCLED NUMBER TEN] + fallthrough + case '\u2789': // ➉ [DINGBAT CIRCLED SANS-SERIF NUMBER TEN] + fallthrough + case '\u2793': // ➓ [DINGBAT NEGATIVE CIRCLED SANS-SERIF NUMBER TEN] + output = output[:(len(output) + 1)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '0' + outputPos++ + + case '\u2491': // ⒑ [NUMBER TEN FULL STOP] + output = output[:(len(output) + 2)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '0' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u247D': // ⑽ [PARENTHESIZED NUMBER TEN] + output = output[:(len(output) + 3)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '1' + outputPos++ + output[outputPos] = '0' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u246A': // ⑪ [CIRCLED NUMBER ELEVEN] + fallthrough + case '\u24EB': // ⓫ [NEGATIVE CIRCLED NUMBER ELEVEN] + output = output[:(len(output) + 1)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '1' + outputPos++ + + case '\u2492': // ⒒ [NUMBER ELEVEN FULL STOP] + output = output[:(len(output) + 2)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '1' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u247E': // ⑾ [PARENTHESIZED NUMBER ELEVEN] + output = output[:(len(output) + 3)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '1' + outputPos++ + output[outputPos] = '1' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u246B': // ⑫ [CIRCLED NUMBER TWELVE] + fallthrough + case '\u24EC': // ⓬ [NEGATIVE CIRCLED NUMBER TWELVE] + output = output[:(len(output) + 1)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '2' + outputPos++ + + case '\u2493': // ⒓ [NUMBER TWELVE FULL STOP] + output = output[:(len(output) + 2)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '2' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u247F': // ⑿ [PARENTHESIZED NUMBER TWELVE] + output = output[:(len(output) + 3)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '1' + outputPos++ + output[outputPos] = '2' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u246C': // ⑬ [CIRCLED NUMBER THIRTEEN] + fallthrough + case '\u24ED': // ⓭ [NEGATIVE CIRCLED NUMBER THIRTEEN] + output = output[:(len(output) + 1)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '3' + outputPos++ + + case '\u2494': // ⒔ [NUMBER THIRTEEN FULL STOP] + output = output[:(len(output) + 2)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '3' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u2480': // ⒀ [PARENTHESIZED NUMBER THIRTEEN] + output = output[:(len(output) + 3)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '1' + outputPos++ + output[outputPos] = '3' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u246D': // ⑭ [CIRCLED NUMBER FOURTEEN] + fallthrough + case '\u24EE': // ⓮ [NEGATIVE CIRCLED NUMBER FOURTEEN] + output = output[:(len(output) + 1)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '4' + outputPos++ + + case '\u2495': // ⒕ [NUMBER FOURTEEN FULL STOP] + output = output[:(len(output) + 2)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '4' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u2481': // ⒁ [PARENTHESIZED NUMBER FOURTEEN] + output = output[:(len(output) + 3)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '1' + outputPos++ + output[outputPos] = '4' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u246E': // ⑮ [CIRCLED NUMBER FIFTEEN] + fallthrough + case '\u24EF': // ⓯ [NEGATIVE CIRCLED NUMBER FIFTEEN] + output = output[:(len(output) + 1)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '5' + outputPos++ + + case '\u2496': // ⒖ [NUMBER FIFTEEN FULL STOP] + output = output[:(len(output) + 2)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '5' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u2482': // ⒂ [PARENTHESIZED NUMBER FIFTEEN] + output = output[:(len(output) + 3)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '1' + outputPos++ + output[outputPos] = '5' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u246F': // ⑯ [CIRCLED NUMBER SIXTEEN] + fallthrough + case '\u24F0': // ⓰ [NEGATIVE CIRCLED NUMBER SIXTEEN] + output = output[:(len(output) + 1)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '6' + outputPos++ + + case '\u2497': // ⒗ [NUMBER SIXTEEN FULL STOP] + output = output[:(len(output) + 2)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '6' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u2483': // ⒃ [PARENTHESIZED NUMBER SIXTEEN] + output = output[:(len(output) + 3)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '1' + outputPos++ + output[outputPos] = '6' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u2470': // ⑰ [CIRCLED NUMBER SEVENTEEN] + fallthrough + case '\u24F1': // ⓱ [NEGATIVE CIRCLED NUMBER SEVENTEEN] + output = output[:(len(output) + 1)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '7' + outputPos++ + + case '\u2498': // ⒘ [NUMBER SEVENTEEN FULL STOP] + output = output[:(len(output) + 2)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '7' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u2484': // ⒄ [PARENTHESIZED NUMBER SEVENTEEN] + output = output[:(len(output) + 3)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '1' + outputPos++ + output[outputPos] = '7' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u2471': // ⑱ [CIRCLED NUMBER EIGHTEEN] + fallthrough + case '\u24F2': // ⓲ [NEGATIVE CIRCLED NUMBER EIGHTEEN] + output = output[:(len(output) + 1)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '8' + outputPos++ + + case '\u2499': // ⒙ [NUMBER EIGHTEEN FULL STOP] + output = output[:(len(output) + 2)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '8' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u2485': // ⒅ [PARENTHESIZED NUMBER EIGHTEEN] + output = output[:(len(output) + 3)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '1' + outputPos++ + output[outputPos] = '8' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u2472': // ⑲ [CIRCLED NUMBER NINETEEN] + fallthrough + case '\u24F3': // ⓳ [NEGATIVE CIRCLED NUMBER NINETEEN] + output = output[:(len(output) + 1)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '9' + outputPos++ + + case '\u249A': // ⒚ [NUMBER NINETEEN FULL STOP] + output = output[:(len(output) + 2)] + output[outputPos] = '1' + outputPos++ + output[outputPos] = '9' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u2486': // ⒆ [PARENTHESIZED NUMBER NINETEEN] + output = output[:(len(output) + 3)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '1' + outputPos++ + output[outputPos] = '9' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u2473': // ⑳ [CIRCLED NUMBER TWENTY] + fallthrough + case '\u24F4': // ⓴ [NEGATIVE CIRCLED NUMBER TWENTY] + output = output[:(len(output) + 1)] + output[outputPos] = '2' + outputPos++ + output[outputPos] = '0' + outputPos++ + + case '\u249B': // ⒛ [NUMBER TWENTY FULL STOP] + output = output[:(len(output) + 2)] + output[outputPos] = '2' + outputPos++ + output[outputPos] = '0' + outputPos++ + output[outputPos] = '.' + outputPos++ + + case '\u2487': // ⒇ [PARENTHESIZED NUMBER TWENTY] + output = output[:(len(output) + 3)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '2' + outputPos++ + output[outputPos] = '0' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u00AB': // « [LEFT-POINTING DOUBLE ANGLE QUOTATION MARK] + fallthrough + case '\u00BB': // » [RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK] + fallthrough + case '\u201C': // “ [LEFT DOUBLE QUOTATION MARK] + fallthrough + case '\u201D': // ” [RIGHT DOUBLE QUOTATION MARK] + fallthrough + case '\u201E': // „ [DOUBLE LOW-9 QUOTATION MARK] + fallthrough + case '\u2033': // ″ [DOUBLE PRIME] + fallthrough + case '\u2036': // ‶ [REVERSED DOUBLE PRIME] + fallthrough + case '\u275D': // ❝ [HEAVY DOUBLE TURNED COMMA QUOTATION MARK ORNAMENT] + fallthrough + case '\u275E': // ❞ [HEAVY DOUBLE COMMA QUOTATION MARK ORNAMENT] + fallthrough + case '\u276E': // ❮ [HEAVY LEFT-POINTING ANGLE QUOTATION MARK ORNAMENT] + fallthrough + case '\u276F': // ❯ [HEAVY RIGHT-POINTING ANGLE QUOTATION MARK ORNAMENT] + fallthrough + case '\uFF02': // ＂ [FULLWIDTH QUOTATION MARK] + output[outputPos] = '"' + outputPos++ + + case '\u2018': // ‘ [LEFT SINGLE QUOTATION MARK] + fallthrough + case '\u2019': // ’ [RIGHT SINGLE QUOTATION MARK] + fallthrough + case '\u201A': // ‚ [SINGLE LOW-9 QUOTATION MARK] + fallthrough + case '\u201B': // ‛ [SINGLE HIGH-REVERSED-9 QUOTATION MARK] + fallthrough + case '\u2032': // ′ [PRIME] + fallthrough + case '\u2035': // ‵ [REVERSED PRIME] + fallthrough + case '\u2039': // ‹ [SINGLE LEFT-POINTING ANGLE QUOTATION MARK] + fallthrough + case '\u203A': // › [SINGLE RIGHT-POINTING ANGLE QUOTATION MARK] + fallthrough + case '\u275B': // ❛ [HEAVY SINGLE TURNED COMMA QUOTATION MARK ORNAMENT] + fallthrough + case '\u275C': // ❜ [HEAVY SINGLE COMMA QUOTATION MARK ORNAMENT] + fallthrough + case '\uFF07': // ＇ [FULLWIDTH APOSTROPHE] + output[outputPos] = '\'' + outputPos++ + + case '\u2010': // ‐ [HYPHEN] + fallthrough + case '\u2011': // ‑ [NON-BREAKING HYPHEN] + fallthrough + case '\u2012': // ‒ [FIGURE DASH] + fallthrough + case '\u2013': // – [EN DASH] + fallthrough + case '\u2014': // — [EM DASH] + fallthrough + case '\u207B': // ⁻ [SUPERSCRIPT MINUS] + fallthrough + case '\u208B': // ₋ [SUBSCRIPT MINUS] + fallthrough + case '\uFF0D': // － [FULLWIDTH HYPHEN-MINUS] + output[outputPos] = '-' + outputPos++ + + case '\u2045': // ⁅ [LEFT SQUARE BRACKET WITH QUILL] + fallthrough + case '\u2772': // ❲ [LIGHT LEFT TORTOISE SHELL BRACKET ORNAMENT] + fallthrough + case '\uFF3B': // ［ [FULLWIDTH LEFT SQUARE BRACKET] + output[outputPos] = '[' + outputPos++ + + case '\u2046': // ⁆ [RIGHT SQUARE BRACKET WITH QUILL] + fallthrough + case '\u2773': // ❳ [LIGHT RIGHT TORTOISE SHELL BRACKET ORNAMENT] + fallthrough + case '\uFF3D': // ］ [FULLWIDTH RIGHT SQUARE BRACKET] + output[outputPos] = ']' + outputPos++ + + case '\u207D': // ⁽ [SUPERSCRIPT LEFT PARENTHESIS] + fallthrough + case '\u208D': // ₍ [SUBSCRIPT LEFT PARENTHESIS] + fallthrough + case '\u2768': // ❨ [MEDIUM LEFT PARENTHESIS ORNAMENT] + fallthrough + case '\u276A': // ❪ [MEDIUM FLATTENED LEFT PARENTHESIS ORNAMENT] + fallthrough + case '\uFF08': // （ [FULLWIDTH LEFT PARENTHESIS] + output[outputPos] = '(' + outputPos++ + + case '\u2E28': // ⸨ [LEFT DOUBLE PARENTHESIS] + output = output[:(len(output) + 1)] + output[outputPos] = '(' + outputPos++ + output[outputPos] = '(' + outputPos++ + + case '\u207E': // ⁾ [SUPERSCRIPT RIGHT PARENTHESIS] + fallthrough + case '\u208E': // ₎ [SUBSCRIPT RIGHT PARENTHESIS] + fallthrough + case '\u2769': // ❩ [MEDIUM RIGHT PARENTHESIS ORNAMENT] + fallthrough + case '\u276B': // ❫ [MEDIUM FLATTENED RIGHT PARENTHESIS ORNAMENT] + fallthrough + case '\uFF09': // ） [FULLWIDTH RIGHT PARENTHESIS] + output[outputPos] = ')' + outputPos++ + + case '\u2E29': // ⸩ [RIGHT DOUBLE PARENTHESIS] + output = output[:(len(output) + 1)] + output[outputPos] = ')' + outputPos++ + output[outputPos] = ')' + outputPos++ + + case '\u276C': // ❬ [MEDIUM LEFT-POINTING ANGLE BRACKET ORNAMENT] + fallthrough + case '\u2770': // ❰ [HEAVY LEFT-POINTING ANGLE BRACKET ORNAMENT] + fallthrough + case '\uFF1C': // ＜ [FULLWIDTH LESS-THAN SIGN] + output[outputPos] = '<' + outputPos++ + + case '\u276D': // ❭ [MEDIUM RIGHT-POINTING ANGLE BRACKET ORNAMENT] + fallthrough + case '\u2771': // ❱ [HEAVY RIGHT-POINTING ANGLE BRACKET ORNAMENT] + fallthrough + case '\uFF1E': // ＞ [FULLWIDTH GREATER-THAN SIGN] + output[outputPos] = '>' + outputPos++ + + case '\u2774': // ❴ [MEDIUM LEFT CURLY BRACKET ORNAMENT] + fallthrough + case '\uFF5B': // ｛ [FULLWIDTH LEFT CURLY BRACKET] + output[outputPos] = '{' + outputPos++ + + case '\u2775': // ❵ [MEDIUM RIGHT CURLY BRACKET ORNAMENT] + fallthrough + case '\uFF5D': // ｝ [FULLWIDTH RIGHT CURLY BRACKET] + output[outputPos] = '}' + outputPos++ + + case '\u207A': // ⁺ [SUPERSCRIPT PLUS SIGN] + fallthrough + case '\u208A': // ₊ [SUBSCRIPT PLUS SIGN] + fallthrough + case '\uFF0B': // ＋ [FULLWIDTH PLUS SIGN] + output[outputPos] = '+' + outputPos++ + + case '\u207C': // ⁼ [SUPERSCRIPT EQUALS SIGN] + fallthrough + case '\u208C': // ₌ [SUBSCRIPT EQUALS SIGN] + fallthrough + case '\uFF1D': // ＝ [FULLWIDTH EQUALS SIGN] + output[outputPos] = '=' + outputPos++ + + case '\uFF01': // ！ [FULLWIDTH EXCLAMATION MARK] + output[outputPos] = '!' + outputPos++ + + case '\u203C': // ‼ [DOUBLE EXCLAMATION MARK] + output = output[:(len(output) + 1)] + output[outputPos] = '!' + outputPos++ + output[outputPos] = '!' + outputPos++ + + case '\u2049': // ⁉ [EXCLAMATION QUESTION MARK] + output = output[:(len(output) + 1)] + output[outputPos] = '!' + outputPos++ + output[outputPos] = '?' + outputPos++ + + case '\uFF03': // ＃ [FULLWIDTH NUMBER SIGN] + output[outputPos] = '#' + outputPos++ + + case '\uFF04': // ＄ [FULLWIDTH DOLLAR SIGN] + output[outputPos] = '$' + outputPos++ + + case '\u2052': // ⁒ [COMMERCIAL MINUS SIGN] + fallthrough + case '\uFF05': // ％ [FULLWIDTH PERCENT SIGN] + output[outputPos] = '%' + outputPos++ + + case '\uFF06': // ＆ [FULLWIDTH AMPERSAND] + output[outputPos] = '&' + outputPos++ + + case '\u204E': // ⁎ [LOW ASTERISK] + fallthrough + case '\uFF0A': // ＊ [FULLWIDTH ASTERISK] + output[outputPos] = '*' + outputPos++ + + case '\uFF0C': // ， [FULLWIDTH COMMA] + output[outputPos] = ',' + outputPos++ + + case '\uFF0E': // ． [FULLWIDTH FULL STOP] + output[outputPos] = '.' + outputPos++ + + case '\u2044': // ⁄ [FRACTION SLASH] + fallthrough + case '\uFF0F': // ／ [FULLWIDTH SOLIDUS] + output[outputPos] = '/' + outputPos++ + + case '\uFF1A': // ： [FULLWIDTH COLON] + output[outputPos] = ':' + outputPos++ + + case '\u204F': // ⁏ [REVERSED SEMICOLON] + fallthrough + case '\uFF1B': // ； [FULLWIDTH SEMICOLON] + output[outputPos] = ';' + outputPos++ + + case '\uFF1F': // ？ [FULLWIDTH QUESTION MARK] + output[outputPos] = '?' + outputPos++ + + case '\u2047': // ⁇ [DOUBLE QUESTION MARK] + output = output[:(len(output) + 1)] + output[outputPos] = '?' + outputPos++ + output[outputPos] = '?' + outputPos++ + + case '\u2048': // ⁈ [QUESTION EXCLAMATION MARK] + output = output[:(len(output) + 1)] + output[outputPos] = '?' + outputPos++ + output[outputPos] = '!' + outputPos++ + + case '\uFF20': // ＠ [FULLWIDTH COMMERCIAL AT] + output[outputPos] = '@' + outputPos++ + + case '\uFF3C': // ＼ [FULLWIDTH REVERSE SOLIDUS] + output[outputPos] = '\\' + outputPos++ + + case '\u2038': // ‸ [CARET] + fallthrough + case '\uFF3E': // ＾ [FULLWIDTH CIRCUMFLEX ACCENT] + output[outputPos] = '^' + outputPos++ + + case '\uFF3F': // ＿ [FULLWIDTH LOW LINE] + output[outputPos] = '_' + outputPos++ + + case '\u2053': // ⁓ [SWUNG DASH] + fallthrough + case '\uFF5E': // ～ [FULLWIDTH TILDE] + output[outputPos] = '~' + outputPos++ + + default: + output[outputPos] = c + outputPos++ + } + } + } + return output +} diff --git a/analysis/char/asciifolding/asciifolding_test.go b/analysis/char/asciifolding/asciifolding_test.go new file mode 100644 index 0000000..52f1594 --- /dev/null +++ b/analysis/char/asciifolding/asciifolding_test.go @@ -0,0 +1,124 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package asciifolding + +import ( + "fmt" + "reflect" + "testing" +) + +func TestAsciiFoldingFilter(t *testing.T) { + tests := []struct { + input []byte + output []byte + }{ + { + // empty input passes + input: []byte(``), + output: []byte(``), + }, + { + // no modification for plain ASCII + input: []byte(`The quick brown fox jumps over the lazy dog`), + output: []byte(`The quick brown fox jumps over the lazy dog`), + }, + { + // Umlauts are folded to plain ASCII + input: []byte(`The quick bröwn fox jümps over the läzy dog`), + output: []byte(`The quick brown fox jumps over the lazy dog`), + }, + { + // composite unicode runes are folded to more than one ASCII rune + input: []byte(`ÆꜴ`), + output: []byte(`AEAO`), + }, + { + // apples from https://issues.couchbase.com/browse/MB-33486 + input: []byte(`Ápple Àpple Äpple Âpple Ãpple Åpple`), + output: []byte(`Apple Apple Apple Apple Apple Apple`), + }, + { + // Fix ASCII folding of \u24A2 + input: []byte(`⒢`), + output: []byte(`(g)`), + }, + { + // Test folding of \u2053 (SWUNG DASH) + input: []byte(`a⁓b`), + output: []byte(`a~b`), + }, + { + // Test folding of \uFF5E (FULLWIDTH TILDE) + input: []byte(`c～d`), + output: []byte(`c~d`), + }, + { + // Test folding of \uFF3F (FULLWIDTH LOW LINE) - case before tilde + input: []byte(`e＿f`), + output: []byte(`e_f`), + }, + { + // Test mix including tilde and default fallthrough (using a character not explicitly folded) + input: []byte(`a⁓b✅c～d`), + output: []byte(`a~b✅c~d`), + }, + { + // Test start of 'A' fallthrough block + input: []byte(`ÀBC`), + output: []byte(`ABC`), + }, + { + // Test end of 'A' fallthrough block + input: []byte(`DEFẶ`), + output: []byte(`DEFA`), + }, + { + // Test start of 'AE' fallthrough block + input: []byte(`Æ`), + output: []byte(`AE`), + }, + { + // Test end of 'AE' fallthrough block + input: []byte(`ᴁ`), + output: []byte(`AE`), + }, + { + // Test 'DZ' multi-rune output + input: []byte(`DŽebra`), + output: []byte(`DZebra`), + }, + { + // Test start of 'a' fallthrough block + input: []byte(`àbc`), + output: []byte(`abc`), + }, + { + // Test end of 'a' fallthrough block + input: []byte(`defａ`), + output: []byte(`defa`), + }, + } + + for _, test := range tests { + filter := New() + t.Run(fmt.Sprintf("on %s", test.input), func(t *testing.T) { + output := filter.Filter(test.input) + if !reflect.DeepEqual(output, test.output) { + t.Errorf("\nExpected:\n`%s`\ngot:\n`%s`\n", string(test.output), string(output)) + } + }) + } +} diff --git a/analysis/char/html/html.go b/analysis/char/html/html.go new file mode 100644 index 0000000..d939fdb --- /dev/null +++ b/analysis/char/html/html.go @@ -0,0 +1,57 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package html + +import ( + "bytes" + "regexp" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "html" + +var htmlCharFilterRegexp = regexp.MustCompile(`\s]+))?)+\s*|\s*)/?>`) + +type CharFilter struct { + r *regexp.Regexp + replacement []byte +} + +func New() *CharFilter { + return &CharFilter{ + r: htmlCharFilterRegexp, + replacement: []byte(" "), + } +} + +func (s *CharFilter) Filter(input []byte) []byte { + return s.r.ReplaceAllFunc( + input, func(in []byte) []byte { + return bytes.Repeat(s.replacement, len(in)) + }) +} + +func CharFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.CharFilter, error) { + return New(), nil +} + +func init() { + err := registry.RegisterCharFilter(Name, CharFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/char/regexp/regexp.go b/analysis/char/regexp/regexp.go new file mode 100644 index 0000000..a94236a --- /dev/null +++ b/analysis/char/regexp/regexp.go @@ -0,0 +1,65 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package regexp + +import ( + "fmt" + "regexp" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "regexp" + +type CharFilter struct { + r *regexp.Regexp + replacement []byte +} + +func New(r *regexp.Regexp, replacement []byte) *CharFilter { + return &CharFilter{ + r: r, + replacement: replacement, + } +} + +func (s *CharFilter) Filter(input []byte) []byte { + return s.r.ReplaceAll(input, s.replacement) +} + +func CharFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.CharFilter, error) { + regexpStr, ok := config["regexp"].(string) + if !ok { + return nil, fmt.Errorf("must specify regexp") + } + r, err := regexp.Compile(regexpStr) + if err != nil { + return nil, fmt.Errorf("unable to build regexp char filter: %v", err) + } + replaceBytes := []byte(" ") + replaceStr, ok := config["replace"].(string) + if ok { + replaceBytes = []byte(replaceStr) + } + return New(r, replaceBytes), nil +} + +func init() { + err := registry.RegisterCharFilter(Name, CharFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/char/regexp/regexp_test.go b/analysis/char/regexp/regexp_test.go new file mode 100644 index 0000000..ff1b04b --- /dev/null +++ b/analysis/char/regexp/regexp_test.go @@ -0,0 +1,88 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package regexp + +import ( + "fmt" + "reflect" + "regexp" + "testing" +) + +func TestRegexpCharFilter(t *testing.T) { + + tests := []struct { + regexStr string + replace []byte + input []byte + output []byte + }{ + { + regexStr: `\s]+))?)+\s*|\s*)/?>`, + replace: []byte{' '}, + input: []byte(`test`), + output: []byte(` test `), + }, + { + regexStr: `\x{200C}`, + replace: []byte{' '}, + input: []byte("water\u200Cunder\u200Cthe\u200Cbridge"), + output: []byte("water under the bridge"), + }, + { + regexStr: `([a-z])\s+(\d)`, + replace: []byte(`$1-$2`), + input: []byte(`temp 1`), + output: []byte(`temp-1`), + }, + { + regexStr: `foo.?`, + replace: []byte(`X`), + input: []byte(`seafood, fool`), + output: []byte(`seaX, X`), + }, + { + regexStr: `def`, + replace: []byte(`_`), + input: []byte(`abcdefghi`), + output: []byte(`abc_ghi`), + }, + { + regexStr: `456`, + replace: []byte(`000000`), + input: []byte(`123456789`), + output: []byte(`123000000789`), + }, + { + regexStr: `“|”`, + replace: []byte(`"`), + input: []byte(`“hello”`), + output: []byte(`"hello"`), + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("match %s replace %s", test.regexStr, string(test.replace)), func(t *testing.T) { + regex := regexp.MustCompile(test.regexStr) + filter := New(regex, test.replace) + + output := filter.Filter(test.input) + if !reflect.DeepEqual(test.output, output) { + t.Errorf("Expected: `%s`, Got: `%s`\n", string(test.output), string(output)) + } + }) + + } +} diff --git a/analysis/char/zerowidthnonjoiner/zerowidthnonjoiner.go b/analysis/char/zerowidthnonjoiner/zerowidthnonjoiner.go new file mode 100644 index 0000000..0a39fa6 --- /dev/null +++ b/analysis/char/zerowidthnonjoiner/zerowidthnonjoiner.go @@ -0,0 +1,39 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zerowidthnonjoiner + +import ( + "regexp" + + "github.com/blevesearch/bleve/v2/analysis" + regexpCharFilter "github.com/blevesearch/bleve/v2/analysis/char/regexp" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "zero_width_spaces" + +var zeroWidthNonJoinerRegexp = regexp.MustCompile(`\x{200C}`) + +func CharFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.CharFilter, error) { + replaceBytes := []byte(" ") + return regexpCharFilter.New(zeroWidthNonJoinerRegexp, replaceBytes), nil +} + +func init() { + err := registry.RegisterCharFilter(Name, CharFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/datetime/flexible/flexible.go b/analysis/datetime/flexible/flexible.go new file mode 100644 index 0000000..36cc9e8 --- /dev/null +++ b/analysis/datetime/flexible/flexible.go @@ -0,0 +1,67 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flexible + +import ( + "fmt" + "time" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "flexiblego" + +type DateTimeParser struct { + layouts []string +} + +func New(layouts []string) *DateTimeParser { + return &DateTimeParser{ + layouts: layouts, + } +} + +func (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error) { + for _, layout := range p.layouts { + rv, err := time.Parse(layout, input) + if err == nil { + return rv, layout, nil + } + } + return time.Time{}, "", analysis.ErrInvalidDateTime +} + +func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) { + layouts, ok := config["layouts"].([]interface{}) + if !ok { + return nil, fmt.Errorf("must specify layouts") + } + var layoutStrs []string + for _, layout := range layouts { + layoutStr, ok := layout.(string) + if ok { + layoutStrs = append(layoutStrs, layoutStr) + } + } + return New(layoutStrs), nil +} + +func init() { + err := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/datetime/flexible/flexible_test.go b/analysis/datetime/flexible/flexible_test.go new file mode 100644 index 0000000..12aedb2 --- /dev/null +++ b/analysis/datetime/flexible/flexible_test.go @@ -0,0 +1,100 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flexible + +import ( + "reflect" + "testing" + "time" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestFlexibleDateTimeParser(t *testing.T) { + testLocation := time.FixedZone("", -8*60*60) + + rfc3339NoTimezone := "2006-01-02T15:04:05" + rfc3339NoTimezoneNoT := "2006-01-02 15:04:05" + rfc3339NoTime := "2006-01-02" + + dateOptionalTimeParser := New( + []string{ + time.RFC3339Nano, + time.RFC3339, + rfc3339NoTimezone, + rfc3339NoTimezoneNoT, + rfc3339NoTime, + }) + + tests := []struct { + input string + expectedTime time.Time + expectedLayout string + expectedError error + }{ + { + input: "2014-08-03", + expectedTime: time.Date(2014, 8, 3, 0, 0, 0, 0, time.UTC), + expectedLayout: rfc3339NoTime, + expectedError: nil, + }, + { + input: "2014-08-03T15:59:30", + expectedTime: time.Date(2014, 8, 3, 15, 59, 30, 0, time.UTC), + expectedLayout: rfc3339NoTimezone, + expectedError: nil, + }, + { + input: "2014-08-03 15:59:30", + expectedTime: time.Date(2014, 8, 3, 15, 59, 30, 0, time.UTC), + expectedLayout: rfc3339NoTimezoneNoT, + expectedError: nil, + }, + { + input: "2014-08-03T15:59:30-08:00", + expectedTime: time.Date(2014, 8, 3, 15, 59, 30, 0, testLocation), + expectedLayout: time.RFC3339Nano, + expectedError: nil, + }, + { + + input: "2014-08-03T15:59:30.999999999-08:00", + expectedTime: time.Date(2014, 8, 3, 15, 59, 30, 999999999, testLocation), + expectedLayout: time.RFC3339Nano, + expectedError: nil, + }, + { + input: "not a date time", + expectedTime: time.Time{}, + expectedLayout: "", + expectedError: analysis.ErrInvalidDateTime, + }, + } + + for _, test := range tests { + t.Run(test.input, func(t *testing.T) { + actualTime, actualLayout, actualErr := dateOptionalTimeParser.ParseDateTime(test.input) + if actualErr != test.expectedError { + t.Fatalf("expected error %#v, got %#v", test.expectedError, actualErr) + } + if !reflect.DeepEqual(actualTime, test.expectedTime) { + t.Errorf("expected time %v, got %v", test.expectedTime, actualTime) + } + if !reflect.DeepEqual(actualLayout, test.expectedLayout) { + t.Errorf("expected layout %v, got %v", test.expectedLayout, actualLayout) + } + }) + } +} diff --git a/analysis/datetime/iso/iso.go b/analysis/datetime/iso/iso.go new file mode 100644 index 0000000..7a81869 --- /dev/null +++ b/analysis/datetime/iso/iso.go @@ -0,0 +1,250 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iso + +import ( + "fmt" + "strings" + "time" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "isostyle" + +var textLiteralDelimiter byte = '\'' // single quote + +// ISO style date strings are represented in +// https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html +// +// Some format specifiers are not specified in go time package, such as: +// - 'V' for timezone name, like 'Europe/Berlin' or 'America/New_York'. +// - 'Q' for quarter of year, like Q3 or 3rd Quarter. +// - 'zzzz' for full name of timezone like "Japan Standard Time" or "Eastern Standard Time". +// - 'O' for localized zone-offset, like GMT+8 or GMT+08:00. +// - '[]' for optional section of the format. +// - 'G' for era, like AD or BC. +// - 'W' for week of month. +// - 'D' for day of year. +// So date strings with these date elements cannot be parsed. +var timeElementToLayout = map[byte]map[int]string{ + 'M': { + 4: "January", // MMMM = full month name + 3: "Jan", // MMM = short month name + 2: "01", // MM = month of year (2 digits) (01-12) + 1: "1", // M = month of year (1 digit) (1-12) + }, + 'd': { + 2: "02", // dd = day of month (2 digits) (01-31) + 1: "2", // d = day of month (1 digit) (1-31) + }, + 'a': { + 2: "pm", // aa = pm/am + 1: "PM", // a = PM/AM + }, + 'H': { + 2: "15", // HH = hour (24 hour clock) (2 digits) + 1: "15", // H = hour (24 hour clock) (1 digit) + }, + 'm': { + 2: "04", // mm = minute (2 digits) + 1: "4", // m = minute (1 digit) + }, + 's': { + 2: "05", // ss = seconds (2 digits) + 1: "5", // s = seconds (1 digit) + }, + + // timezone offsets from UTC below + 'X': { + 5: "Z07:00:00", // XXXXX = timezone offset (+-hh:mm:ss) + 4: "Z070000", // XXXX = timezone offset (+-hhmmss) + 3: "Z07:00", // XXX = timezone offset (+-hh:mm) + 2: "Z0700", // XX = timezone offset (+-hhmm) + 1: "Z07", // X = timezone offset (+-hh) + }, + 'x': { + 5: "-07:00:00", // xxxxx = timezone offset (+-hh:mm:ss) + 4: "-070000", // xxxx = timezone offset (+-hhmmss) + 3: "-07:00", // xxx = timezone offset (+-hh:mm) + 2: "-0700", // xx = timezone offset (+-hhmm) + 1: "-07", // x = timezone offset (+-hh) + }, +} + +type DateTimeParser struct { + layouts []string +} + +func New(layouts []string) *DateTimeParser { + return &DateTimeParser{ + layouts: layouts, + } +} + +func (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error) { + for _, layout := range p.layouts { + rv, err := time.Parse(layout, input) + if err == nil { + return rv, layout, nil + } + } + return time.Time{}, "", analysis.ErrInvalidDateTime +} + +func letterCounter(layout string, idx int) int { + count := 1 + for idx+count < len(layout) { + if layout[idx+count] == layout[idx] { + count++ + } else { + break + } + } + return count +} + +func invalidFormatError(character byte, count int) error { + return fmt.Errorf("invalid format string, unknown format specifier: %s", strings.Repeat(string(character), count)) +} + +func parseISOString(layout string) (string, error) { + var dateTimeLayout strings.Builder + + for idx := 0; idx < len(layout); { + // check if the character is a text literal delimiter (') + if layout[idx] == textLiteralDelimiter { + if idx+1 < len(layout) && layout[idx+1] == textLiteralDelimiter { + // if the next character is also a text literal delimiter, then + // copy the character as is + dateTimeLayout.WriteByte(textLiteralDelimiter) + idx += 2 + continue + } + // find the next text literal delimiter + for idx++; idx < len(layout); idx++ { + if layout[idx] == textLiteralDelimiter { + break + } + dateTimeLayout.WriteByte(layout[idx]) + } + // idx can either be equal to len(layout) if the text literal delimiter is not found + // after the first text literal delimiter or it will be equal to the index of the + // second text literal delimiter + if idx == len(layout) { + // text literal delimiter not found error + return "", fmt.Errorf("invalid format string, expected text literal delimiter: %s", string(textLiteralDelimiter)) + } + // increment idx to skip the second text literal delimiter + idx++ + continue + } + // check if character is a letter in english alphabet - a-zA-Z which are reserved + // for format specifiers + if (layout[idx] >= 'a' && layout[idx] <= 'z') || (layout[idx] >= 'A' && layout[idx] <= 'Z') { + // find the number of times the character occurs consecutively + count := letterCounter(layout, idx) + character := layout[idx] + // first check the table + if layout, ok := timeElementToLayout[character][count]; ok { + dateTimeLayout.WriteString(layout) + } else { + switch character { + case 'y', 'u', 'Y': + // year + if count == 2 { + dateTimeLayout.WriteString("06") + } else { + format := fmt.Sprintf("%%0%ds", count) + dateTimeLayout.WriteString(fmt.Sprintf(format, "2006")) + } + case 'h', 'K': + // hour (1-12) + switch count { + case 2: + // hh, KK -> 03 + dateTimeLayout.WriteString("03") + case 1: + // h, K -> 3 + dateTimeLayout.WriteString("3") + default: + // e.g., hhh + return "", invalidFormatError(character, count) + } + case 'E': + // day of week + if count == 4 { + dateTimeLayout.WriteString("Monday") // EEEE -> Monday + } else if count <= 3 { + dateTimeLayout.WriteString("Mon") // E, EE, EEE -> Mon + } else { + return "", invalidFormatError(character, count) // e.g., EEEEE + } + case 'S': + // fraction of second + // .SSS = millisecond + // .SSSSSS = microsecond + // .SSSSSSSSS = nanosecond + if count > 9 { + return "", invalidFormatError(character, count) + } + dateTimeLayout.WriteString(strings.Repeat(string('0'), count)) + case 'z': + // timezone id + if count < 5 { + dateTimeLayout.WriteString("MST") + } else { + return "", invalidFormatError(character, count) + } + default: + return "", invalidFormatError(character, count) + } + } + idx += count + } else { + // copy the character as is + dateTimeLayout.WriteByte(layout[idx]) + idx++ + } + } + return dateTimeLayout.String(), nil +} + +func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) { + layouts, ok := config["layouts"].([]interface{}) + if !ok { + return nil, fmt.Errorf("must specify layouts") + } + var layoutStrs []string + for _, layout := range layouts { + layoutStr, ok := layout.(string) + if ok { + layout, err := parseISOString(layoutStr) + if err != nil { + return nil, err + } + layoutStrs = append(layoutStrs, layout) + } + } + return New(layoutStrs), nil +} + +func init() { + err := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/datetime/iso/iso_test.go b/analysis/datetime/iso/iso_test.go new file mode 100644 index 0000000..742e96b --- /dev/null +++ b/analysis/datetime/iso/iso_test.go @@ -0,0 +1,182 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package iso + +import ( + "fmt" + "testing" +) + +func TestConversionFromISOStyle(t *testing.T) { + tests := []struct { + input string + output string + err error + }{ + { + input: "yyyy-MM-dd", + output: "2006-01-02", + err: nil, + }, + { + input: "uuu/M''''dd'T'HH:m:ss.SSS", + output: "2006/1''02T15:4:05.000", + err: nil, + }, + { + input: "YYYY-MM-dd'T'H:mm:ss zzz", + output: "2006-01-02T15:04:05 MST", + err: nil, + }, + { + input: "MMMM dd yyyy', 'HH:mm:ss.SSS", + output: "January 02 2006, 15:04:05.000", + err: nil, + }, + { + input: "h 'o'''' clock' a, XXX", + output: "3 o' clock PM, Z07:00", + err: nil, + }, + { + input: "YYYY-MM-dd'T'HH:mm:ss'Z'", + output: "2006-01-02T15:04:05Z", + err: nil, + }, + { + input: "E MMM d H:mm:ss z Y", + output: "Mon Jan 2 15:04:05 MST 2006", + err: nil, + }, + { + input: "E MMM DD H:m:s z Y", + output: "", + err: fmt.Errorf("invalid format string, unknown format specifier: DD"), + }, + { + input: "E MMM''''' H:m:s z Y", + output: "", + err: fmt.Errorf("invalid format string, expected text literal delimiter: '"), + }, + { + input: "MMMMM dd yyyy', 'HH:mm:ss.SSS", + output: "", + err: fmt.Errorf("invalid format string, unknown format specifier: MMMMM"), + }, + { + input: "yy", // year (2 digits) + output: "06", + err: nil, + }, + { + input: "yyyyy", // year (5 digits, padded) + output: "02006", + err: nil, + }, + { + input: "h", // hour 1-12 (1 digit) + output: "3", + err: nil, + }, + { + input: "hh", // hour 1-12 (2 digits) + output: "03", + err: nil, + }, + { + input: "KK", // hour 1-12 (2 digits, alt) + output: "03", + err: nil, + }, + { + input: "hhh", // invalid hour count + output: "", + err: fmt.Errorf("invalid format string, unknown format specifier: hhh"), + }, + { + input: "E", // Day of week (short) + output: "Mon", + err: nil, + }, + { + input: "EEE", // Day of week (short) + output: "Mon", + err: nil, + }, + { + input: "EEEE", // Day of week (long) + output: "Monday", + err: nil, + }, + { + input: "EEEEE", // Day of week (long) + output: "", + err: fmt.Errorf("invalid format string, unknown format specifier: EEEEE"), + }, + { + input: "S", // Fraction of second (1 digit) + output: "0", + err: nil, + }, + { + input: "SSSSSSSSS", // Fraction of second (9 digits) + output: "000000000", + err: nil, + }, + { + input: "SSSSSSSSSS", // Invalid fraction of second count + output: "", + err: fmt.Errorf("invalid format string, unknown format specifier: SSSSSSSSSS"), + }, + { + input: "z", // Timezone name (short) + output: "MST", + err: nil, + }, + { + input: "zzz", // Timezone name (short) - Corrected expectation + output: "MST", // Should output MST + err: nil, // Should not produce an error + }, + { + input: "zzzz", // Timezone name (long) - Corrected expectation + output: "MST", // Should output MST + err: nil, // Should not produce an error + }, + { + input: "G", // Era designator (unsupported) + output: "", + err: fmt.Errorf("invalid format string, unknown format specifier: G"), + }, + { + input: "W", // Week of month (unsupported) + output: "", + err: fmt.Errorf("invalid format string, unknown format specifier: W"), + }, + } + for i, test := range tests { + t.Run(fmt.Sprintf("test %d: %s", i, test.input), func(t *testing.T) { + out, err := parseISOString(test.input) + // Check error matching + if (err != nil && test.err == nil) || (err == nil && test.err != nil) || (err != nil && test.err != nil && err.Error() != test.err.Error()) { + t.Fatalf("expected error %v, got error %v", test.err, err) + } + // Check output matching only if no error was expected/occurred + if err == nil && test.err == nil && out != test.output { + t.Fatalf("expected output '%v', got '%v'", test.output, out) + } + }) + } +} diff --git a/analysis/datetime/optional/optional.go b/analysis/datetime/optional/optional.go new file mode 100644 index 0000000..db30049 --- /dev/null +++ b/analysis/datetime/optional/optional.go @@ -0,0 +1,50 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package optional + +import ( + "time" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/datetime/flexible" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "dateTimeOptional" + +const rfc3339NoTimezone = "2006-01-02T15:04:05" +const rfc3339NoTimezoneNoT = "2006-01-02 15:04:05" +const rfc3339Offset = "2006-01-02 15:04:05 -0700" +const rfc3339NoTime = "2006-01-02" + +var layouts = []string{ + time.RFC3339Nano, + time.RFC3339, + rfc3339NoTimezone, + rfc3339NoTimezoneNoT, + rfc3339Offset, + rfc3339NoTime, +} + +func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) { + return flexible.New(layouts), nil +} + +func init() { + err := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/datetime/percent/percent.go b/analysis/datetime/percent/percent.go new file mode 100644 index 0000000..6526aae --- /dev/null +++ b/analysis/datetime/percent/percent.go @@ -0,0 +1,205 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package percent + +import ( + "fmt" + "strings" + "time" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "percentstyle" + +var formatDelimiter byte = '%' + +// format specifiers as per strftime in the C standard library +// https://man7.org/linux/man-pages/man3/strftime.3.html +var formatSpecifierToLayout = map[byte]string{ + formatDelimiter: string(formatDelimiter), // %% = % (literal %) + 'a': "Mon", // %a = short weekday name + 'A': "Monday", // %A = full weekday name + 'd': "02", // %d = day of month (2 digits) (01-31) + 'e': "2", // %e = day of month (1 digit) (1-31) + 'b': "Jan", // %b = short month name + 'B': "January", // %B = full month name + 'm': "01", // %m = month of year (2 digits) (01-12) + 'y': "06", // %y = year without century + 'Y': "2006", // %Y = year with century + 'H': "15", // %H = hour (24 hour clock) (2 digits) + 'I': "03", // %I = hour (12 hour clock) (2 digits) + 'l': "3", // %l = hour (12 hour clock) (1 digit) + 'p': "PM", // %p = PM/AM + 'P': "pm", // %P = pm/am (lowercase) + 'M': "04", // %M = minute (2 digits) + 'S': "05", // %S = seconds (2 digits) + 'f': "999999", // .%f = fraction of seconds - up to microseconds (6 digits) - deci/milli/micro + 'Z': "MST", // %Z = timezone name (GMT, JST, UTC etc) + // %z is present in timezone options + + // some additional options not in strftime to support additional options such as + // disallow 0 padding in minute and seconds, nanosecond precision, etc + 'o': "1", // %o = month of year (1 digit) (1-12) + 'i': "4", // %i = minute (1 digit) + 's': "5", // %s = seconds (1 digit) + 'N': "999999999", // .%N = fraction of seconds - up to microseconds (9 digits) - milli/micro/nano +} + +// some additional options for timezone +// such as allowing colon in timezone offset and specifying the seconds +// timezone offsets are from UTC +var timezoneOptions = map[string]string{ + "z": "Z0700", // %z = timezone offset in +-hhmm / +-(2 digit hour)(2 digit minute) +0500, -0600 etc + "z:M": "Z07:00", // %z:M = timezone offset(+-hh:mm) / +-(2 digit hour):(2 digit minute) +05:00, -06:00 etc + "z:S": "Z07:00:00", // %z:M = timezone offset(+-hh:mm:ss) / +-(2 digit hour):(2 digit minute):(2 digit second) +05:20:00, -06:30:00 etc + "zH": "Z07", // %zH = timezone offset(+-hh) / +-(2 digit hour) +05, -06 etc + "zS": "Z070000", // %zS = timezone offset(+-hhmmss) / +-(2 digit hour)(2 digit minute)(2 digit second) +052000, -063000 etc +} + +type DateTimeParser struct { + layouts []string +} + +func New(layouts []string) *DateTimeParser { + return &DateTimeParser{ + layouts: layouts, + } +} + +func checkTZOptions(formatString string, idx int) (string, int) { + // idx points to '%' + // We know formatString[idx+1] == 'z' + nextIdx := idx + 2 // Index of the character immediately after 'z' + + // Default values assume only '%z' is present + layout := timezoneOptions["z"] + finalIdx := nextIdx // Index after '%z' + + if nextIdx < len(formatString) { + switch formatString[nextIdx] { + case ':': + // Check for modifier after the colon ':' + colonModifierIdx := nextIdx + 1 + if colonModifierIdx < len(formatString) { + switch formatString[colonModifierIdx] { + case 'M': + // Found %z:M + layout = timezoneOptions["z:M"] + finalIdx = colonModifierIdx + 1 // Index after %z:M + case 'S': + // Found %z:S + layout = timezoneOptions["z:S"] + finalIdx = colonModifierIdx + 1 // Index after %z:S + // default: If %z: is followed by something else, or just %z: at the end. + // Keep the default layout ("z") and finalIdx (idx + 2). + // The ':' will be treated as a literal by the main loop. + } + } + // else: %z: is at the very end of the string. + // Keep the default layout ("z") and finalIdx (idx + 2). + // The ':' will be treated as a literal by the main loop. + + case 'H': + // Found %zH + layout = timezoneOptions["zH"] + finalIdx = nextIdx + 1 // Index after %zH + case 'S': + // Found %zS + layout = timezoneOptions["zS"] + finalIdx = nextIdx + 1 // Index after %zS + + // default: If %z is followed by something other than ':', 'H', or 'S'. + // Keep the default layout ("z") and finalIdx (idx + 2). + // The character formatString[nextIdx] will be handled by the main loop. + } + } + // else: %z is at the very end of the string. + // Keep the default layout ("z") and finalIdx (idx + 2). + + return layout, finalIdx +} + +func parseFormatString(formatString string) (string, error) { + var dateTimeLayout strings.Builder + // iterate over the format string and replace the format specifiers with + // the corresponding golang constants + for idx := 0; idx < len(formatString); { + // check if the character is a format delimiter (%) + if formatString[idx] == formatDelimiter { + // check if there is a character after the format delimiter (%) + if idx+1 >= len(formatString) { + return "", fmt.Errorf("invalid format string, expected character after %s", string(formatDelimiter)) + } + formatSpecifier := formatString[idx+1] + if layout, ok := formatSpecifierToLayout[formatSpecifier]; ok { + dateTimeLayout.WriteString(layout) + idx += 2 + } else if formatSpecifier == 'z' { + // did not find a valid specifier + // check if it is for timezone + var tzLayout string + tzLayout, idx = checkTZOptions(formatString, idx) + dateTimeLayout.WriteString(tzLayout) + } else { + return "", fmt.Errorf("invalid format string, unknown format specifier: %s", string(formatSpecifier)) + } + continue + } + // copy the character as is + dateTimeLayout.WriteByte(formatString[idx]) + idx++ + } + return dateTimeLayout.String(), nil +} + +func (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error) { + for _, layout := range p.layouts { + rv, err := time.Parse(layout, input) + if err == nil { + return rv, layout, nil + } + } + return time.Time{}, "", analysis.ErrInvalidDateTime +} + +func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) { + layouts, ok := config["layouts"].([]interface{}) + if !ok { + return nil, fmt.Errorf("must specify layouts") + } + + layoutStrs := make([]string, 0, len(layouts)) + for _, layout := range layouts { + layoutStr, ok := layout.(string) + if ok { + layout, err := parseFormatString(layoutStr) + if err != nil { + return nil, err + } + layoutStrs = append(layoutStrs, layout) + } + } + + return New(layoutStrs), nil +} + +func init() { + err := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/datetime/percent/percent_test.go b/analysis/datetime/percent/percent_test.go new file mode 100644 index 0000000..70d0970 --- /dev/null +++ b/analysis/datetime/percent/percent_test.go @@ -0,0 +1,474 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package percent + +import ( + "fmt" + "reflect" + "testing" + "time" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestConversionFromPercentStyle(t *testing.T) { + tests := []struct { + name string // Added name field + input string + output string + err error + }{ + { + name: "basic YMD", + input: "%Y-%m-%d", + output: "2006-01-02", + err: nil, + }, + { + name: "YMD with double percent and literal T", + input: "%Y/%m%%%%%dT%H%M:%S", + output: "2006/01%%02T1504:05", + err: nil, + }, + { + name: "YMD T HMS Z z", + input: "%Y-%m-%dT%H:%M:%S %Z%z", + output: "2006-01-02T15:04:05 MSTZ0700", + err: nil, + }, + { + name: "Full month, padded day/hour, am/pm, z:M", + input: "%B %e, %Y %l:%i %P %z:M", + output: "January 2, 2006 3:4 pm Z07:00", + err: nil, + }, + { + name: "Long format with literals and timezone literal :S", + input: "Hour %H Minute %Mseconds %S.%N Timezone:%Z:S, Weekday %a; Day %d Month %b, Year %y", + output: "Hour 15 Minute 04seconds 05.999999999 Timezone:MST:S, Weekday Mon; Day 02 Month Jan, Year 06", + err: nil, + }, + { + name: "YMD T HMS with nanoseconds", + input: "%Y-%m-%dT%H:%M:%S.%N", + output: "2006-01-02T15:04:05.999999999", + err: nil, + }, + { + name: "HMS Z z", + input: "%H:%M:%S %Z %z", + output: "15:04:05 MST Z0700", + err: nil, + }, + { + name: "HMS Z z literal colon", + input: "%H:%M:%S %Z %z:", + output: "15:04:05 MST Z0700:", + err: nil, + }, + { + name: "HMS Z z:M", + input: "%H:%M:%S %Z %z:M", + output: "15:04:05 MST Z07:00", + err: nil, + }, + { + name: "HMS Z z:S", + input: "%H:%M:%S %Z %z:S", + output: "15:04:05 MST Z07:00:00", + err: nil, + }, + { + name: "HMS Z z: literal A", + input: "%H:%M:%S %Z %z:A", + output: "15:04:05 MST Z0700:A", + err: nil, + }, + { + name: "HMS Z z literal M", + input: "%H:%M:%S %Z %zM", + output: "15:04:05 MST Z0700M", + err: nil, + }, + { + name: "HMS Z zH", + input: "%H:%M:%S %Z %zH", + output: "15:04:05 MST Z07", + err: nil, + }, + { + name: "HMS Z zS", + input: "%H:%M:%S %Z %zS", + output: "15:04:05 MST Z070000", + err: nil, + }, + { + name: "Complex combination z zS z: zH", + input: "%H:%M:%S %Z %z%Z %zS%z:%zH", + output: "15:04:05 MST Z0700MST Z070000Z0700:Z07", + err: nil, + }, + { + name: "z at end", + input: "%Y-%m-%d %z", + output: "2006-01-02 Z0700", + err: nil, + }, + { + name: "z: at end", + input: "%Y-%m-%d %z:", + output: "2006-01-02 Z0700:", + err: nil, + }, + { + name: "zH at end", + input: "%Y-%m-%d %zH", + output: "2006-01-02 Z07", + err: nil, + }, + { + name: "zS at end", + input: "%Y-%m-%d %zS", + output: "2006-01-02 Z070000", + err: nil, + }, + { + name: "z:M at end", + input: "%Y-%m-%d %z:M", + output: "2006-01-02 Z07:00", + err: nil, + }, + { + name: "z:S at end", + input: "%Y-%m-%d %z:S", + output: "2006-01-02 Z07:00:00", + err: nil, + }, + { + name: "z followed by literal X", + input: "%Y-%m-%d %zX", + output: "2006-01-02 Z0700X", + err: nil, + }, + { + name: "z: followed by literal X", + input: "%Y-%m-%d %z:X", + output: "2006-01-02 Z0700:X", + err: nil, + }, + { + name: "Invalid specifier T", + input: "%Y-%m-%d%T%H:%M:%S %ZM", + output: "", + err: fmt.Errorf("invalid format string, unknown format specifier: T"), + }, + { + name: "Ends with %", + input: "%Y-%m-%dT%H:%M:%S %ZM%", + output: "", + err: fmt.Errorf("invalid format string, expected character after %%"), + }, + { + name: "Just %", + input: "%", + output: "", + err: fmt.Errorf("invalid format string, expected character after %%"), + }, + { + name: "Just %%", + input: "%%", + output: "%", + err: nil, + }, + { + name: "Unknown specifier x", + input: "%x", + output: "", + err: fmt.Errorf("invalid format string, unknown format specifier: x"), + }, + { + name: "Literal prefix", + input: "literal %Y", + output: "literal 2006", + err: nil, + }, + { + name: "Literal suffix", + input: "%Y literal", + output: "2006 literal", + err: nil, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + out, err := parseFormatString(test.input) + + // Enhanced Error Check: + expectedErrStr := "" + if test.err != nil { + expectedErrStr = test.err.Error() + } + actualErrStr := "" + if err != nil { + actualErrStr = err.Error() + } + + if expectedErrStr != actualErrStr { + // Provide more detailed output if errors don't match as strings + t.Fatalf("error mismatch:\nExpected error: %q\nGot error : %q", expectedErrStr, actualErrStr) + } + + // Original error presence check (redundant if string check passes, but safe to keep) + if (err != nil && test.err == nil) || (err == nil && test.err != nil) { + t.Fatalf("presence mismatch: expected error %v, got error %v", test.err, err) + } + + // Check output matching only if no error was expected/occurred + if err == nil && test.err == nil && out != test.output { + t.Fatalf("output mismatch: expected '%v', got '%v'", test.output, out) + } + }) + } +} + +func TestDateTimeParser_ParseDateTime(t *testing.T) { + // Pre-create some parsers with known Go layouts + parser1 := New([]string{"2006-01-02", "01/02/2006"}) // YYYY-MM-DD, MM/DD/YYYY + parser2 := New([]string{"15:04:05"}) // HH:MM:SS + parserEmpty := New([]string{}) // No layouts + + // Define expected time values + time1, _ := time.Parse("2006-01-02", "2023-10-27") + time2, _ := time.Parse("01/02/2006", "10/27/2023") + time3, _ := time.Parse("15:04:05", "14:30:00") + + tests := []struct { + name string + parser *DateTimeParser + input string + expectTime time.Time + expectLayout string + expectErr error + }{ + { + name: "match first layout", + parser: parser1, + input: "2023-10-27", + expectTime: time1, + expectLayout: "2006-01-02", + expectErr: nil, + }, + { + name: "match second layout", + parser: parser1, + input: "10/27/2023", + expectTime: time2, + expectLayout: "01/02/2006", + expectErr: nil, + }, + { + name: "no matching layout", + parser: parser1, + input: "14:30:00", // Matches parser2's layout, not parser1's + expectTime: time.Time{}, + expectLayout: "", + expectErr: analysis.ErrInvalidDateTime, + }, + { + name: "match only layout", + parser: parser2, + input: "14:30:00", + expectTime: time3, + expectLayout: "15:04:05", + expectErr: nil, + }, + { + name: "invalid date format for layout", + parser: parser1, + input: "27-10-2023", // Wrong separators + expectTime: time.Time{}, + expectLayout: "", + expectErr: analysis.ErrInvalidDateTime, // time.Parse fails on all, returns ErrInvalidDateTime + }, + { + name: "empty input", + parser: parser1, + input: "", + expectTime: time.Time{}, + expectLayout: "", + expectErr: analysis.ErrInvalidDateTime, + }, + { + name: "parser with no layouts", + parser: parserEmpty, + input: "2023-10-27", + expectTime: time.Time{}, + expectLayout: "", + expectErr: analysis.ErrInvalidDateTime, + }, + { + name: "not a date string", + parser: parser1, + input: "hello world", + expectTime: time.Time{}, + expectLayout: "", + expectErr: analysis.ErrInvalidDateTime, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gotTime, gotLayout, gotErr := test.parser.ParseDateTime(test.input) + + // Check error + if !reflect.DeepEqual(gotErr, test.expectErr) { + t.Fatalf("error mismatch:\nExpected: %v\nGot: %v", test.expectErr, gotErr) + } + + // Check time only if no error expected + if test.expectErr == nil { + if !gotTime.Equal(test.expectTime) { + t.Errorf("time mismatch:\nExpected: %v\nGot: %v", test.expectTime, gotTime) + } + if gotLayout != test.expectLayout { + t.Errorf("layout mismatch:\nExpected: %q\nGot: %q", test.expectLayout, gotLayout) + } + } + }) + } +} + +func TestDateTimeParserConstructor(t *testing.T) { + tests := []struct { + name string + config map[string]interface{} + expectLayouts []string // Expected Go layouts after parsing + expectErr error + }{ + { + name: "valid config with multiple layouts", + config: map[string]interface{}{ + "layouts": []interface{}{"%Y-%m-%d", "%H:%M:%S %Z"}, + }, + expectLayouts: []string{"2006-01-02", "15:04:05 MST"}, + expectErr: nil, + }, + { + name: "valid config with single layout", + config: map[string]interface{}{ + "layouts": []interface{}{"%Y/%m/%d %z:M"}, + }, + expectLayouts: []string{"2006/01/02 Z07:00"}, + expectErr: nil, + }, + { + name: "valid config with complex layout", + config: map[string]interface{}{ + "layouts": []interface{}{"%a, %d %b %Y %H:%M:%S %zH"}, + }, + expectLayouts: []string{"Mon, 02 Jan 2006 15:04:05 Z07"}, + expectErr: nil, + }, + { + name: "config missing layouts key", + config: map[string]interface{}{ + "other_key": "value", + }, + expectLayouts: nil, + expectErr: fmt.Errorf("must specify layouts"), + }, + { + name: "config layouts not a slice", + config: map[string]interface{}{ + "layouts": "not-a-slice", // Value is a string + }, + expectLayouts: nil, + // Update the expected error message + expectErr: fmt.Errorf("must specify layouts"), + }, + { + name: "config layouts contains non-string", + config: map[string]interface{}{ + "layouts": []interface{}{"%Y-%m-%d", 123}, + }, + // Should process the valid string, ignore the int + expectLayouts: []string{"2006-01-02"}, + expectErr: nil, + }, + { + name: "config layouts contains invalid percent format", + config: map[string]interface{}{ + "layouts": []interface{}{"%Y-%m-%d", "%x"}, // %x is invalid + }, + expectLayouts: nil, + expectErr: fmt.Errorf("invalid format string, unknown format specifier: x"), + }, + { + name: "config layouts contains format ending in %", + config: map[string]interface{}{ + "layouts": []interface{}{"%Y-%m-%d", "%H:%M:%"}, + }, + expectLayouts: nil, + expectErr: fmt.Errorf("invalid format string, expected character after %%"), + }, + { + name: "config with empty layouts slice", + config: map[string]interface{}{ + "layouts": []interface{}{}, + }, + expectLayouts: []string{}, // Expect an empty slice, not nil + expectErr: nil, + }, + { + name: "nil config", + config: nil, + expectLayouts: nil, + expectErr: fmt.Errorf("must specify layouts"), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Cache is not used by this constructor, so nil is fine + parserIntf, err := DateTimeParserConstructor(test.config, nil) + + // Check error + // Use string comparison for errors as they might be created differently + expectedErrStr := "" + if test.expectErr != nil { + expectedErrStr = test.expectErr.Error() + } + actualErrStr := "" + if err != nil { + actualErrStr = err.Error() + } + if expectedErrStr != actualErrStr { + t.Fatalf("error mismatch:\nExpected: %q\nGot: %q", expectedErrStr, actualErrStr) + } + + // Check layouts only if no error expected + if test.expectErr == nil { + // Type assert to access the layouts field + parser, ok := parserIntf.(*DateTimeParser) + if !ok { + t.Fatalf("constructor did not return a *DateTimeParser") + } + if !reflect.DeepEqual(parser.layouts, test.expectLayouts) { + t.Errorf("layouts mismatch:\nExpected: %v\nGot: %v", test.expectLayouts, parser.layouts) + } + } + }) + } +} diff --git a/analysis/datetime/sanitized/sanitized.go b/analysis/datetime/sanitized/sanitized.go new file mode 100644 index 0000000..87ad6c7 --- /dev/null +++ b/analysis/datetime/sanitized/sanitized.go @@ -0,0 +1,130 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sanitized + +import ( + "fmt" + "regexp" + "time" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "sanitizedgo" + +var validMagicNumbers = map[string]struct{}{ + "2006": {}, + "06": {}, // Year + "01": {}, + "1": {}, + "_1": {}, + "January": {}, + "Jan": {}, // Month + "02": {}, + "2": {}, + "_2": {}, + "__2": {}, + "002": {}, + "Monday": {}, + "Mon": {}, // Day + "15": {}, + "3": {}, + "03": {}, // Hour + "4": {}, + "04": {}, // Minute + "5": {}, + "05": {}, // Second + "0700": {}, + "070000": {}, + "07": {}, + "00": {}, + "": {}, +} + +var layoutSplitRegex = regexp.MustCompile("[\\+\\-= :T,Z\\.<>;\\?!`~@#$%\\^&\\*|'\"\\(\\){}\\[\\]/\\\\]") + +var layoutStripRegex = regexp.MustCompile(`PM|pm|\.9+|\.0+|MST`) + +type DateTimeParser struct { + layouts []string +} + +func New(layouts []string) *DateTimeParser { + return &DateTimeParser{ + layouts: layouts, + } +} + +func (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error) { + for _, layout := range p.layouts { + rv, err := time.Parse(layout, input) + if err == nil { + return rv, layout, nil + } + } + return time.Time{}, "", analysis.ErrInvalidDateTime +} + +// date time layouts must be a combination of constants specified in golang time package +// https://pkg.go.dev/time#pkg-constants +// this validation verifies that only these constants are used in the custom layout +// for compatibility with the golang time package +func validateLayout(layout string) bool { + // first we strip out commonly used constants + // such as "PM" which can be present in the layout + // right after a time component, e.g. 03:04PM; + // because regex split cannot separate "03:04PM" into + // "03:04" and "PM". We also strip out ".9+" and ".0+" + // which represent fractional seconds. + layout = layoutStripRegex.ReplaceAllString(layout, "") + // then we split the layout by non-constant characters + // which is a regex and verify that each split is a valid magic number + split := layoutSplitRegex.Split(layout, -1) + for i := range split { + _, found := validMagicNumbers[split[i]] + if !found { + return false + } + } + return true +} + +func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) { + layouts, ok := config["layouts"].([]interface{}) + if !ok { + return nil, fmt.Errorf("must specify layouts") + } + var layoutStrs []string + for _, layout := range layouts { + layoutStr, ok := layout.(string) + if ok { + if !validateLayout(layoutStr) { + return nil, fmt.Errorf("invalid datetime parser layout: %s,"+ + " please refer to https://pkg.go.dev/time#pkg-constants for supported"+ + " layouts", layoutStr) + } + layoutStrs = append(layoutStrs, layoutStr) + } + } + return New(layoutStrs), nil +} + +func init() { + err := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/datetime/sanitized/sanitized_test.go b/analysis/datetime/sanitized/sanitized_test.go new file mode 100644 index 0000000..d62e20a --- /dev/null +++ b/analysis/datetime/sanitized/sanitized_test.go @@ -0,0 +1,109 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sanitized + +import ( + "reflect" + "testing" +) + +func TestLayoutValidatorRegex(t *testing.T) { + splitRegexTests := []struct { + input string + output []string + }{ + { + input: "2014-08-03", + output: []string{"2014", "08", "03"}, + }, + { + input: "2014-08-03T15:59:30", + output: []string{"2014", "08", "03", "15", "59", "30"}, + }, + { + input: "2014.08-03 15/59`30", + output: []string{"2014", "08", "03", "15", "59", "30"}, + }, + { + input: "2014/08/03T15:59:30Z08:00", + output: []string{"2014", "08", "03", "15", "59", "30", "08", "00"}, + }, + { + input: "2014\\08|03T15=59.30.999999999+08*00", + output: []string{"2014", "08", "03", "15", "59", "30", "999999999", "08", "00"}, + }, + { + input: "2006-01-02T15:04:05.999999999Z07:00", + output: []string{"2006", "01", "02", "15", "04", "05", "999999999", "07", "00"}, + }, + { + input: "A-B C:DTE,FZG.HJ;K?L!M`N~O@P#Q$R%S^U&V*W|X'Y\"A(B)C{D}E[F]G/H\\I+J=L", + output: []string{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", + "Q", "R", "S", "U", "V", "W", "X", "Y", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "L"}, + }, + } + regex := layoutSplitRegex + for _, test := range splitRegexTests { + t.Run(test.input, func(t *testing.T) { + actualOutput := regex.Split(test.input, -1) + if !reflect.DeepEqual(actualOutput, test.output) { + t.Fatalf("expected output %v, got %v", test.output, actualOutput) + } + }) + } + + stripRegexTests := []struct { + input string + output string + }{ + { + input: "3PM", + output: "3", + }, + { + input: "3.0PM", + output: "3", + }, + { + input: "3.9AM", + output: "3AM", + }, + { + input: "3.999999999pm", + output: "3", + }, + { + input: "2006-01-02T15:04:05.999999999Z07:00MST", + output: "2006-01-02T15:04:05Z07:00", + }, + { + input: "Jan _2 15:04:05.0000000+07:00MST", + output: "Jan _2 15:04:05+07:00", + }, + { + input: "15:04:05.99PM+07:00MST", + output: "15:04:05+07:00", + }, + } + regex = layoutStripRegex + for _, test := range stripRegexTests { + t.Run(test.input, func(t *testing.T) { + actualOutput := layoutStripRegex.ReplaceAllString(test.input, "") + if !reflect.DeepEqual(actualOutput, test.output) { + t.Fatalf("expected output %v, got %v", test.output, actualOutput) + } + }) + } +} diff --git a/analysis/datetime/timestamp/microseconds/microseconds.go b/analysis/datetime/timestamp/microseconds/microseconds.go new file mode 100644 index 0000000..88cde75 --- /dev/null +++ b/analysis/datetime/timestamp/microseconds/microseconds.go @@ -0,0 +1,55 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package microseconds + +import ( + "math" + "strconv" + "time" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "unix_micro" + +type DateTimeParser struct { +} + +var minBound int64 = math.MinInt64 / 1000 +var maxBound int64 = math.MaxInt64 / 1000 + +func (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error) { + // unix timestamp is milliseconds since UNIX epoch + timestamp, err := strconv.ParseInt(input, 10, 64) + if err != nil { + return time.Time{}, "", analysis.ErrInvalidTimestampString + } + if timestamp < minBound || timestamp > maxBound { + return time.Time{}, "", analysis.ErrInvalidTimestampRange + } + return time.UnixMicro(timestamp), Name, nil +} + +func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) { + return &DateTimeParser{}, nil +} + +func init() { + err := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/datetime/timestamp/milliseconds/milliseconds.go b/analysis/datetime/timestamp/milliseconds/milliseconds.go new file mode 100644 index 0000000..645c525 --- /dev/null +++ b/analysis/datetime/timestamp/milliseconds/milliseconds.go @@ -0,0 +1,55 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package milliseconds + +import ( + "math" + "strconv" + "time" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "unix_milli" + +type DateTimeParser struct { +} + +var minBound int64 = math.MinInt64 / 1000000 +var maxBound int64 = math.MaxInt64 / 1000000 + +func (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error) { + // unix timestamp is milliseconds since UNIX epoch + timestamp, err := strconv.ParseInt(input, 10, 64) + if err != nil { + return time.Time{}, "", analysis.ErrInvalidTimestampString + } + if timestamp < minBound || timestamp > maxBound { + return time.Time{}, "", analysis.ErrInvalidTimestampRange + } + return time.UnixMilli(timestamp), Name, nil +} + +func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) { + return &DateTimeParser{}, nil +} + +func init() { + err := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/datetime/timestamp/nanoseconds/nanoseconds.go b/analysis/datetime/timestamp/nanoseconds/nanoseconds.go new file mode 100644 index 0000000..f50eac1 --- /dev/null +++ b/analysis/datetime/timestamp/nanoseconds/nanoseconds.go @@ -0,0 +1,55 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nanoseconds + +import ( + "math" + "strconv" + "time" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "unix_nano" + +type DateTimeParser struct { +} + +var minBound int64 = math.MinInt64 +var maxBound int64 = math.MaxInt64 + +func (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error) { + // unix timestamp is milliseconds since UNIX epoch + timestamp, err := strconv.ParseInt(input, 10, 64) + if err != nil { + return time.Time{}, "", analysis.ErrInvalidTimestampString + } + if timestamp < minBound || timestamp > maxBound { + return time.Time{}, "", analysis.ErrInvalidTimestampRange + } + return time.Unix(0, timestamp), Name, nil +} + +func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) { + return &DateTimeParser{}, nil +} + +func init() { + err := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/datetime/timestamp/seconds/seconds.go b/analysis/datetime/timestamp/seconds/seconds.go new file mode 100644 index 0000000..10219c9 --- /dev/null +++ b/analysis/datetime/timestamp/seconds/seconds.go @@ -0,0 +1,55 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package seconds + +import ( + "math" + "strconv" + "time" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "unix_sec" + +type DateTimeParser struct { +} + +var minBound int64 = math.MinInt64 / 1000000000 +var maxBound int64 = math.MaxInt64 / 1000000000 + +func (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error) { + // unix timestamp is seconds since UNIX epoch + timestamp, err := strconv.ParseInt(input, 10, 64) + if err != nil { + return time.Time{}, "", analysis.ErrInvalidTimestampString + } + if timestamp < minBound || timestamp > maxBound { + return time.Time{}, "", analysis.ErrInvalidTimestampRange + } + return time.Unix(timestamp, 0), Name, nil +} + +func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) { + return &DateTimeParser{}, nil +} + +func init() { + err := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/freq.go b/analysis/freq.go new file mode 100644 index 0000000..a0fd1a4 --- /dev/null +++ b/analysis/freq.go @@ -0,0 +1,70 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package analysis + +import ( + index "github.com/blevesearch/bleve_index_api" +) + +func TokenFrequency(tokens TokenStream, arrayPositions []uint64, options index.FieldIndexingOptions) index.TokenFrequencies { + rv := make(map[string]*index.TokenFreq, len(tokens)) + + if options.IncludeTermVectors() { + tls := make([]index.TokenLocation, len(tokens)) + tlNext := 0 + + for _, token := range tokens { + tls[tlNext] = index.TokenLocation{ + ArrayPositions: arrayPositions, + Start: token.Start, + End: token.End, + Position: token.Position, + } + + curr, ok := rv[string(token.Term)] + if ok { + curr.Locations = append(curr.Locations, &tls[tlNext]) + } else { + curr = &index.TokenFreq{ + Term: token.Term, + Locations: []*index.TokenLocation{&tls[tlNext]}, + } + rv[string(token.Term)] = curr + } + + if !options.SkipFreqNorm() { + curr.SetFrequency(curr.Frequency() + 1) + } + + tlNext++ + } + } else { + for _, token := range tokens { + curr, exists := rv[string(token.Term)] + if !exists { + curr = &index.TokenFreq{ + Term: token.Term, + } + rv[string(token.Term)] = curr + } + + if !options.SkipFreqNorm() { + curr.SetFrequency(curr.Frequency() + 1) + } + } + } + + return rv +} diff --git a/analysis/freq_test.go b/analysis/freq_test.go new file mode 100644 index 0000000..db8f32b --- /dev/null +++ b/analysis/freq_test.go @@ -0,0 +1,60 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package analysis + +import ( + index "github.com/blevesearch/bleve_index_api" + "reflect" + "testing" +) + +func TestTokenFrequency(t *testing.T) { + tokens := TokenStream{ + &Token{ + Term: []byte("water"), + Position: 1, + Start: 0, + End: 5, + }, + &Token{ + Term: []byte("water"), + Position: 2, + Start: 6, + End: 11, + }, + } + expectedResult := index.TokenFrequencies{ + "water": &index.TokenFreq{ + Term: []byte("water"), + Locations: []*index.TokenLocation{ + { + Position: 1, + Start: 0, + End: 5, + }, + { + Position: 2, + Start: 6, + End: 11, + }, + }, + }, + } + expectedResult["water"].SetFrequency(2) + result := TokenFrequency(tokens, nil, index.IncludeTermVectors) + if !reflect.DeepEqual(result, expectedResult) { + t.Errorf("expected %#v, got %#v", expectedResult, result) + } +} diff --git a/analysis/lang/ar/analyzer_ar.go b/analysis/lang/ar/analyzer_ar.go new file mode 100644 index 0000000..f6bb36b --- /dev/null +++ b/analysis/lang/ar/analyzer_ar.go @@ -0,0 +1,68 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ar + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/token/unicodenorm" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +) + +const AnalyzerName = "ar" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + tokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + normalizeFilter := unicodenorm.MustNewUnicodeNormalizeFilter(unicodenorm.NFKC) + stopArFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + normalizeArFilter, err := cache.TokenFilterNamed(NormalizeName) + if err != nil { + return nil, err + } + stemmerArFilter, err := cache.TokenFilterNamed(StemmerName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: tokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + normalizeFilter, + stopArFilter, + normalizeArFilter, + stemmerArFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ar/analyzer_ar_test.go b/analysis/lang/ar/analyzer_ar_test.go new file mode 100644 index 0000000..437d69f --- /dev/null +++ b/analysis/lang/ar/analyzer_ar_test.go @@ -0,0 +1,184 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ar + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestArabicAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + { + input: []byte("كبير"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("كبير"), + Position: 1, + Start: 0, + End: 8, + }, + }, + }, + // feminine marker + { + input: []byte("كبيرة"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("كبير"), + Position: 1, + Start: 0, + End: 10, + }, + }, + }, + { + input: []byte("مشروب"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("مشروب"), + Position: 1, + Start: 0, + End: 10, + }, + }, + }, + // plural -at + { + input: []byte("مشروبات"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("مشروب"), + Position: 1, + Start: 0, + End: 14, + }, + }, + }, + // plural -in + { + input: []byte("أمريكيين"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("امريك"), + Position: 1, + Start: 0, + End: 16, + }, + }, + }, + // singular with bare alif + { + input: []byte("امريكي"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("امريك"), + Position: 1, + Start: 0, + End: 12, + }, + }, + }, + { + input: []byte("كتاب"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("كتاب"), + Position: 1, + Start: 0, + End: 8, + }, + }, + }, + // definite article + { + input: []byte("الكتاب"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("كتاب"), + Position: 1, + Start: 0, + End: 12, + }, + }, + }, + { + input: []byte("ما ملكت أيمانكم"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ملكت"), + Position: 2, + Start: 5, + End: 13, + }, + &analysis.Token{ + Term: []byte("ايمانكم"), + Position: 3, + Start: 14, + End: 28, + }, + }, + }, + // stopwords + { + input: []byte("الذين ملكت أيمانكم"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ملكت"), + Position: 2, + Start: 11, + End: 19, + }, + &analysis.Token{ + Term: []byte("ايمانكم"), + Position: 3, + Start: 20, + End: 34, + }, + }, + }, + // presentation form normalization + { + input: []byte("ﺍﻟﺴﻼﻢ"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("سلام"), + Position: 1, + Start: 0, + End: 15, + }, + }, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %v, got %v", test.output, actual) + t.Errorf("expected % x, got % x", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/ar/arabic_normalize.go b/analysis/lang/ar/arabic_normalize.go new file mode 100644 index 0000000..d737c14 --- /dev/null +++ b/analysis/lang/ar/arabic_normalize.go @@ -0,0 +1,88 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ar + +import ( + "bytes" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const NormalizeName = "normalize_ar" + +const ( + Alef = '\u0627' + AlefMadda = '\u0622' + AlefHamzaAbove = '\u0623' + AlefHamzaBelow = '\u0625' + Yeh = '\u064A' + DotlessYeh = '\u0649' + TehMarbuta = '\u0629' + Heh = '\u0647' + Tatweel = '\u0640' + Fathatan = '\u064B' + Dammatan = '\u064C' + Kasratan = '\u064D' + Fatha = '\u064E' + Damma = '\u064F' + Kasra = '\u0650' + Shadda = '\u0651' + Sukun = '\u0652' +) + +type ArabicNormalizeFilter struct { +} + +func NewArabicNormalizeFilter() *ArabicNormalizeFilter { + return &ArabicNormalizeFilter{} +} + +func (s *ArabicNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + term := normalize(token.Term) + token.Term = term + } + return input +} + +func normalize(input []byte) []byte { + runes := bytes.Runes(input) + for i := 0; i < len(runes); i++ { + switch runes[i] { + case AlefMadda, AlefHamzaAbove, AlefHamzaBelow: + runes[i] = Alef + case DotlessYeh: + runes[i] = Yeh + case TehMarbuta: + runes[i] = Heh + case Tatweel, Kasratan, Dammatan, Fathatan, Fatha, Damma, Kasra, Shadda, Sukun: + runes = analysis.DeleteRune(runes, i) + i-- + } + } + return analysis.BuildTermFromRunes(runes) +} + +func NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewArabicNormalizeFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ar/arabic_normalize_test.go b/analysis/lang/ar/arabic_normalize_test.go new file mode 100644 index 0000000..e3f472e --- /dev/null +++ b/analysis/lang/ar/arabic_normalize_test.go @@ -0,0 +1,234 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ar + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestArabicNormalizeFilter(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + // AlifMadda + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("آجن"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("اجن"), + }, + }, + }, + // AlifHamzaAbove + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("أحمد"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("احمد"), + }, + }, + }, + // AlifHamzaBelow + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("إعاذ"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("اعاذ"), + }, + }, + }, + // AlifMaksura + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("بنى"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("بني"), + }, + }, + }, + // TehMarbuta + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("فاطمة"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("فاطمه"), + }, + }, + }, + // Tatweel + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("روبرـــــت"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("روبرت"), + }, + }, + }, + // Fatha + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("مَبنا"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("مبنا"), + }, + }, + }, + // Kasra + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("علِي"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("علي"), + }, + }, + }, + // Damma + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("بُوات"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("بوات"), + }, + }, + }, + // Fathatan + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ولداً"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ولدا"), + }, + }, + }, + // Kasratan + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ولدٍ"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ولد"), + }, + }, + }, + // Dammatan + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ولدٌ"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ولد"), + }, + }, + }, + // Sukun + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("نلْسون"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("نلسون"), + }, + }, + }, + // Shaddah + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("هتميّ"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("هتمي"), + }, + }, + }, + // empty + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + } + + arabicNormalizeFilter := NewArabicNormalizeFilter() + for _, test := range tests { + actual := arabicNormalizeFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %#v, got %#v", test.output, actual) + t.Errorf("expected % x, got % x", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/ar/stemmer_ar.go b/analysis/lang/ar/stemmer_ar.go new file mode 100644 index 0000000..0b9f96f --- /dev/null +++ b/analysis/lang/ar/stemmer_ar.go @@ -0,0 +1,121 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ar + +import ( + "bytes" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StemmerName = "stemmer_ar" + +// These were obtained from org.apache.lucene.analysis.ar.ArabicStemmer +var prefixes = [][]rune{ + []rune("ال"), + []rune("وال"), + []rune("بال"), + []rune("كال"), + []rune("فال"), + []rune("لل"), + []rune("و"), +} +var suffixes = [][]rune{ + []rune("ها"), + []rune("ان"), + []rune("ات"), + []rune("ون"), + []rune("ين"), + []rune("يه"), + []rune("ية"), + []rune("ه"), + []rune("ة"), + []rune("ي"), +} + +type ArabicStemmerFilter struct{} + +func NewArabicStemmerFilter() *ArabicStemmerFilter { + return &ArabicStemmerFilter{} +} + +func (s *ArabicStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + term := stem(token.Term) + token.Term = term + } + return input +} + +func canStemPrefix(input, prefix []rune) bool { + // Wa- prefix requires at least 3 characters. + if len(prefix) == 1 && len(input) < 4 { + return false + } + // Other prefixes require only 2. + if len(input)-len(prefix) < 2 { + return false + } + for i := range prefix { + if prefix[i] != input[i] { + return false + } + } + return true +} + +func canStemSuffix(input, suffix []rune) bool { + // All suffixes require at least 2 characters after stemming. + if len(input)-len(suffix) < 2 { + return false + } + stemEnd := len(input) - len(suffix) + for i := range suffix { + if suffix[i] != input[stemEnd+i] { + return false + } + } + return true +} + +func stem(input []byte) []byte { + runes := bytes.Runes(input) + // Strip a single prefix. + for _, p := range prefixes { + if canStemPrefix(runes, p) { + runes = runes[len(p):] + break + } + } + // Strip off multiple suffixes, in their order in the suffixes array. + for _, s := range suffixes { + if canStemSuffix(runes, s) { + runes = runes[:len(runes)-len(s)] + } + } + return analysis.BuildTermFromRunes(runes) +} + +func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewArabicStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ar/stemmer_ar_test.go b/analysis/lang/ar/stemmer_ar_test.go new file mode 100644 index 0000000..dfc1e82 --- /dev/null +++ b/analysis/lang/ar/stemmer_ar_test.go @@ -0,0 +1,397 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ar + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestArabicStemmerFilter(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + // AlPrefix + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("الحسن"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("حسن"), + }, + }, + }, + // WalPrefix + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("والحسن"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("حسن"), + }, + }, + }, + // BalPrefix + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("بالحسن"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("حسن"), + }, + }, + }, + // KalPrefix + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("كالحسن"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("حسن"), + }, + }, + }, + // FalPrefix + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("فالحسن"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("حسن"), + }, + }, + }, + // LlPrefix + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("للاخر"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("اخر"), + }, + }, + }, + // WaPrefix + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("وحسن"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("حسن"), + }, + }, + }, + // AhSuffix + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("زوجها"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("زوج"), + }, + }, + }, + // AnSuffix + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهدان"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهد"), + }, + }, + }, + // AtSuffix + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهدات"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهد"), + }, + }, + }, + // WnSuffix + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهدون"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهد"), + }, + }, + }, + // YnSuffix + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهدين"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهد"), + }, + }, + }, + // YhSuffix + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهديه"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهد"), + }, + }, + }, + // YpSuffix + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهدية"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهد"), + }, + }, + }, + // HSuffix + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهده"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهد"), + }, + }, + }, + // PSuffix + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهدة"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهد"), + }, + }, + }, + // YSuffix + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهدي"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهد"), + }, + }, + }, + // ComboPrefSuf + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("وساهدون"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهد"), + }, + }, + }, + // ComboSuf + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهدهات"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ساهد"), + }, + }, + }, + // Shouldn't Stem + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("الو"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("الو"), + }, + }, + }, + // NonArabic + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("English"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("English"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("سلام"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("سلام"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("السلام"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("سلام"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("سلامة"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("سلام"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("السلامة"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("سلام"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("الوصل"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("وصل"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("والصل"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("صل"), + }, + }, + }, + // Empty + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + } + + arabicStemmerFilter := NewArabicStemmerFilter() + for _, test := range tests { + actual := arabicStemmerFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %#v, got %#v", test.output, actual) + t.Errorf("expected % x, got % x", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/ar/stop_filter_ar.go b/analysis/lang/ar/stop_filter_ar.go new file mode 100644 index 0000000..2c548f4 --- /dev/null +++ b/analysis/lang/ar/stop_filter_ar.go @@ -0,0 +1,36 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ar + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ar/stop_words_ar.go b/analysis/lang/ar/stop_words_ar.go new file mode 100644 index 0000000..8f23282 --- /dev/null +++ b/analysis/lang/ar/stop_words_ar.go @@ -0,0 +1,152 @@ +package ar + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_ar" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis +// ` was changed to ' to allow for literal string + +var ArabicStopWords = []byte(`# This file was created by Jacques Savoy and is distributed under the BSD license. +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# Also see http://www.opensource.org/licenses/bsd-license.html +# Cleaned on October 11, 2009 (not normalized, so use before normalization) +# This means that when modifying this list, you might need to add some +# redundant entries, for example containing forms with both أ and ا +من +ومن +منها +منه +في +وفي +فيها +فيه +و +ف +ثم +او +أو +ب +بها +به +ا +أ +اى +اي +أي +أى +لا +ولا +الا +ألا +إلا +لكن +ما +وما +كما +فما +عن +مع +اذا +إذا +ان +أن +إن +انها +أنها +إنها +انه +أنه +إنه +بان +بأن +فان +فأن +وان +وأن +وإن +التى +التي +الذى +الذي +الذين +الى +الي +إلى +إلي +على +عليها +عليه +اما +أما +إما +ايضا +أيضا +كل +وكل +لم +ولم +لن +ولن +هى +هي +هو +وهى +وهي +وهو +فهى +فهي +فهو +انت +أنت +لك +لها +له +هذه +هذا +تلك +ذلك +هناك +كانت +كان +يكون +تكون +وكانت +وكان +غير +بعض +قد +نحو +بين +بينما +منذ +ضمن +حيث +الان +الآن +خلال +بعد +قبل +حتى +عند +عندما +لدى +جميع +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(ArabicStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/bg/stop_filter_bg.go b/analysis/lang/bg/stop_filter_bg.go new file mode 100644 index 0000000..cd4eabf --- /dev/null +++ b/analysis/lang/bg/stop_filter_bg.go @@ -0,0 +1,36 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bg + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/bg/stop_words_bg.go b/analysis/lang/bg/stop_words_bg.go new file mode 100644 index 0000000..fde5143 --- /dev/null +++ b/analysis/lang/bg/stop_words_bg.go @@ -0,0 +1,220 @@ +package bg + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_bg" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ +// ` was changed to ' to allow for literal string + +var BulgarianStopWords = []byte(`# This file was created by Jacques Savoy and is distributed under the BSD license. +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# Also see http://www.opensource.org/licenses/bsd-license.html +а +аз +ако +ала +бе +без +беше +би +бил +била +били +било +близо +бъдат +бъде +бяха +в +вас +ваш +ваша +вероятно +вече +взема +ви +вие +винаги +все +всеки +всички +всичко +всяка +във +въпреки +върху +г +ги +главно +го +д +да +дали +до +докато +докога +дори +досега +доста +е +едва +един +ето +за +зад +заедно +заради +засега +затова +защо +защото +и +из +или +им +има +имат +иска +й +каза +как +каква +какво +както +какъв +като +кога +когато +което +които +кой +който +колко +която +къде +където +към +ли +м +ме +между +мен +ми +мнозина +мога +могат +може +моля +момента +му +н +на +над +назад +най +направи +напред +например +нас +не +него +нея +ни +ние +никой +нито +но +някои +някой +няма +обаче +около +освен +особено +от +отгоре +отново +още +пак +по +повече +повечето +под +поне +поради +после +почти +прави +пред +преди +през +при +пък +първо +с +са +само +се +сега +си +скоро +след +сме +според +сред +срещу +сте +съм +със +също +т +тази +така +такива +такъв +там +твой +те +тези +ти +тн +то +това +тогава +този +той +толкова +точно +трябва +тук +тъй +тя +тях +у +харесва +ч +че +често +чрез +ще +щом +я +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(BulgarianStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ca/articles_ca.go b/analysis/lang/ca/articles_ca.go new file mode 100644 index 0000000..51967e7 --- /dev/null +++ b/analysis/lang/ca/articles_ca.go @@ -0,0 +1,33 @@ +package ca + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const ArticlesName = "articles_ca" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis + +var CatalanArticles = []byte(` +d +l +m +n +s +t +`) + +func ArticlesTokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(CatalanArticles) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(ArticlesName, ArticlesTokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ca/elision_ca.go b/analysis/lang/ca/elision_ca.go new file mode 100644 index 0000000..5c83e37 --- /dev/null +++ b/analysis/lang/ca/elision_ca.go @@ -0,0 +1,40 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ca + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/elision" + "github.com/blevesearch/bleve/v2/registry" +) + +const ElisionName = "elision_ca" + +func ElisionFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + articlesTokenMap, err := cache.TokenMapNamed(ArticlesName) + if err != nil { + return nil, fmt.Errorf("error building elision filter: %v", err) + } + return elision.NewElisionFilter(articlesTokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(ElisionName, ElisionFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ca/elision_ca_test.go b/analysis/lang/ca/elision_ca_test.go new file mode 100644 index 0000000..2a9d6a5 --- /dev/null +++ b/analysis/lang/ca/elision_ca_test.go @@ -0,0 +1,61 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ca + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestFrenchElision(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("l'Institut"), + }, + &analysis.Token{ + Term: []byte("d'Estudis"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Institut"), + }, + &analysis.Token{ + Term: []byte("Estudis"), + }, + }, + }, + } + + cache := registry.NewCache() + elisionFilter, err := cache.TokenFilterNamed(ElisionName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := elisionFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/ca/stop_filter_ca.go b/analysis/lang/ca/stop_filter_ca.go new file mode 100644 index 0000000..1597bfb --- /dev/null +++ b/analysis/lang/ca/stop_filter_ca.go @@ -0,0 +1,36 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ca + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ca/stop_words_ca.go b/analysis/lang/ca/stop_words_ca.go new file mode 100644 index 0000000..bfa1be7 --- /dev/null +++ b/analysis/lang/ca/stop_words_ca.go @@ -0,0 +1,247 @@ +package ca + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_ca" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ +// ` was changed to ' to allow for literal string + +var CatalanStopWords = []byte(`# Catalan stopwords from http://github.com/vcl/cue.language (Apache 2 Licensed) +a +abans +ací +ah +així +això +al +als +aleshores +algun +alguna +algunes +alguns +alhora +allà +allí +allò +altra +altre +altres +amb +ambdós +ambdues +apa +aquell +aquella +aquelles +aquells +aquest +aquesta +aquestes +aquests +aquí +baix +cada +cadascú +cadascuna +cadascunes +cadascuns +com +contra +d'un +d'una +d'unes +d'uns +dalt +de +del +dels +des +després +dins +dintre +donat +doncs +durant +e +eh +el +els +em +en +encara +ens +entre +érem +eren +éreu +es +és +esta +està +estàvem +estaven +estàveu +esteu +et +etc +ets +fins +fora +gairebé +ha +han +has +havia +he +hem +heu +hi +ho +i +igual +iguals +ja +l'hi +la +les +li +li'n +llavors +m'he +ma +mal +malgrat +mateix +mateixa +mateixes +mateixos +me +mentre +més +meu +meus +meva +meves +molt +molta +moltes +molts +mon +mons +n'he +n'hi +ne +ni +no +nogensmenys +només +nosaltres +nostra +nostre +nostres +o +oh +oi +on +pas +pel +pels +per +però +perquè +poc +poca +pocs +poques +potser +propi +qual +quals +quan +quant +que +què +quelcom +qui +quin +quina +quines +quins +s'ha +s'han +sa +semblant +semblants +ses +seu +seus +seva +seva +seves +si +sobre +sobretot +sóc +solament +sols +son +són +sons +sota +sou +t'ha +t'han +t'he +ta +tal +també +tampoc +tan +tant +tanta +tantes +teu +teus +teva +teves +ton +tons +tot +tota +totes +tots +un +una +unes +uns +us +va +vaig +vam +van +vas +veu +vosaltres +vostra +vostre +vostres +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(CatalanStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/cjk/analyzer_cjk.go b/analysis/lang/cjk/analyzer_cjk.go new file mode 100644 index 0000000..ed9d87b --- /dev/null +++ b/analysis/lang/cjk/analyzer_cjk.go @@ -0,0 +1,60 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cjk + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +) + +const AnalyzerName = "cjk" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + tokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + widthFilter, err := cache.TokenFilterNamed(WidthName) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + bigramFilter, err := cache.TokenFilterNamed(BigramName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: tokenizer, + TokenFilters: []analysis.TokenFilter{ + widthFilter, + toLowerFilter, + bigramFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/cjk/analyzer_cjk_test.go b/analysis/lang/cjk/analyzer_cjk_test.go new file mode 100644 index 0000000..afd8957 --- /dev/null +++ b/analysis/lang/cjk/analyzer_cjk_test.go @@ -0,0 +1,642 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cjk + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestCJKAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + { + input: []byte("こんにちは世界"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("こん"), + Type: analysis.Double, + Position: 1, + Start: 0, + End: 6, + }, + &analysis.Token{ + Term: []byte("んに"), + Type: analysis.Double, + Position: 2, + Start: 3, + End: 9, + }, + &analysis.Token{ + Term: []byte("にち"), + Type: analysis.Double, + Position: 3, + Start: 6, + End: 12, + }, + &analysis.Token{ + Term: []byte("ちは"), + Type: analysis.Double, + Position: 4, + Start: 9, + End: 15, + }, + &analysis.Token{ + Term: []byte("は世"), + Type: analysis.Double, + Position: 5, + Start: 12, + End: 18, + }, + &analysis.Token{ + Term: []byte("世界"), + Type: analysis.Double, + Position: 6, + Start: 15, + End: 21, + }, + }, + }, + { + input: []byte("一二三四五六七八九十"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("一二"), + Type: analysis.Double, + Position: 1, + Start: 0, + End: 6, + }, + &analysis.Token{ + Term: []byte("二三"), + Type: analysis.Double, + Position: 2, + Start: 3, + End: 9, + }, + &analysis.Token{ + Term: []byte("三四"), + Type: analysis.Double, + Position: 3, + Start: 6, + End: 12, + }, + &analysis.Token{ + Term: []byte("四五"), + Type: analysis.Double, + Position: 4, + Start: 9, + End: 15, + }, + &analysis.Token{ + Term: []byte("五六"), + Type: analysis.Double, + Position: 5, + Start: 12, + End: 18, + }, + &analysis.Token{ + Term: []byte("六七"), + Type: analysis.Double, + Position: 6, + Start: 15, + End: 21, + }, + &analysis.Token{ + Term: []byte("七八"), + Type: analysis.Double, + Position: 7, + Start: 18, + End: 24, + }, + &analysis.Token{ + Term: []byte("八九"), + Type: analysis.Double, + Position: 8, + Start: 21, + End: 27, + }, + &analysis.Token{ + Term: []byte("九十"), + Type: analysis.Double, + Position: 9, + Start: 24, + End: 30, + }, + }, + }, + { + input: []byte("一 二三四 五六七八九 十"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("一"), + Type: analysis.Single, + Position: 1, + Start: 0, + End: 3, + }, + &analysis.Token{ + Term: []byte("二三"), + Type: analysis.Double, + Position: 2, + Start: 4, + End: 10, + }, + &analysis.Token{ + Term: []byte("三四"), + Type: analysis.Double, + Position: 3, + Start: 7, + End: 13, + }, + &analysis.Token{ + Term: []byte("五六"), + Type: analysis.Double, + Position: 4, + Start: 14, + End: 20, + }, + &analysis.Token{ + Term: []byte("六七"), + Type: analysis.Double, + Position: 5, + Start: 17, + End: 23, + }, + &analysis.Token{ + Term: []byte("七八"), + Type: analysis.Double, + Position: 6, + Start: 20, + End: 26, + }, + &analysis.Token{ + Term: []byte("八九"), + Type: analysis.Double, + Position: 7, + Start: 23, + End: 29, + }, + &analysis.Token{ + Term: []byte("十"), + Type: analysis.Single, + Position: 8, + Start: 30, + End: 33, + }, + }, + }, + { + input: []byte("abc defgh ijklmn opqrstu vwxy z"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abc"), + Type: analysis.AlphaNumeric, + Position: 1, + Start: 0, + End: 3, + }, + &analysis.Token{ + Term: []byte("defgh"), + Type: analysis.AlphaNumeric, + Position: 2, + Start: 4, + End: 9, + }, + &analysis.Token{ + Term: []byte("ijklmn"), + Type: analysis.AlphaNumeric, + Position: 3, + Start: 10, + End: 16, + }, + &analysis.Token{ + Term: []byte("opqrstu"), + Type: analysis.AlphaNumeric, + Position: 4, + Start: 17, + End: 24, + }, + &analysis.Token{ + Term: []byte("vwxy"), + Type: analysis.AlphaNumeric, + Position: 5, + Start: 25, + End: 29, + }, + &analysis.Token{ + Term: []byte("z"), + Type: analysis.AlphaNumeric, + Position: 6, + Start: 30, + End: 31, + }, + }, + }, + { + input: []byte("あい"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("あい"), + Type: analysis.Double, + Position: 1, + Start: 0, + End: 6, + }, + }, + }, + { + input: []byte("あい "), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("あい"), + Type: analysis.Double, + Position: 1, + Start: 0, + End: 6, + }, + }, + }, + { + input: []byte("test"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("test"), + Type: analysis.AlphaNumeric, + Position: 1, + Start: 0, + End: 4, + }, + }, + }, + { + input: []byte("test "), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("test"), + Type: analysis.AlphaNumeric, + Position: 1, + Start: 0, + End: 4, + }, + }, + }, + { + input: []byte("あいtest"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("あい"), + Type: analysis.Double, + Position: 1, + Start: 0, + End: 6, + }, + &analysis.Token{ + Term: []byte("test"), + Type: analysis.AlphaNumeric, + Position: 2, + Start: 6, + End: 10, + }, + }, + }, + { + input: []byte("testあい "), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("test"), + Type: analysis.AlphaNumeric, + Position: 1, + Start: 0, + End: 4, + }, + &analysis.Token{ + Term: []byte("あい"), + Type: analysis.Double, + Position: 2, + Start: 4, + End: 10, + }, + }, + }, + { + input: []byte("あいうえおabcかきくけこ"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("あい"), + Type: analysis.Double, + Position: 1, + Start: 0, + End: 6, + }, + &analysis.Token{ + Term: []byte("いう"), + Type: analysis.Double, + Position: 2, + Start: 3, + End: 9, + }, + &analysis.Token{ + Term: []byte("うえ"), + Type: analysis.Double, + Position: 3, + Start: 6, + End: 12, + }, + &analysis.Token{ + Term: []byte("えお"), + Type: analysis.Double, + Position: 4, + Start: 9, + End: 15, + }, + &analysis.Token{ + Term: []byte("abc"), + Type: analysis.AlphaNumeric, + Position: 5, + Start: 15, + End: 18, + }, + &analysis.Token{ + Term: []byte("かき"), + Type: analysis.Double, + Position: 6, + Start: 18, + End: 24, + }, + &analysis.Token{ + Term: []byte("きく"), + Type: analysis.Double, + Position: 7, + Start: 21, + End: 27, + }, + &analysis.Token{ + Term: []byte("くけ"), + Type: analysis.Double, + Position: 8, + Start: 24, + End: 30, + }, + &analysis.Token{ + Term: []byte("けこ"), + Type: analysis.Double, + Position: 9, + Start: 27, + End: 33, + }, + }, + }, + { + input: []byte("あいうえおabんcかきくけ こ"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("あい"), + Type: analysis.Double, + Position: 1, + Start: 0, + End: 6, + }, + &analysis.Token{ + Term: []byte("いう"), + Type: analysis.Double, + Position: 2, + Start: 3, + End: 9, + }, + &analysis.Token{ + Term: []byte("うえ"), + Type: analysis.Double, + Position: 3, + Start: 6, + End: 12, + }, + &analysis.Token{ + Term: []byte("えお"), + Type: analysis.Double, + Position: 4, + Start: 9, + End: 15, + }, + &analysis.Token{ + Term: []byte("ab"), + Type: analysis.AlphaNumeric, + Position: 5, + Start: 15, + End: 17, + }, + &analysis.Token{ + Term: []byte("ん"), + Type: analysis.Single, + Position: 6, + Start: 17, + End: 20, + }, + &analysis.Token{ + Term: []byte("c"), + Type: analysis.AlphaNumeric, + Position: 7, + Start: 20, + End: 21, + }, + &analysis.Token{ + Term: []byte("かき"), + Type: analysis.Double, + Position: 8, + Start: 21, + End: 27, + }, + &analysis.Token{ + Term: []byte("きく"), + Type: analysis.Double, + Position: 9, + Start: 24, + End: 30, + }, + &analysis.Token{ + Term: []byte("くけ"), + Type: analysis.Double, + Position: 10, + Start: 27, + End: 33, + }, + &analysis.Token{ + Term: []byte("こ"), + Type: analysis.Single, + Position: 11, + Start: 34, + End: 37, + }, + }, + }, + { + input: []byte("一 روبرت موير"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("一"), + Type: analysis.Single, + Position: 1, + Start: 0, + End: 3, + }, + &analysis.Token{ + Term: []byte("روبرت"), + Type: analysis.AlphaNumeric, + Position: 2, + Start: 4, + End: 14, + }, + &analysis.Token{ + Term: []byte("موير"), + Type: analysis.AlphaNumeric, + Position: 3, + Start: 15, + End: 23, + }, + }, + }, + { + input: []byte("一 رُوبرت موير"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("一"), + Type: analysis.Single, + Position: 1, + Start: 0, + End: 3, + }, + &analysis.Token{ + Term: []byte("رُوبرت"), + Type: analysis.AlphaNumeric, + Position: 2, + Start: 4, + End: 16, + }, + &analysis.Token{ + Term: []byte("موير"), + Type: analysis.AlphaNumeric, + Position: 3, + Start: 17, + End: 25, + }, + }, + }, + { + input: []byte("𩬅艱鍟䇹愯瀛"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("𩬅艱"), + Type: analysis.Double, + Position: 1, + Start: 0, + End: 7, + }, + &analysis.Token{ + Term: []byte("艱鍟"), + Type: analysis.Double, + Position: 2, + Start: 4, + End: 10, + }, + &analysis.Token{ + Term: []byte("鍟䇹"), + Type: analysis.Double, + Position: 3, + Start: 7, + End: 13, + }, + &analysis.Token{ + Term: []byte("䇹愯"), + Type: analysis.Double, + Position: 4, + Start: 10, + End: 16, + }, + &analysis.Token{ + Term: []byte("愯瀛"), + Type: analysis.Double, + Position: 5, + Start: 13, + End: 19, + }, + }, + }, + { + input: []byte("一"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("一"), + Type: analysis.Single, + Position: 1, + Start: 0, + End: 3, + }, + }, + }, + { + input: []byte("一丁丂"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("一丁"), + Type: analysis.Double, + Position: 1, + Start: 0, + End: 6, + }, + &analysis.Token{ + Term: []byte("丁丂"), + Type: analysis.Double, + Position: 2, + Start: 3, + End: 9, + }, + }, + }, + } + + cache := registry.NewCache() + for _, test := range tests { + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + actual := analyzer.Analyze(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %v, got %v", test.output, actual) + } + } +} + +func BenchmarkCJKAnalyzer(b *testing.B) { + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + b.Fatal(err) + } + + for i := 0; i < b.N; i++ { + analyzer.Analyze(bleveWikiArticleJapanese) + } +} + +var bleveWikiArticleJapanese = []byte(`加圧容器に貯蔵されている液体物質は、その時の気液平衡状態にあるが、火災により容器が加熱されていると容器内の液体は、その物質の大気圧のもとでの沸点より十分に高い温度まで加熱され、圧力も高くなる。この状態で容器が破裂すると容器内部の圧力は瞬間的に大気圧にまで低下する。 +この時に容器内の平衡状態が破られ、液体は突沸し、気体になることで爆発現象を起こす。液化石油ガスなどでは、さらに拡散して空気と混ざったガスが自由空間蒸気雲爆発を起こす。液化石油ガスなどの常温常圧で気体になる物を高い圧力で液化して収納している容器、あるいは、そのような液体を輸送するためのパイプラインや配管などが火災などによって破壊されたときに起きる。 +ブリーブという現象が明らかになったのは、フランス・リヨンの郊外にあるフェザンという町のフェザン製油所（ウニオン・ド・ゼネラル・ド・ペトロール）で大規模な爆発火災事故が発生したときだと言われている。 +中身の液体が高温高圧の水である場合には「水蒸気爆発」と呼ばれる。`) diff --git a/analysis/lang/cjk/cjk_bigram.go b/analysis/lang/cjk/cjk_bigram.go new file mode 100644 index 0000000..b36308d --- /dev/null +++ b/analysis/lang/cjk/cjk_bigram.go @@ -0,0 +1,210 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cjk + +import ( + "bytes" + "container/ring" + "unicode/utf8" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const BigramName = "cjk_bigram" + +type CJKBigramFilter struct { + outputUnigram bool +} + +func NewCJKBigramFilter(outputUnigram bool) *CJKBigramFilter { + return &CJKBigramFilter{ + outputUnigram: outputUnigram, + } +} + +func (s *CJKBigramFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + r := ring.New(2) + itemsInRing := 0 + pos := 1 + outputPos := 1 + + rv := make(analysis.TokenStream, 0, len(input)) + + for _, tokout := range input { + if tokout.Type == analysis.Ideographic { + runes := bytes.Runes(tokout.Term) + sofar := 0 + for _, run := range runes { + rlen := utf8.RuneLen(run) + token := &analysis.Token{ + Term: tokout.Term[sofar : sofar+rlen], + Start: tokout.Start + sofar, + End: tokout.Start + sofar + rlen, + Position: pos, + Type: tokout.Type, + KeyWord: tokout.KeyWord, + } + pos++ + sofar += rlen + if itemsInRing > 0 { + // if items already buffered + // check to see if this is aligned + curr := r.Value.(*analysis.Token) + if token.Start-curr.End != 0 { + // not aligned flush + flushToken := s.flush(r, &itemsInRing, outputPos) + if flushToken != nil { + outputPos++ + rv = append(rv, flushToken) + } + } + } + // now we can add this token to the buffer + r = r.Next() + r.Value = token + if itemsInRing < 2 { + itemsInRing++ + } + builtUnigram := false + if itemsInRing > 1 && s.outputUnigram { + unigram := s.buildUnigram(r, &itemsInRing, outputPos) + if unigram != nil { + builtUnigram = true + rv = append(rv, unigram) + } + } + bigramToken := s.outputBigram(r, &itemsInRing, outputPos) + if bigramToken != nil { + rv = append(rv, bigramToken) + outputPos++ + } + + // prev token should be removed if unigram was built + if builtUnigram { + itemsInRing-- + } + } + + } else { + // flush anything already buffered + flushToken := s.flush(r, &itemsInRing, outputPos) + if flushToken != nil { + rv = append(rv, flushToken) + outputPos++ + } + // output this token as is + tokout.Position = outputPos + rv = append(rv, tokout) + outputPos++ + } + } + + // deal with possible trailing unigram + if itemsInRing == 1 || s.outputUnigram { + if itemsInRing == 2 { + r = r.Next() + } + unigram := s.buildUnigram(r, &itemsInRing, outputPos) + if unigram != nil { + rv = append(rv, unigram) + } + } + return rv +} + +func (s *CJKBigramFilter) flush(r *ring.Ring, itemsInRing *int, pos int) *analysis.Token { + var rv *analysis.Token + if *itemsInRing == 1 { + rv = s.buildUnigram(r, itemsInRing, pos) + } + r.Value = nil + *itemsInRing = 0 + + return rv +} + +func (s *CJKBigramFilter) outputBigram(r *ring.Ring, itemsInRing *int, pos int) *analysis.Token { + if *itemsInRing == 2 { + thisShingleRing := r.Move(-1) + shingledBytes := make([]byte, 0) + + // do first token + prev := thisShingleRing.Value.(*analysis.Token) + shingledBytes = append(shingledBytes, prev.Term...) + + // do second token + thisShingleRing = thisShingleRing.Next() + curr := thisShingleRing.Value.(*analysis.Token) + shingledBytes = append(shingledBytes, curr.Term...) + + token := analysis.Token{ + Type: analysis.Double, + Term: shingledBytes, + Position: pos, + Start: prev.Start, + End: curr.End, + } + return &token + } + + return nil +} + +func (s *CJKBigramFilter) buildUnigram(r *ring.Ring, itemsInRing *int, pos int) *analysis.Token { + switch *itemsInRing { + case 2: + thisShingleRing := r.Move(-1) + // do first token + prev := thisShingleRing.Value.(*analysis.Token) + token := analysis.Token{ + Type: analysis.Single, + Term: prev.Term, + Position: pos, + Start: prev.Start, + End: prev.End, + } + return &token + case 1: + // do first token + prev := r.Value.(*analysis.Token) + token := analysis.Token{ + Type: analysis.Single, + Term: prev.Term, + Position: pos, + Start: prev.Start, + End: prev.End, + } + return &token + } + + return nil +} + +func CJKBigramFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + outputUnigram := false + outVal, ok := config["output_unigram"].(bool) + if ok { + outputUnigram = outVal + } + return NewCJKBigramFilter(outputUnigram), nil +} + +func init() { + err := registry.RegisterTokenFilter(BigramName, CJKBigramFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/cjk/cjk_bigram_test.go b/analysis/lang/cjk/cjk_bigram_test.go new file mode 100644 index 0000000..0528eea --- /dev/null +++ b/analysis/lang/cjk/cjk_bigram_test.go @@ -0,0 +1,848 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cjk + +import ( + "container/ring" + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +// Helper function to create a token +func makeToken(term string, start, end, pos int) *analysis.Token { + return &analysis.Token{ + Term: []byte(term), + Start: start, + End: end, + Position: pos, // Note: buildUnigram uses the 'pos' argument, not the token's original pos + Type: analysis.Ideographic, + } +} + +func TestCJKBigramFilter_buildUnigram(t *testing.T) { + filter := NewCJKBigramFilter(false) + + tests := []struct { + name string + ringSetup func() (*ring.Ring, int) // Function to set up the ring and itemsInRing + inputPos int // Position to pass to buildUnigram + expectToken *analysis.Token + }{ + { + name: "itemsInRing == 2", + ringSetup: func() (*ring.Ring, int) { + r := ring.New(2) + token1 := makeToken("一", 0, 3, 1) // Original pos 1 + token2 := makeToken("二", 3, 6, 2) // Original pos 2 + r.Value = token1 + r = r.Next() + r.Value = token2 + // r currently points to token2, r.Move(-1) points to token1 + return r, 2 + }, + inputPos: 10, // Expected output position + expectToken: &analysis.Token{ + Type: analysis.Single, + Term: []byte("一"), + Position: 10, // Should use inputPos + Start: 0, + End: 3, + }, + }, + { + name: "itemsInRing == 1 (ring points to the single item)", + ringSetup: func() (*ring.Ring, int) { + r := ring.New(2) + token1 := makeToken("三", 6, 9, 3) + r.Value = token1 + // r points to token1 + return r, 1 + }, + inputPos: 11, + expectToken: &analysis.Token{ + Type: analysis.Single, + Term: []byte("三"), + Position: 11, // Should use inputPos + Start: 6, + End: 9, + }, + }, + { + name: "itemsInRing == 1 (ring points to nil, next is the single item)", + ringSetup: func() (*ring.Ring, int) { + r := ring.New(2) + token1 := makeToken("四", 9, 12, 4) + r = r.Next() // r points to nil initially + r.Value = token1 + // r points to token1 + return r, 1 + }, + inputPos: 12, + expectToken: &analysis.Token{ + Type: analysis.Single, + Term: []byte("四"), + Position: 12, // Should use inputPos + Start: 9, + End: 12, + }, + }, + { + name: "itemsInRing == 0", + ringSetup: func() (*ring.Ring, int) { + r := ring.New(2) + // Ring is empty + return r, 0 + }, + inputPos: 13, + expectToken: nil, // Expect nil when itemsInRing is not 1 or 2 + }, + { + name: "itemsInRing > 2 (should behave like 0)", + ringSetup: func() (*ring.Ring, int) { + r := ring.New(2) + token1 := makeToken("五", 12, 15, 5) + token2 := makeToken("六", 15, 18, 6) + r.Value = token1 + r = r.Next() + r.Value = token2 + // Simulate incorrect itemsInRing count + return r, 3 + }, + inputPos: 14, + expectToken: nil, // Expect nil when itemsInRing is not 1 or 2 + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ringPtr, itemsInRing := tt.ringSetup() + itemsInRingCopy := itemsInRing // Pass a pointer to a copy + + gotToken := filter.buildUnigram(ringPtr, &itemsInRingCopy, tt.inputPos) + + if !reflect.DeepEqual(gotToken, tt.expectToken) { + t.Errorf("buildUnigram() got = %v, want %v", gotToken, tt.expectToken) + } + + // Check if itemsInRing was modified (it shouldn't be by buildUnigram) + if itemsInRingCopy != itemsInRing { + t.Errorf("buildUnigram() modified itemsInRing, got = %d, want %d", itemsInRingCopy, itemsInRing) + } + }) + } +} + +func TestCJKBigramFilter_outputBigram(t *testing.T) { + // Create a filter instance (outputUnigram value doesn't matter for outputBigram) + filter := NewCJKBigramFilter(false) + + tests := []struct { + name string + ringSetup func() (*ring.Ring, int) // Function to set up the ring and itemsInRing + inputPos int // Position to pass to outputBigram + expectToken *analysis.Token + }{ + { + name: "itemsInRing == 2", + ringSetup: func() (*ring.Ring, int) { + r := ring.New(2) + token1 := makeToken("一", 0, 3, 1) // Original pos 1 + token2 := makeToken("二", 3, 6, 2) // Original pos 2 + r.Value = token1 + r = r.Next() + r.Value = token2 + // r currently points to token2, r.Move(-1) points to token1 + return r, 2 + }, + inputPos: 10, // Expected output position + expectToken: &analysis.Token{ + Type: analysis.Double, + Term: []byte("一二"), // Combined term + Position: 10, // Should use inputPos + Start: 0, // Start of first token + End: 6, // End of second token + }, + }, + { + name: "itemsInRing == 2 with different terms", + ringSetup: func() (*ring.Ring, int) { + r := ring.New(2) + token1 := makeToken("你好", 0, 6, 1) + token2 := makeToken("世界", 6, 12, 2) + r.Value = token1 + r = r.Next() + r.Value = token2 + return r, 2 + }, + inputPos: 5, + expectToken: &analysis.Token{ + Type: analysis.Double, + Term: []byte("你好世界"), + Position: 5, + Start: 0, + End: 12, + }, + }, + { + name: "itemsInRing == 1", + ringSetup: func() (*ring.Ring, int) { + r := ring.New(2) + token1 := makeToken("三", 6, 9, 3) + r.Value = token1 + return r, 1 + }, + inputPos: 11, + expectToken: nil, // Expect nil when itemsInRing is not 2 + }, + { + name: "itemsInRing == 0", + ringSetup: func() (*ring.Ring, int) { + r := ring.New(2) + // Ring is empty + return r, 0 + }, + inputPos: 13, + expectToken: nil, // Expect nil when itemsInRing is not 2 + }, + { + name: "itemsInRing > 2 (should behave like 0)", + ringSetup: func() (*ring.Ring, int) { + r := ring.New(2) + token1 := makeToken("五", 12, 15, 5) + token2 := makeToken("六", 15, 18, 6) + r.Value = token1 + r = r.Next() + r.Value = token2 + // Simulate incorrect itemsInRing count + return r, 3 + }, + inputPos: 14, + expectToken: nil, // Expect nil when itemsInRing is not 2 + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ringPtr, itemsInRing := tt.ringSetup() + itemsInRingCopy := itemsInRing // Pass a pointer to a copy + + gotToken := filter.outputBigram(ringPtr, &itemsInRingCopy, tt.inputPos) + + if !reflect.DeepEqual(gotToken, tt.expectToken) { + t.Errorf("outputBigram() got = %v, want %v", gotToken, tt.expectToken) + } + + // Check if itemsInRing was modified (it shouldn't be by outputBigram) + if itemsInRingCopy != itemsInRing { + t.Errorf("outputBigram() modified itemsInRing, got = %d, want %d", itemsInRingCopy, itemsInRing) + } + }) + } +} + +func TestCJKBigramFilter(t *testing.T) { + tests := []struct { + outputUnigram bool + input analysis.TokenStream + output analysis.TokenStream + }{ + // first test that non-adjacent terms are not combined + { + outputUnigram: false, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("こ"), + Type: analysis.Ideographic, + Position: 1, + Start: 0, + End: 3, + }, + &analysis.Token{ + Term: []byte("ん"), + Type: analysis.Ideographic, + Position: 2, + Start: 5, + End: 8, + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("こ"), + Type: analysis.Single, + Position: 1, + Start: 0, + End: 3, + }, + &analysis.Token{ + Term: []byte("ん"), + Type: analysis.Single, + Position: 2, + Start: 5, + End: 8, + }, + }, + }, + { + outputUnigram: false, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("こ"), + Type: analysis.Ideographic, + Position: 1, + Start: 0, + End: 3, + }, + &analysis.Token{ + Term: []byte("ん"), + Type: analysis.Ideographic, + Position: 2, + Start: 3, + End: 6, + }, + &analysis.Token{ + Term: []byte("に"), + Type: analysis.Ideographic, + Position: 3, + Start: 6, + End: 9, + }, + &analysis.Token{ + Term: []byte("ち"), + Type: analysis.Ideographic, + Position: 4, + Start: 9, + End: 12, + }, + &analysis.Token{ + Term: []byte("は"), + Type: analysis.Ideographic, + Position: 5, + Start: 12, + End: 15, + }, + &analysis.Token{ + Term: []byte("世"), + Type: analysis.Ideographic, + Position: 6, + Start: 15, + End: 18, + }, + &analysis.Token{ + Term: []byte("界"), + Type: analysis.Ideographic, + Position: 7, + Start: 18, + End: 21, + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("こん"), + Type: analysis.Double, + Position: 1, + Start: 0, + End: 6, + }, + &analysis.Token{ + Term: []byte("んに"), + Type: analysis.Double, + Position: 2, + Start: 3, + End: 9, + }, + &analysis.Token{ + Term: []byte("にち"), + Type: analysis.Double, + Position: 3, + Start: 6, + End: 12, + }, + &analysis.Token{ + Term: []byte("ちは"), + Type: analysis.Double, + Position: 4, + Start: 9, + End: 15, + }, + &analysis.Token{ + Term: []byte("は世"), + Type: analysis.Double, + Position: 5, + Start: 12, + End: 18, + }, + &analysis.Token{ + Term: []byte("世界"), + Type: analysis.Double, + Position: 6, + Start: 15, + End: 21, + }, + }, + }, + { + outputUnigram: true, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("こ"), + Type: analysis.Ideographic, + Position: 1, + Start: 0, + End: 3, + }, + &analysis.Token{ + Term: []byte("ん"), + Type: analysis.Ideographic, + Position: 2, + Start: 3, + End: 6, + }, + &analysis.Token{ + Term: []byte("に"), + Type: analysis.Ideographic, + Position: 3, + Start: 6, + End: 9, + }, + &analysis.Token{ + Term: []byte("ち"), + Type: analysis.Ideographic, + Position: 4, + Start: 9, + End: 12, + }, + &analysis.Token{ + Term: []byte("は"), + Type: analysis.Ideographic, + Position: 5, + Start: 12, + End: 15, + }, + &analysis.Token{ + Term: []byte("世"), + Type: analysis.Ideographic, + Position: 6, + Start: 15, + End: 18, + }, + &analysis.Token{ + Term: []byte("界"), + Type: analysis.Ideographic, + Position: 7, + Start: 18, + End: 21, + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("こ"), + Type: analysis.Single, + Position: 1, + Start: 0, + End: 3, + }, + &analysis.Token{ + Term: []byte("こん"), + Type: analysis.Double, + Position: 1, + Start: 0, + End: 6, + }, + &analysis.Token{ + Term: []byte("ん"), + Type: analysis.Single, + Position: 2, + Start: 3, + End: 6, + }, + &analysis.Token{ + Term: []byte("んに"), + Type: analysis.Double, + Position: 2, + Start: 3, + End: 9, + }, + &analysis.Token{ + Term: []byte("に"), + Type: analysis.Single, + Position: 3, + Start: 6, + End: 9, + }, + &analysis.Token{ + Term: []byte("にち"), + Type: analysis.Double, + Position: 3, + Start: 6, + End: 12, + }, + &analysis.Token{ + Term: []byte("ち"), + Type: analysis.Single, + Position: 4, + Start: 9, + End: 12, + }, + &analysis.Token{ + Term: []byte("ちは"), + Type: analysis.Double, + Position: 4, + Start: 9, + End: 15, + }, + &analysis.Token{ + Term: []byte("は"), + Type: analysis.Single, + Position: 5, + Start: 12, + End: 15, + }, + &analysis.Token{ + Term: []byte("は世"), + Type: analysis.Double, + Position: 5, + Start: 12, + End: 18, + }, + &analysis.Token{ + Term: []byte("世"), + Type: analysis.Single, + Position: 6, + Start: 15, + End: 18, + }, + &analysis.Token{ + Term: []byte("世界"), + Type: analysis.Double, + Position: 6, + Start: 15, + End: 21, + }, + &analysis.Token{ + Term: []byte("界"), + Type: analysis.Single, + Position: 7, + Start: 18, + End: 21, + }, + }, + }, + { + // Assuming that `、` is removed by unicode tokenizer from `こんにちは、世界` + outputUnigram: true, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("こ"), + Type: analysis.Ideographic, + Position: 1, + Start: 0, + End: 3, + }, + &analysis.Token{ + Term: []byte("ん"), + Type: analysis.Ideographic, + Position: 2, + Start: 3, + End: 6, + }, + &analysis.Token{ + Term: []byte("に"), + Type: analysis.Ideographic, + Position: 3, + Start: 6, + End: 9, + }, + &analysis.Token{ + Term: []byte("ち"), + Type: analysis.Ideographic, + Position: 4, + Start: 9, + End: 12, + }, + &analysis.Token{ + Term: []byte("は"), + Type: analysis.Ideographic, + Position: 5, + Start: 12, + End: 15, + }, + &analysis.Token{ + Term: []byte("世"), + Type: analysis.Ideographic, + Position: 7, + Start: 18, + End: 21, + }, + &analysis.Token{ + Term: []byte("界"), + Type: analysis.Ideographic, + Position: 8, + Start: 21, + End: 24, + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("こ"), + Type: analysis.Single, + Position: 1, + Start: 0, + End: 3, + }, + &analysis.Token{ + Term: []byte("こん"), + Type: analysis.Double, + Position: 1, + Start: 0, + End: 6, + }, + &analysis.Token{ + Term: []byte("ん"), + Type: analysis.Single, + Position: 2, + Start: 3, + End: 6, + }, + &analysis.Token{ + Term: []byte("んに"), + Type: analysis.Double, + Position: 2, + Start: 3, + End: 9, + }, + &analysis.Token{ + Term: []byte("に"), + Type: analysis.Single, + Position: 3, + Start: 6, + End: 9, + }, + &analysis.Token{ + Term: []byte("にち"), + Type: analysis.Double, + Position: 3, + Start: 6, + End: 12, + }, + &analysis.Token{ + Term: []byte("ち"), + Type: analysis.Single, + Position: 4, + Start: 9, + End: 12, + }, + &analysis.Token{ + Term: []byte("ちは"), + Type: analysis.Double, + Position: 4, + Start: 9, + End: 15, + }, + &analysis.Token{ + Term: []byte("は"), + Type: analysis.Single, + Position: 5, + Start: 12, + End: 15, + }, + &analysis.Token{ + Term: []byte("世"), + Type: analysis.Single, + Position: 6, + Start: 18, + End: 21, + }, + &analysis.Token{ + Term: []byte("世界"), + Type: analysis.Double, + Position: 6, + Start: 18, + End: 24, + }, + &analysis.Token{ + Term: []byte("界"), + Type: analysis.Single, + Position: 7, + Start: 21, + End: 24, + }, + }, + }, + { + outputUnigram: false, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("こ"), + Type: analysis.Ideographic, + Position: 1, + Start: 0, + End: 3, + }, + &analysis.Token{ + Term: []byte("ん"), + Type: analysis.Ideographic, + Position: 2, + Start: 3, + End: 6, + }, + &analysis.Token{ + Term: []byte("に"), + Type: analysis.Ideographic, + Position: 3, + Start: 6, + End: 9, + }, + &analysis.Token{ + Term: []byte("ち"), + Type: analysis.Ideographic, + Position: 4, + Start: 9, + End: 12, + }, + &analysis.Token{ + Term: []byte("は"), + Type: analysis.Ideographic, + Position: 5, + Start: 12, + End: 15, + }, + &analysis.Token{ + Term: []byte("cat"), + Type: analysis.AlphaNumeric, + Position: 6, + Start: 12, + End: 15, + }, + &analysis.Token{ + Term: []byte("世"), + Type: analysis.Ideographic, + Position: 7, + Start: 18, + End: 21, + }, + &analysis.Token{ + Term: []byte("界"), + Type: analysis.Ideographic, + Position: 8, + Start: 21, + End: 24, + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("こん"), + Type: analysis.Double, + Position: 1, + Start: 0, + End: 6, + }, + &analysis.Token{ + Term: []byte("んに"), + Type: analysis.Double, + Position: 2, + Start: 3, + End: 9, + }, + &analysis.Token{ + Term: []byte("にち"), + Type: analysis.Double, + Position: 3, + Start: 6, + End: 12, + }, + &analysis.Token{ + Term: []byte("ちは"), + Type: analysis.Double, + Position: 4, + Start: 9, + End: 15, + }, + &analysis.Token{ + Term: []byte("cat"), + Type: analysis.AlphaNumeric, + Position: 5, + Start: 12, + End: 15, + }, + &analysis.Token{ + Term: []byte("世界"), + Type: analysis.Double, + Position: 6, + Start: 18, + End: 24, + }, + }, + }, + { + outputUnigram: false, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("パイプライン"), + Type: analysis.Ideographic, + Position: 1, + Start: 0, + End: 18, + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("パイ"), + Type: analysis.Double, + Position: 1, + Start: 0, + End: 6, + }, + &analysis.Token{ + Term: []byte("イプ"), + Type: analysis.Double, + Position: 2, + Start: 3, + End: 9, + }, + &analysis.Token{ + Term: []byte("プラ"), + Type: analysis.Double, + Position: 3, + Start: 6, + End: 12, + }, + &analysis.Token{ + Term: []byte("ライ"), + Type: analysis.Double, + Position: 4, + Start: 9, + End: 15, + }, + &analysis.Token{ + Term: []byte("イン"), + Type: analysis.Double, + Position: 5, + Start: 12, + End: 18, + }, + }, + }, + } + + for _, test := range tests { + cjkBigramFilter := NewCJKBigramFilter(test.outputUnigram) + actual := cjkBigramFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output, actual) + } + } +} diff --git a/analysis/lang/cjk/cjk_width.go b/analysis/lang/cjk/cjk_width.go new file mode 100644 index 0000000..a9d6171 --- /dev/null +++ b/analysis/lang/cjk/cjk_width.go @@ -0,0 +1,104 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cjk + +import ( + "bytes" + "unicode/utf8" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const WidthName = "cjk_width" + +type CJKWidthFilter struct{} + +func NewCJKWidthFilter() *CJKWidthFilter { + return &CJKWidthFilter{} +} + +func (s *CJKWidthFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + runeCount := utf8.RuneCount(token.Term) + runes := bytes.Runes(token.Term) + for i := 0; i < runeCount; i++ { + ch := runes[i] + if ch >= 0xFF01 && ch <= 0xFF5E { + // fullwidth ASCII variants + runes[i] -= 0xFEE0 + } else if ch >= 0xFF65 && ch <= 0xFF9F { + // halfwidth Katakana variants + if (ch == 0xFF9E || ch == 0xFF9F) && i > 0 && combine(runes, i, ch) { + runes = analysis.DeleteRune(runes, i) + i-- + runeCount = len(runes) + } else { + runes[i] = kanaNorm[ch-0xFF65] + } + } + } + token.Term = analysis.BuildTermFromRunes(runes) + } + + return input +} + +var kanaNorm = []rune{ + 0x30fb, 0x30f2, 0x30a1, 0x30a3, 0x30a5, 0x30a7, 0x30a9, 0x30e3, 0x30e5, + 0x30e7, 0x30c3, 0x30fc, 0x30a2, 0x30a4, 0x30a6, 0x30a8, 0x30aa, 0x30ab, + 0x30ad, 0x30af, 0x30b1, 0x30b3, 0x30b5, 0x30b7, 0x30b9, 0x30bb, 0x30bd, + 0x30bf, 0x30c1, 0x30c4, 0x30c6, 0x30c8, 0x30ca, 0x30cb, 0x30cc, 0x30cd, + 0x30ce, 0x30cf, 0x30d2, 0x30d5, 0x30d8, 0x30db, 0x30de, 0x30df, 0x30e0, + 0x30e1, 0x30e2, 0x30e4, 0x30e6, 0x30e8, 0x30e9, 0x30ea, 0x30eb, 0x30ec, + 0x30ed, 0x30ef, 0x30f3, 0x3099, 0x309A, +} + +var kanaCombineVoiced = []rune{ + 78, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, + 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, +} +var kanaCombineHalfVoiced = []rune{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 2, + 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +} + +func combine(text []rune, pos int, r rune) bool { + prev := text[pos-1] + if prev >= 0x30A6 && prev <= 0x30FD { + if r == 0xFF9F { + text[pos-1] += kanaCombineHalfVoiced[prev-0x30A6] + } else { + text[pos-1] += kanaCombineVoiced[prev-0x30A6] + } + return text[pos-1] != prev + } + return false +} + +func CJKWidthFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewCJKWidthFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(WidthName, CJKWidthFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/cjk/cjk_width_test.go b/analysis/lang/cjk/cjk_width_test.go new file mode 100644 index 0000000..5533ae2 --- /dev/null +++ b/analysis/lang/cjk/cjk_width_test.go @@ -0,0 +1,93 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cjk + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestCJKWidthFilter(t *testing.T) { + + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Ｔｅｓｔ"), + }, + &analysis.Token{ + Term: []byte("１２３４"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Test"), + }, + &analysis.Token{ + Term: []byte("1234"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ｶﾀｶﾅ"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("カタカナ"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ｳﾞｨｯﾂ"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ヴィッツ"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ﾊﾟﾅｿﾆｯｸ"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("パナソニック"), + }, + }, + }, + } + + for _, test := range tests { + cjkWidthFilter := NewCJKWidthFilter() + actual := cjkWidthFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output, actual) + } + } +} diff --git a/analysis/lang/ckb/analyzer_ckb.go b/analysis/lang/ckb/analyzer_ckb.go new file mode 100644 index 0000000..2b44b57 --- /dev/null +++ b/analysis/lang/ckb/analyzer_ckb.go @@ -0,0 +1,64 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ckb + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" + "github.com/blevesearch/bleve/v2/registry" +) + +const AnalyzerName = "ckb" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + unicodeTokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + normCkbFilter, err := cache.TokenFilterNamed(NormalizeName) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopCkbFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + stemmerCkbFilter, err := cache.TokenFilterNamed(StemmerName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: unicodeTokenizer, + TokenFilters: []analysis.TokenFilter{ + normCkbFilter, + toLowerFilter, + stopCkbFilter, + stemmerCkbFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ckb/analyzer_ckb_test.go b/analysis/lang/ckb/analyzer_ckb_test.go new file mode 100644 index 0000000..9e6adab --- /dev/null +++ b/analysis/lang/ckb/analyzer_ckb_test.go @@ -0,0 +1,77 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ckb + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestSoraniAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // stop word removal + { + input: []byte("ئەم پیاوە"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("پیاو"), + Position: 2, + Start: 7, + End: 17, + }, + }, + }, + { + input: []byte("پیاوە"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("پیاو"), + Position: 1, + Start: 0, + End: 10, + }, + }, + }, + { + input: []byte("پیاو"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("پیاو"), + Position: 1, + Start: 0, + End: 8, + }, + }, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %v, got %v", test.output, actual) + } + } +} diff --git a/analysis/lang/ckb/sorani_normalize.go b/analysis/lang/ckb/sorani_normalize.go new file mode 100644 index 0000000..fade3f5 --- /dev/null +++ b/analysis/lang/ckb/sorani_normalize.go @@ -0,0 +1,121 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ckb + +import ( + "bytes" + "unicode" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const NormalizeName = "normalize_ckb" + +const ( + Yeh = '\u064A' + DotlessYeh = '\u0649' + FarsiYeh = '\u06CC' + + Kaf = '\u0643' + Keheh = '\u06A9' + + Heh = '\u0647' + Ae = '\u06D5' + Zwnj = '\u200C' + HehDoachashmee = '\u06BE' + TehMarbuta = '\u0629' + + Reh = '\u0631' + Rreh = '\u0695' + RrehAbove = '\u0692' + + Tatweel = '\u0640' + Fathatan = '\u064B' + Dammatan = '\u064C' + Kasratan = '\u064D' + Fatha = '\u064E' + Damma = '\u064F' + Kasra = '\u0650' + Shadda = '\u0651' + Sukun = '\u0652' +) + +type SoraniNormalizeFilter struct { +} + +func NewSoraniNormalizeFilter() *SoraniNormalizeFilter { + return &SoraniNormalizeFilter{} +} + +func (s *SoraniNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + term := normalize(token.Term) + token.Term = term + } + return input +} + +func normalize(input []byte) []byte { + runes := bytes.Runes(input) + for i := 0; i < len(runes); i++ { + switch runes[i] { + case Yeh, DotlessYeh: + runes[i] = FarsiYeh + case Kaf: + runes[i] = Keheh + case Zwnj: + if i > 0 && runes[i-1] == Heh { + runes[i-1] = Ae + } + runes = analysis.DeleteRune(runes, i) + i-- + case Heh: + if i == len(runes)-1 { + runes[i] = Ae + } + case TehMarbuta: + runes[i] = Ae + case HehDoachashmee: + runes[i] = Heh + case Reh: + if i == 0 { + runes[i] = Rreh + } + case RrehAbove: + runes[i] = Rreh + case Tatweel, Kasratan, Dammatan, Fathatan, Fatha, Damma, Kasra, Shadda, Sukun: + runes = analysis.DeleteRune(runes, i) + i-- + default: + if unicode.In(runes[i], unicode.Cf) { + runes = analysis.DeleteRune(runes, i) + i-- + } + } + } + return analysis.BuildTermFromRunes(runes) +} + +func NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewSoraniNormalizeFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ckb/sorani_normalize_test.go b/analysis/lang/ckb/sorani_normalize_test.go new file mode 100644 index 0000000..0fb0fcd --- /dev/null +++ b/analysis/lang/ckb/sorani_normalize_test.go @@ -0,0 +1,323 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ckb + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestSoraniNormalizeFilter(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + // test Y + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u064A"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u06CC"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u0649"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u06CC"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u06CC"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u06CC"), + }, + }, + }, + // test K + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u0643"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u06A9"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u06A9"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u06A9"), + }, + }, + }, + // test H + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u0647\u200C"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u06D5"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u0647\u200C\u06A9"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u06D5\u06A9"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u06BE"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u0647"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u0629"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u06D5"), + }, + }, + }, + // test final H + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u0647\u0647\u0647"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u0647\u0647\u06D5"), + }, + }, + }, + // test RR + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u0692"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u0695"), + }, + }, + }, + // test initial RR + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u0631\u0631\u0631"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u0695\u0631\u0631"), + }, + }, + }, + // test remove + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u0640"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u064B"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u064C"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u064D"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u064E"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u064F"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u0650"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u0651"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u0652"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u200C"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + // empty + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + } + + soraniNormalizeFilter := NewSoraniNormalizeFilter() + for _, test := range tests { + actual := soraniNormalizeFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %#v, got %#v", test.output, actual) + t.Errorf("expected % x, got % x", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/ckb/sorani_stemmer_filter.go b/analysis/lang/ckb/sorani_stemmer_filter.go new file mode 100644 index 0000000..533c4ba --- /dev/null +++ b/analysis/lang/ckb/sorani_stemmer_filter.go @@ -0,0 +1,151 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ckb + +import ( + "bytes" + "unicode/utf8" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StemmerName = "stemmer_ckb" + +type SoraniStemmerFilter struct { +} + +func NewSoraniStemmerFilter() *SoraniStemmerFilter { + return &SoraniStemmerFilter{} +} + +func (s *SoraniStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + // if not protected keyword, stem it + if !token.KeyWord { + stemmed := stem(token.Term) + token.Term = stemmed + } + } + return input +} + +func stem(input []byte) []byte { + inputLen := utf8.RuneCount(input) + + // postposition + if inputLen > 5 && bytes.HasSuffix(input, []byte("دا")) { + input = truncateRunes(input, 2) + inputLen = utf8.RuneCount(input) + } else if inputLen > 4 && bytes.HasSuffix(input, []byte("نا")) { + input = truncateRunes(input, 1) + inputLen = utf8.RuneCount(input) + } else if inputLen > 6 && bytes.HasSuffix(input, []byte("ەوە")) { + input = truncateRunes(input, 3) + inputLen = utf8.RuneCount(input) + } + + // possessive pronoun + if inputLen > 6 && + (bytes.HasSuffix(input, []byte("مان")) || + bytes.HasSuffix(input, []byte("یان")) || + bytes.HasSuffix(input, []byte("تان"))) { + input = truncateRunes(input, 3) + inputLen = utf8.RuneCount(input) + } + + // indefinite singular ezafe + if inputLen > 6 && bytes.HasSuffix(input, []byte("ێکی")) { + return truncateRunes(input, 3) + } else if inputLen > 7 && bytes.HasSuffix(input, []byte("یەکی")) { + return truncateRunes(input, 4) + } + + if inputLen > 5 && bytes.HasSuffix(input, []byte("ێک")) { + // indefinite singular + return truncateRunes(input, 2) + } else if inputLen > 6 && bytes.HasSuffix(input, []byte("یەک")) { + // indefinite singular + return truncateRunes(input, 3) + } else if inputLen > 6 && bytes.HasSuffix(input, []byte("ەکە")) { + // definite singular + return truncateRunes(input, 3) + } else if inputLen > 5 && bytes.HasSuffix(input, []byte("کە")) { + // definite singular + return truncateRunes(input, 2) + } else if inputLen > 7 && bytes.HasSuffix(input, []byte("ەکان")) { + // definite plural + return truncateRunes(input, 4) + } else if inputLen > 6 && bytes.HasSuffix(input, []byte("کان")) { + // definite plural + return truncateRunes(input, 3) + } else if inputLen > 7 && bytes.HasSuffix(input, []byte("یانی")) { + // indefinite plural ezafe + return truncateRunes(input, 4) + } else if inputLen > 6 && bytes.HasSuffix(input, []byte("انی")) { + // indefinite plural ezafe + return truncateRunes(input, 3) + } else if inputLen > 6 && bytes.HasSuffix(input, []byte("یان")) { + // indefinite plural + return truncateRunes(input, 3) + } else if inputLen > 5 && bytes.HasSuffix(input, []byte("ان")) { + // indefinite plural + return truncateRunes(input, 2) + } else if inputLen > 7 && bytes.HasSuffix(input, []byte("یانە")) { + // demonstrative plural + return truncateRunes(input, 4) + } else if inputLen > 6 && bytes.HasSuffix(input, []byte("انە")) { + // demonstrative plural + return truncateRunes(input, 3) + } else if inputLen > 5 && (bytes.HasSuffix(input, []byte("ایە")) || bytes.HasSuffix(input, []byte("ەیە"))) { + // demonstrative singular + return truncateRunes(input, 2) + } else if inputLen > 4 && bytes.HasSuffix(input, []byte("ە")) { + // demonstrative singular + return truncateRunes(input, 1) + } else if inputLen > 4 && bytes.HasSuffix(input, []byte("ی")) { + // absolute singular ezafe + return truncateRunes(input, 1) + } + return input +} + +func truncateRunes(input []byte, num int) []byte { + runes := bytes.Runes(input) + runes = runes[:len(runes)-num] + out := buildTermFromRunes(runes) + return out +} + +func buildTermFromRunes(runes []rune) []byte { + rv := make([]byte, 0, len(runes)*4) + for _, r := range runes { + runeBytes := make([]byte, utf8.RuneLen(r)) + utf8.EncodeRune(runeBytes, r) + rv = append(rv, runeBytes...) + } + return rv +} + +func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewSoraniStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ckb/sorani_stemmer_filter_test.go b/analysis/lang/ckb/sorani_stemmer_filter_test.go new file mode 100644 index 0000000..4dd9a9f --- /dev/null +++ b/analysis/lang/ckb/sorani_stemmer_filter_test.go @@ -0,0 +1,299 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ckb + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/single" +) + +func TestSoraniStemmerFilter(t *testing.T) { + + // in order to match the lucene tests + // we will test with an analyzer, not just the stemmer + analyzer := analysis.DefaultAnalyzer{ + Tokenizer: single.NewSingleTokenTokenizer(), + TokenFilters: []analysis.TokenFilter{ + NewSoraniNormalizeFilter(), + NewSoraniStemmerFilter(), + }, + } + + tests := []struct { + input []byte + output analysis.TokenStream + }{ + { // -ek + input: []byte("پیاوێک"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("پیاو"), + Position: 1, + Start: 0, + End: 12, + }, + }, + }, + { // -yek + input: []byte("دەرگایەک"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("دەرگا"), + Position: 1, + Start: 0, + End: 16, + }, + }, + }, + { // -aka + input: []byte("پیاوەكە"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("پیاو"), + Position: 1, + Start: 0, + End: 14, + }, + }, + }, + { // -ka + input: []byte("دەرگاكە"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("دەرگا"), + Position: 1, + Start: 0, + End: 14, + }, + }, + }, + { // -a + input: []byte("کتاویە"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("کتاوی"), + Position: 1, + Start: 0, + End: 12, + }, + }, + }, + { // -ya + input: []byte("دەرگایە"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("دەرگا"), + Position: 1, + Start: 0, + End: 14, + }, + }, + }, + { // -An + input: []byte("پیاوان"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("پیاو"), + Position: 1, + Start: 0, + End: 12, + }, + }, + }, + { // -yAn + input: []byte("دەرگایان"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("دەرگا"), + Position: 1, + Start: 0, + End: 16, + }, + }, + }, + { // -akAn + input: []byte("پیاوەکان"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("پیاو"), + Position: 1, + Start: 0, + End: 16, + }, + }, + }, + { // -kAn + input: []byte("دەرگاکان"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("دەرگا"), + Position: 1, + Start: 0, + End: 16, + }, + }, + }, + { // -Ana + input: []byte("پیاوانە"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("پیاو"), + Position: 1, + Start: 0, + End: 14, + }, + }, + }, + { // -yAna + input: []byte("دەرگایانە"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("دەرگا"), + Position: 1, + Start: 0, + End: 18, + }, + }, + }, + { // Ezafe singular + input: []byte("هۆتیلی"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("هۆتیل"), + Position: 1, + Start: 0, + End: 12, + }, + }, + }, + { // Ezafe indefinite + input: []byte("هۆتیلێکی"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("هۆتیل"), + Position: 1, + Start: 0, + End: 16, + }, + }, + }, + { // Ezafe plural + input: []byte("هۆتیلانی"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("هۆتیل"), + Position: 1, + Start: 0, + End: 16, + }, + }, + }, + { // -awa + input: []byte("دوورەوە"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("دوور"), + Position: 1, + Start: 0, + End: 14, + }, + }, + }, + { // -dA + input: []byte("نیوەشەودا"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("نیوەشەو"), + Position: 1, + Start: 0, + End: 18, + }, + }, + }, + { // -A + input: []byte("سۆرانا"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("سۆران"), + Position: 1, + Start: 0, + End: 12, + }, + }, + }, + { // -mAn + input: []byte("پارەمان"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("پارە"), + Position: 1, + Start: 0, + End: 14, + }, + }, + }, + { // -tAn + input: []byte("پارەتان"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("پارە"), + Position: 1, + Start: 0, + End: 14, + }, + }, + }, + { // -yAn + input: []byte("پارەیان"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("پارە"), + Position: 1, + Start: 0, + End: 14, + }, + }, + }, + { // empty + input: []byte(""), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + Position: 1, + Start: 0, + End: 0, + }, + }, + }, + } + + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("for input %s(% x)", test.input, test.input) + t.Errorf("\texpected:") + for _, token := range test.output { + t.Errorf("\t\t%v %s(% x)", token, token.Term, token.Term) + } + t.Errorf("\tactual:") + for _, token := range actual { + t.Errorf("\t\t%v %s(% x)", token, token.Term, token.Term) + } + } + } +} diff --git a/analysis/lang/ckb/stop_filter_ckb.go b/analysis/lang/ckb/stop_filter_ckb.go new file mode 100644 index 0000000..ed4ec8d --- /dev/null +++ b/analysis/lang/ckb/stop_filter_ckb.go @@ -0,0 +1,36 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ckb + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ckb/stop_words_ckb.go b/analysis/lang/ckb/stop_words_ckb.go new file mode 100644 index 0000000..87b9e81 --- /dev/null +++ b/analysis/lang/ckb/stop_words_ckb.go @@ -0,0 +1,163 @@ +package ckb + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_ckb" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ +// ` was changed to ' to allow for literal string + +var SoraniStopWords = []byte(`# set of kurdish stopwords +# note these have been normalized with our scheme (e represented with U+06D5, etc) +# constructed from: +# * Fig 5 of "Building A Test Collection For Sorani Kurdish" (Esmaili et al) +# * "Sorani Kurdish: A Reference Grammar with selected readings" (Thackston) +# * Corpus-based analysis of 77M word Sorani collection: wikipedia, news, blogs, etc + +# and +و +# which +کە +# of +ی +# made/did +کرد +# that/which +ئەوەی +# on/head +سەر +# two +دوو +# also +هەروەها +# from/that +لەو +# makes/does +دەکات +# some +چەند +# every +هەر + +# demonstratives +# that +ئەو +# this +ئەم + +# personal pronouns +# I +من +# we +ئێمە +# you +تۆ +# you +ئێوە +# he/she/it +ئەو +# they +ئەوان + +# prepositions +# to/with/by +بە +پێ +# without +بەبێ +# along with/while/during +بەدەم +# in the opinion of +بەلای +# according to +بەپێی +# before +بەرلە +# in the direction of +بەرەوی +# in front of/toward +بەرەوە +# before/in the face of +بەردەم +# without +بێ +# except for +بێجگە +# for +بۆ +# on/in +دە +تێ +# with +دەگەڵ +# after +دوای +# except for/aside from +جگە +# in/from +لە +لێ +# in front of/before/because of +لەبەر +# between/among +لەبەینی +# concerning/about +لەبابەت +# concerning +لەبارەی +# instead of +لەباتی +# beside +لەبن +# instead of +لەبرێتی +# behind +لەدەم +# with/together with +لەگەڵ +# by +لەلایەن +# within +لەناو +# between/among +لەنێو +# for the sake of +لەپێناوی +# with respect to +لەرەوی +# by means of/for +لەرێ +# for the sake of +لەرێگا +# on/on top of/according to +لەسەر +# under +لەژێر +# between/among +ناو +# between/among +نێوان +# after +پاش +# before +پێش +# like +وەک +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(SoraniStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/cs/stop_filter_cs.go b/analysis/lang/cs/stop_filter_cs.go new file mode 100644 index 0000000..52f1e93 --- /dev/null +++ b/analysis/lang/cs/stop_filter_cs.go @@ -0,0 +1,36 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cs + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/cs/stop_words_cs.go b/analysis/lang/cs/stop_words_cs.go new file mode 100644 index 0000000..5c3143e --- /dev/null +++ b/analysis/lang/cs/stop_words_cs.go @@ -0,0 +1,199 @@ +package cs + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_cs" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ +// ` was changed to ' to allow for literal string + +var CzechStopWords = []byte(`a +s +k +o +i +u +v +z +dnes +cz +tímto +budeš +budem +byli +jseš +můj +svým +ta +tomto +tohle +tuto +tyto +jej +zda +proč +máte +tato +kam +tohoto +kdo +kteří +mi +nám +tom +tomuto +mít +nic +proto +kterou +byla +toho +protože +asi +ho +naši +napište +re +což +tím +takže +svých +její +svými +jste +aj +tu +tedy +teto +bylo +kde +ke +pravé +ji +nad +nejsou +či +pod +téma +mezi +přes +ty +pak +vám +ani +když +však +neg +jsem +tento +článku +články +aby +jsme +před +pta +jejich +byl +ještě +až +bez +také +pouze +první +vaše +která +nás +nový +tipy +pokud +může +strana +jeho +své +jiné +zprávy +nové +není +vás +jen +podle +zde +už +být +více +bude +již +než +který +by +které +co +nebo +ten +tak +má +při +od +po +jsou +jak +další +ale +si +se +ve +to +jako +za +zpět +ze +do +pro +je +na +atd +atp +jakmile +přičemž +já +on +ona +ono +oni +ony +my +vy +jí +ji +mě +mne +jemu +tomu +těm +těmu +němu +němuž +jehož +jíž +jelikož +jež +jakož +načež +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(CzechStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/da/analyzer_da.go b/analysis/lang/da/analyzer_da.go new file mode 100644 index 0000000..8bfd554 --- /dev/null +++ b/analysis/lang/da/analyzer_da.go @@ -0,0 +1,59 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package da + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" + "github.com/blevesearch/bleve/v2/registry" +) + +const AnalyzerName = "da" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + unicodeTokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopDaFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + stemmerDaFilter, err := cache.TokenFilterNamed(SnowballStemmerName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: unicodeTokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + stopDaFilter, + stemmerDaFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/da/analyzer_da_test.go b/analysis/lang/da/analyzer_da_test.go new file mode 100644 index 0000000..e22f325 --- /dev/null +++ b/analysis/lang/da/analyzer_da_test.go @@ -0,0 +1,71 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package da + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestDanishAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // stemming + { + input: []byte("undersøg"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("undersøg"), + Position: 1, + Start: 0, + End: 9, + }, + }, + }, + { + input: []byte("undersøgelse"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("undersøg"), + Position: 1, + Start: 0, + End: 13, + }, + }, + }, + // stop word + { + input: []byte("på"), + output: analysis.TokenStream{}, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %v, got %v", test.output, actual) + } + } +} diff --git a/analysis/lang/da/stemmer_da.go b/analysis/lang/da/stemmer_da.go new file mode 100644 index 0000000..c53aa21 --- /dev/null +++ b/analysis/lang/da/stemmer_da.go @@ -0,0 +1,52 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package da + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/snowballstem" + "github.com/blevesearch/snowballstem/danish" +) + +const SnowballStemmerName = "stemmer_da_snowball" + +type DanishStemmerFilter struct { +} + +func NewDanishStemmerFilter() *DanishStemmerFilter { + return &DanishStemmerFilter{} +} + +func (s *DanishStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + env := snowballstem.NewEnv(string(token.Term)) + danish.Stem(env) + token.Term = []byte(env.Current()) + } + return input +} + +func DanishStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewDanishStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(SnowballStemmerName, DanishStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/da/stop_filter_da.go b/analysis/lang/da/stop_filter_da.go new file mode 100644 index 0000000..6dd0e5c --- /dev/null +++ b/analysis/lang/da/stop_filter_da.go @@ -0,0 +1,36 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package da + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/da/stop_words_da.go b/analysis/lang/da/stop_words_da.go new file mode 100644 index 0000000..caaabf7 --- /dev/null +++ b/analysis/lang/da/stop_words_da.go @@ -0,0 +1,137 @@ +package da + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_da" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ +// ` was changed to ' to allow for literal string + +var DanishStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/danish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A Danish stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This is a ranked list (commonest to rarest) of stopwords derived from + | a large text sample. + + +og | and +i | in +jeg | I +det | that (dem. pronoun)/it (pers. pronoun) +at | that (in front of a sentence)/to (with infinitive) +en | a/an +den | it (pers. pronoun)/that (dem. pronoun) +til | to/at/for/until/against/by/of/into, more +er | present tense of "to be" +som | who, as +på | on/upon/in/on/at/to/after/of/with/for, on +de | they +med | with/by/in, along +han | he +af | of/by/from/off/for/in/with/on, off +for | at/for/to/from/by/of/ago, in front/before, because +ikke | not +der | who/which, there/those +var | past tense of "to be" +mig | me/myself +sig | oneself/himself/herself/itself/themselves +men | but +et | a/an/one, one (number), someone/somebody/one +har | present tense of "to have" +om | round/about/for/in/a, about/around/down, if +vi | we +min | my +havde | past tense of "to have" +ham | him +hun | she +nu | now +over | over/above/across/by/beyond/past/on/about, over/past +da | then, when/as/since +fra | from/off/since, off, since +du | you +ud | out +sin | his/her/its/one's +dem | them +os | us/ourselves +op | up +man | you/one +hans | his +hvor | where +eller | or +hvad | what +skal | must/shall etc. +selv | myself/youself/herself/ourselves etc., even +her | here +alle | all/everyone/everybody etc. +vil | will (verb) +blev | past tense of "to stay/to remain/to get/to become" +kunne | could +ind | in +når | when +være | present tense of "to be" +dog | however/yet/after all +noget | something +ville | would +jo | you know/you see (adv), yes +deres | their/theirs +efter | after/behind/according to/for/by/from, later/afterwards +ned | down +skulle | should +denne | this +end | than +dette | this +mit | my/mine +også | also +under | under/beneath/below/during, below/underneath +have | have +dig | you +anden | other +hende | her +mine | my +alt | everything +meget | much/very, plenty of +sit | his, her, its, one's +sine | his, her, its, one's +vor | our +mod | against +disse | these +hvis | if +din | your/yours +nogle | some +hos | by/at +blive | be/become +mange | many +ad | by/through +bliver | present tense of "to be/to become" +hendes | her/hers +været | be +thi | for (conj) +jer | you +sådan | such, like this/like that +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(DanishStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/de/analyzer_de.go b/analysis/lang/de/analyzer_de.go new file mode 100644 index 0000000..5330266 --- /dev/null +++ b/analysis/lang/de/analyzer_de.go @@ -0,0 +1,64 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" + "github.com/blevesearch/bleve/v2/registry" +) + +const AnalyzerName = "de" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + unicodeTokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopDeFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + normalizeDeFilter, err := cache.TokenFilterNamed(NormalizeName) + if err != nil { + return nil, err + } + lightStemmerDeFilter, err := cache.TokenFilterNamed(LightStemmerName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: unicodeTokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + stopDeFilter, + normalizeDeFilter, + lightStemmerDeFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/de/analyzer_de_test.go b/analysis/lang/de/analyzer_de_test.go new file mode 100644 index 0000000..f404ded --- /dev/null +++ b/analysis/lang/de/analyzer_de_test.go @@ -0,0 +1,155 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestGermanAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + { + input: []byte("Tisch"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("tisch"), + Position: 1, + Start: 0, + End: 5, + }, + }, + }, + { + input: []byte("Tische"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("tisch"), + Position: 1, + Start: 0, + End: 6, + }, + }, + }, + { + input: []byte("Tischen"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("tisch"), + Position: 1, + Start: 0, + End: 7, + }, + }, + }, + // german specials + { + input: []byte("Schaltflächen"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("schaltflach"), + Position: 1, + Start: 0, + End: 14, + }, + }, + }, + { + input: []byte("Schaltflaechen"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("schaltflach"), + Position: 1, + Start: 0, + End: 14, + }, + }, + }, + // tests added by marty to increase coverage + { + input: []byte("Blechern"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("blech"), + Position: 1, + Start: 0, + End: 8, + }, + }, + }, + { + input: []byte("Klecks"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("kleck"), + Position: 1, + Start: 0, + End: 6, + }, + }, + }, + { + input: []byte("Mindestens"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("mindest"), + Position: 1, + Start: 0, + End: 10, + }, + }, + }, + { + input: []byte("Kugelfest"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("kugelf"), + Position: 1, + Start: 0, + End: 9, + }, + }, + }, + { + input: []byte("Baldigst"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("baldig"), + Position: 1, + Start: 0, + End: 8, + }, + }, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %v, got %v", test.output, actual) + } + } +} diff --git a/analysis/lang/de/german_normalize.go b/analysis/lang/de/german_normalize.go new file mode 100644 index 0000000..a288255 --- /dev/null +++ b/analysis/lang/de/german_normalize.go @@ -0,0 +1,98 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de + +import ( + "bytes" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const NormalizeName = "normalize_de" + +const ( + N = 0 /* ordinary state */ + V = 1 /* stops 'u' from entering umlaut state */ + U = 2 /* umlaut state, allows e-deletion */ +) + +type GermanNormalizeFilter struct { +} + +func NewGermanNormalizeFilter() *GermanNormalizeFilter { + return &GermanNormalizeFilter{} +} + +func (s *GermanNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + term := normalize(token.Term) + token.Term = term + } + return input +} + +func normalize(input []byte) []byte { + state := N + runes := bytes.Runes(input) + for i := 0; i < len(runes); i++ { + switch runes[i] { + case 'a', 'o': + state = U + case 'u': + if state == N { + state = U + } else { + state = V + } + case 'e': + if state == U { + runes = analysis.DeleteRune(runes, i) + i-- + } + state = V + case 'i', 'q', 'y': + state = V + case 'ä': + runes[i] = 'a' + state = V + case 'ö': + runes[i] = 'o' + state = V + case 'ü': + runes[i] = 'u' + state = V + case 'ß': + runes[i] = 's' + i++ + runes = analysis.InsertRune(runes, i, 's') + state = N + default: + state = N + } + } + return analysis.BuildTermFromRunes(runes) +} + +func NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewGermanNormalizeFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/de/german_normalize_test.go b/analysis/lang/de/german_normalize_test.go new file mode 100644 index 0000000..81a60f5 --- /dev/null +++ b/analysis/lang/de/german_normalize_test.go @@ -0,0 +1,103 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestGermanNormalizeFilter(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + // Tests that a/o/u + e is equivalent to the umlaut form + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Schaltflächen"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Schaltflachen"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Schaltflaechen"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Schaltflachen"), + }, + }, + }, + // Tests the specific heuristic that ue is not folded after a vowel or q. + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("dauer"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("dauer"), + }, + }, + }, + // Tests german specific folding of sharp-s + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("weißbier"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("weissbier"), + }, + }, + }, + // empty + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + } + + germanNormalizeFilter := NewGermanNormalizeFilter() + for _, test := range tests { + actual := germanNormalizeFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %#v, got %#v", test.output, actual) + t.Errorf("expected %s(% x), got %s(% x)", test.output[0].Term, test.output[0].Term, actual[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/de/light_stemmer_de.go b/analysis/lang/de/light_stemmer_de.go new file mode 100644 index 0000000..d263de6 --- /dev/null +++ b/analysis/lang/de/light_stemmer_de.go @@ -0,0 +1,119 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de + +import ( + "bytes" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const LightStemmerName = "stemmer_de_light" + +type GermanLightStemmerFilter struct { +} + +func NewGermanLightStemmerFilter() *GermanLightStemmerFilter { + return &GermanLightStemmerFilter{} +} + +func (s *GermanLightStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + runes := bytes.Runes(token.Term) + runes = stem(runes) + token.Term = analysis.BuildTermFromRunes(runes) + } + return input +} + +func stem(input []rune) []rune { + + for i, r := range input { + switch r { + case 'ä', 'à', 'á', 'â': + input[i] = 'a' + case 'ö', 'ò', 'ó', 'ô': + input[i] = 'o' + case 'ï', 'ì', 'í', 'î': + input[i] = 'i' + case 'ü', 'ù', 'ú', 'û': + input[i] = 'u' + } + } + + input = step1(input) + return step2(input) +} + +func stEnding(ch rune) bool { + switch ch { + case 'b', 'd', 'f', 'g', 'h', 'k', 'l', 'm', 'n', 't': + return true + } + return false +} + +func step1(s []rune) []rune { + l := len(s) + if l > 5 && s[l-3] == 'e' && s[l-2] == 'r' && s[l-1] == 'n' { + return s[:l-3] + } + + if l > 4 && s[l-2] == 'e' { + switch s[l-1] { + case 'm', 'n', 'r', 's': + return s[:l-2] + } + } + + if l > 3 && s[l-1] == 'e' { + return s[:l-1] + } + + if l > 3 && s[l-1] == 's' && stEnding(s[l-2]) { + return s[:l-1] + } + + return s +} + +func step2(s []rune) []rune { + l := len(s) + if l > 5 && s[l-3] == 'e' && s[l-2] == 's' && s[l-1] == 't' { + return s[:l-3] + } + + if l > 4 && s[l-2] == 'e' && (s[l-1] == 'r' || s[l-1] == 'n') { + return s[:l-2] + } + + if l > 4 && s[l-2] == 's' && s[l-1] == 't' && stEnding(s[l-3]) { + return s[:l-2] + } + + return s +} + +func GermanLightStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewGermanLightStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(LightStemmerName, GermanLightStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/de/stemmer_de_snowball.go b/analysis/lang/de/stemmer_de_snowball.go new file mode 100644 index 0000000..aa61712 --- /dev/null +++ b/analysis/lang/de/stemmer_de_snowball.go @@ -0,0 +1,52 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/snowballstem" + "github.com/blevesearch/snowballstem/german" +) + +const SnowballStemmerName = "stemmer_de_snowball" + +type GermanStemmerFilter struct { +} + +func NewGermanStemmerFilter() *GermanStemmerFilter { + return &GermanStemmerFilter{} +} + +func (s *GermanStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + env := snowballstem.NewEnv(string(token.Term)) + german.Stem(env) + token.Term = []byte(env.Current()) + } + return input +} + +func GermanStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewGermanStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(SnowballStemmerName, GermanStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/de/stemmer_de_test.go b/analysis/lang/de/stemmer_de_test.go new file mode 100644 index 0000000..5810ec4 --- /dev/null +++ b/analysis/lang/de/stemmer_de_test.go @@ -0,0 +1,91 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestSnowballGermanStemmer(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abzuschrecken"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abzuschreck"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abzuwarten"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abzuwart"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("zwirnfabrik"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("zwirnfabr"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("zyniker"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("zynik"), + }, + }, + }, + } + + cache := registry.NewCache() + filter, err := cache.TokenFilterNamed(SnowballStemmerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := filter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/de/stop_filter_de.go b/analysis/lang/de/stop_filter_de.go new file mode 100644 index 0000000..235f510 --- /dev/null +++ b/analysis/lang/de/stop_filter_de.go @@ -0,0 +1,36 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/de/stop_words_de.go b/analysis/lang/de/stop_words_de.go new file mode 100644 index 0000000..d720657 --- /dev/null +++ b/analysis/lang/de/stop_words_de.go @@ -0,0 +1,321 @@ +package de + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_de" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ +// ` was changed to ' to allow for literal string + +var GermanStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/german/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A German stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | The number of forms in this list is reduced significantly by passing it + | through the German stemmer. + + +aber | but + +alle | all +allem +allen +aller +alles + +als | than, as +also | so +am | an + dem +an | at + +ander | other +andere +anderem +anderen +anderer +anderes +anderm +andern +anderr +anders + +auch | also +auf | on +aus | out of +bei | by +bin | am +bis | until +bist | art +da | there +damit | with it +dann | then + +der | the +den +des +dem +die +das + +daß | that + +derselbe | the same +derselben +denselben +desselben +demselben +dieselbe +dieselben +dasselbe + +dazu | to that + +dein | thy +deine +deinem +deinen +deiner +deines + +denn | because + +derer | of those +dessen | of him + +dich | thee +dir | to thee +du | thou + +dies | this +diese +diesem +diesen +dieser +dieses + + +doch | (several meanings) +dort | (over) there + + +durch | through + +ein | a +eine +einem +einen +einer +eines + +einig | some +einige +einigem +einigen +einiger +einiges + +einmal | once + +er | he +ihn | him +ihm | to him + +es | it +etwas | something + +euer | your +eure +eurem +euren +eurer +eures + +für | for +gegen | towards +gewesen | p.p. of sein +hab | have +habe | have +haben | have +hat | has +hatte | had +hatten | had +hier | here +hin | there +hinter | behind + +ich | I +mich | me +mir | to me + + +ihr | you, to her +ihre +ihrem +ihren +ihrer +ihres +euch | to you + +im | in + dem +in | in +indem | while +ins | in + das +ist | is + +jede | each, every +jedem +jeden +jeder +jedes + +jene | that +jenem +jenen +jener +jenes + +jetzt | now +kann | can + +kein | no +keine +keinem +keinen +keiner +keines + +können | can +könnte | could +machen | do +man | one + +manche | some, many a +manchem +manchen +mancher +manches + +mein | my +meine +meinem +meinen +meiner +meines + +mit | with +muss | must +musste | had to +nach | to(wards) +nicht | not +nichts | nothing +noch | still, yet +nun | now +nur | only +ob | whether +oder | or +ohne | without +sehr | very + +sein | his +seine +seinem +seinen +seiner +seines + +selbst | self +sich | herself + +sie | they, she +ihnen | to them + +sind | are +so | so + +solche | such +solchem +solchen +solcher +solches + +soll | shall +sollte | should +sondern | but +sonst | else +über | over +um | about, around +und | and + +uns | us +unse +unsem +unsen +unser +unses + +unter | under +viel | much +vom | von + dem +von | from +vor | before +während | while +war | was +waren | were +warst | wast +was | what +weg | away, off +weil | because +weiter | further + +welche | which +welchem +welchen +welcher +welches + +wenn | when +werde | will +werden | will +wie | how +wieder | again +will | want +wir | we +wird | will +wirst | willst +wo | where +wollen | want +wollte | wanted +würde | would +würden | would +zu | to +zum | zu + dem +zur | zu + der +zwar | indeed +zwischen | between + +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(GermanStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/el/stop_filter_el.go b/analysis/lang/el/stop_filter_el.go new file mode 100644 index 0000000..7b55bf6 --- /dev/null +++ b/analysis/lang/el/stop_filter_el.go @@ -0,0 +1,36 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package el + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/el/stop_words_el.go b/analysis/lang/el/stop_words_el.go new file mode 100644 index 0000000..1d2efcb --- /dev/null +++ b/analysis/lang/el/stop_words_el.go @@ -0,0 +1,105 @@ +package el + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_el" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ +// ` was changed to ' to allow for literal string + +var GreekStopWords = []byte(`# Lucene Greek Stopwords list +# Note: by default this file is used after GreekLowerCaseFilter, +# so when modifying this file use 'σ' instead of 'ς' +ο +η +το +οι +τα +του +τησ +των +τον +την +και +κι +κ +ειμαι +εισαι +ειναι +ειμαστε +ειστε +στο +στον +στη +στην +μα +αλλα +απο +για +προσ +με +σε +ωσ +παρα +αντι +κατα +μετα +θα +να +δε +δεν +μη +μην +επι +ενω +εαν +αν +τοτε +που +πωσ +ποιοσ +ποια +ποιο +ποιοι +ποιεσ +ποιων +ποιουσ +αυτοσ +αυτη +αυτο +αυτοι +αυτων +αυτουσ +αυτεσ +αυτα +εκεινοσ +εκεινη +εκεινο +εκεινοι +εκεινεσ +εκεινα +εκεινων +εκεινουσ +οπωσ +ομωσ +ισωσ +οσο +οτι +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(GreekStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/en/analyzer_en.go b/analysis/lang/en/analyzer_en.go new file mode 100644 index 0000000..b9b53a8 --- /dev/null +++ b/analysis/lang/en/analyzer_en.go @@ -0,0 +1,73 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package en implements an analyzer with reasonable defaults for processing +// English text. +// +// It strips possessive suffixes ('s), transforms tokens to lower case, +// removes stopwords from a built-in list, and applies porter stemming. +// +// The built-in stopwords list is defined in EnglishStopWords. +package en + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/token/porter" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +) + +const AnalyzerName = "en" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + tokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + possEnFilter, err := cache.TokenFilterNamed(PossessiveName) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopEnFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + stemmerEnFilter, err := cache.TokenFilterNamed(porter.Name) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: tokenizer, + TokenFilters: []analysis.TokenFilter{ + possEnFilter, + toLowerFilter, + stopEnFilter, + stemmerEnFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/en/analyzer_en_test.go b/analysis/lang/en/analyzer_en_test.go new file mode 100644 index 0000000..6db7c30 --- /dev/null +++ b/analysis/lang/en/analyzer_en_test.go @@ -0,0 +1,105 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package en + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestEnglishAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // stemming + { + input: []byte("books"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("book"), + Position: 1, + Start: 0, + End: 5, + }, + }, + }, + { + input: []byte("book"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("book"), + Position: 1, + Start: 0, + End: 4, + }, + }, + }, + // stop word removal + { + input: []byte("the"), + output: analysis.TokenStream{}, + }, + // possessive removal + { + input: []byte("steven's"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("steven"), + Position: 1, + Start: 0, + End: 8, + }, + }, + }, + { + input: []byte("steven\u2019s"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("steven"), + Position: 1, + Start: 0, + End: 10, + }, + }, + }, + { + input: []byte("steven\uFF07s"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("steven"), + Position: 1, + Start: 0, + End: 10, + }, + }, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %v, got %v", test.output, actual) + } + } +} diff --git a/analysis/lang/en/plural_stemmer.go b/analysis/lang/en/plural_stemmer.go new file mode 100644 index 0000000..7aebdc8 --- /dev/null +++ b/analysis/lang/en/plural_stemmer.go @@ -0,0 +1,177 @@ +/* + This code was ported from the Open Search Project + https://github.com/opensearch-project/OpenSearch/blob/main/modules/analysis-common/src/main/java/org/opensearch/analysis/common/EnglishPluralStemFilter.java + The algorithm itself was created by Mark Harwood + https://github.com/markharwood +*/ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package en + +import ( + "strings" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const PluralStemmerName = "stemmer_en_plural" + +type EnglishPluralStemmerFilter struct { +} + +func NewEnglishPluralStemmerFilter() *EnglishPluralStemmerFilter { + return &EnglishPluralStemmerFilter{} +} + +func (s *EnglishPluralStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + token.Term = []byte(stem(string(token.Term))) + } + + return input +} + +func EnglishPluralStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewEnglishPluralStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(PluralStemmerName, EnglishPluralStemmerFilterConstructor) + if err != nil { + panic(err) + } +} + +// ---------------------------------------------------------------------------- + +// Words ending in oes that retain the e when stemmed +var oesExceptions = []string{"shoes", "canoes", "oboes"} + +// Words ending in ches that retain the e when stemmed +var chesExceptions = []string{ + "cliches", + "avalanches", + "mustaches", + "moustaches", + "quiches", + "headaches", + "heartaches", + "porsches", + "tranches", + "caches", +} + +func stem(word string) string { + runes := []rune(strings.ToLower(word)) + + if len(runes) < 3 || runes[len(runes)-1] != 's' { + return string(runes) + } + + switch runes[len(runes)-2] { + case 'u': + fallthrough + case 's': + return string(runes) + case 'e': + // Modified ies->y logic from original s-stemmer - only work on strings > 4 + // so spies -> spy still but pies->pie. + // The original code also special-cased aies and eies for no good reason as far as I can tell. + // ( no words of consequence - eg http://www.thefreedictionary.com/words-that-end-in-aies ) + if len(runes) > 4 && runes[len(runes)-3] == 'i' { + runes[len(runes)-3] = 'y' + return string(runes[0 : len(runes)-2]) + } + + // Suffix rules to remove any dangling "e" + if len(runes) > 3 { + // xes (but >1 prefix so we can stem "boxes->box" but keep "axes->axe") + if len(runes) > 4 && runes[len(runes)-3] == 'x' { + return string(runes[0 : len(runes)-2]) + } + + // oes + if len(runes) > 3 && runes[len(runes)-3] == 'o' { + if isException(runes, oesExceptions) { + // Only remove the S + return string(runes[0 : len(runes)-1]) + } + // Remove the es + return string(runes[0 : len(runes)-2]) + } + + if len(runes) > 4 { + // shes/sses + if runes[len(runes)-4] == 's' && (runes[len(runes)-3] == 'h' || runes[len(runes)-3] == 's') { + return string(runes[0 : len(runes)-2]) + } + + // ches + if len(runes) > 4 { + if runes[len(runes)-4] == 'c' && runes[len(runes)-3] == 'h' { + if isException(runes, chesExceptions) { + // Only remove the S + return string(runes[0 : len(runes)-1]) + } + // Remove the es + return string(runes[0 : len(runes)-2]) + } + } + } + } + fallthrough + default: + return string(runes[0 : len(runes)-1]) + } +} + +func isException(word []rune, exceptions []string) bool { + for _, exception := range exceptions { + + exceptionRunes := []rune(exception) + + exceptionPos := len(exceptionRunes) - 1 + wordPos := len(word) - 1 + + matched := true + for exceptionPos >= 0 && wordPos >= 0 { + if exceptionRunes[exceptionPos] != word[wordPos] { + matched = false + break + } + exceptionPos-- + wordPos-- + } + if matched { + return true + } + } + return false +} diff --git a/analysis/lang/en/plural_stemmer_test.go b/analysis/lang/en/plural_stemmer_test.go new file mode 100644 index 0000000..b6c0028 --- /dev/null +++ b/analysis/lang/en/plural_stemmer_test.go @@ -0,0 +1,46 @@ +package en + +import "testing" + +func TestEnglishPluralStemmer(t *testing.T) { + data := []struct { + In, Out string + }{ + {"dresses", "dress"}, + {"dress", "dress"}, + {"axes", "axe"}, + {"ad", "ad"}, + {"ads", "ad"}, + {"gas", "ga"}, + {"sass", "sass"}, + {"berries", "berry"}, + {"dresses", "dress"}, + {"spies", "spy"}, + {"shoes", "shoe"}, + {"headaches", "headache"}, + {"computer", "computer"}, + {"dressing", "dressing"}, + {"clothes", "clothe"}, + {"DRESSES", "dress"}, + {"frog", "frog"}, + {"dress", "dress"}, + {"runs", "run"}, + {"pies", "pie"}, + {"foxes", "fox"}, + {"axes", "axe"}, + {"foes", "fo"}, + {"dishes", "dish"}, + {"snitches", "snitch"}, + {"cliches", "cliche"}, + {"forests", "forest"}, + {"yes", "ye"}, + } + + for _, datum := range data { + stemmed := stem(datum.In) + + if stemmed != datum.Out { + t.Errorf("expected %v but got %v", datum.Out, stemmed) + } + } +} diff --git a/analysis/lang/en/possessive_filter_en.go b/analysis/lang/en/possessive_filter_en.go new file mode 100644 index 0000000..42d51f0 --- /dev/null +++ b/analysis/lang/en/possessive_filter_en.go @@ -0,0 +1,70 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package en + +import ( + "unicode/utf8" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +// PossessiveName is the name PossessiveFilter is registered as +// in the bleve registry. +const PossessiveName = "possessive_en" + +const rightSingleQuotationMark = '’' +const apostrophe = '\'' +const fullWidthApostrophe = '＇' + +const apostropheChars = rightSingleQuotationMark + apostrophe + fullWidthApostrophe + +// PossessiveFilter implements a TokenFilter which +// strips the English possessive suffix ('s) from tokens. +// It handle a variety of apostrophe types, is case-insensitive +// and doesn't distinguish between possessive and contraction. +// (ie "She's So Rad" becomes "She So Rad") +type PossessiveFilter struct { +} + +func NewPossessiveFilter() *PossessiveFilter { + return &PossessiveFilter{} +} + +func (s *PossessiveFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + lastRune, lastRuneSize := utf8.DecodeLastRune(token.Term) + if lastRune == 's' || lastRune == 'S' { + nextLastRune, nextLastRuneSize := utf8.DecodeLastRune(token.Term[:len(token.Term)-lastRuneSize]) + if nextLastRune == rightSingleQuotationMark || + nextLastRune == apostrophe || + nextLastRune == fullWidthApostrophe { + token.Term = token.Term[:len(token.Term)-lastRuneSize-nextLastRuneSize] + } + } + } + return input +} + +func PossessiveFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewPossessiveFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(PossessiveName, PossessiveFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/en/possessive_filter_en_test.go b/analysis/lang/en/possessive_filter_en_test.go new file mode 100644 index 0000000..c45cfc9 --- /dev/null +++ b/analysis/lang/en/possessive_filter_en_test.go @@ -0,0 +1,142 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package en + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestEnglishPossessiveFilter(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("marty's"), + }, + &analysis.Token{ + Term: []byte("MARTY'S"), + }, + &analysis.Token{ + Term: []byte("marty’s"), + }, + &analysis.Token{ + Term: []byte("MARTY’S"), + }, + &analysis.Token{ + Term: []byte("marty＇s"), + }, + &analysis.Token{ + Term: []byte("MARTY＇S"), + }, + &analysis.Token{ + Term: []byte("m"), + }, + &analysis.Token{ + Term: []byte("s"), + }, + &analysis.Token{ + Term: []byte("'s"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("marty"), + }, + &analysis.Token{ + Term: []byte("MARTY"), + }, + &analysis.Token{ + Term: []byte("marty"), + }, + &analysis.Token{ + Term: []byte("MARTY"), + }, + &analysis.Token{ + Term: []byte("marty"), + }, + &analysis.Token{ + Term: []byte("MARTY"), + }, + &analysis.Token{ + Term: []byte("m"), + }, + &analysis.Token{ + Term: []byte("s"), + }, + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + } + + cache := registry.NewCache() + stemmerFilter, err := cache.TokenFilterNamed(PossessiveName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := stemmerFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output, actual) + } + } +} + +func BenchmarkEnglishPossessiveFilter(b *testing.B) { + + input := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("marty's"), + }, + &analysis.Token{ + Term: []byte("MARTY'S"), + }, + &analysis.Token{ + Term: []byte("marty’s"), + }, + &analysis.Token{ + Term: []byte("MARTY’S"), + }, + &analysis.Token{ + Term: []byte("marty＇s"), + }, + &analysis.Token{ + Term: []byte("MARTY＇S"), + }, + &analysis.Token{ + Term: []byte("m"), + }, + } + + cache := registry.NewCache() + stemmerFilter, err := cache.TokenFilterNamed(PossessiveName) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + + for i := 0; i < b.N; i++ { + stemmerFilter.Filter(input) + } + +} diff --git a/analysis/lang/en/stemmer_en_snowball.go b/analysis/lang/en/stemmer_en_snowball.go new file mode 100644 index 0000000..568a2b6 --- /dev/null +++ b/analysis/lang/en/stemmer_en_snowball.go @@ -0,0 +1,52 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package en + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/snowballstem" + "github.com/blevesearch/snowballstem/english" +) + +const SnowballStemmerName = "stemmer_en_snowball" + +type EnglishStemmerFilter struct { +} + +func NewEnglishStemmerFilter() *EnglishStemmerFilter { + return &EnglishStemmerFilter{} +} + +func (s *EnglishStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + env := snowballstem.NewEnv(string(token.Term)) + english.Stem(env) + token.Term = []byte(env.Current()) + } + return input +} + +func EnglishStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewEnglishStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(SnowballStemmerName, EnglishStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/en/stemmer_en_test.go b/analysis/lang/en/stemmer_en_test.go new file mode 100644 index 0000000..7435d98 --- /dev/null +++ b/analysis/lang/en/stemmer_en_test.go @@ -0,0 +1,79 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package en + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestSnowballEnglishStemmer(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("enjoy"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("enjoy"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("enjoyed"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("enjoy"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("enjoyable"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("enjoy"), + }, + }, + }, + } + + cache := registry.NewCache() + filter, err := cache.TokenFilterNamed(SnowballStemmerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := filter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/en/stop_filter_en.go b/analysis/lang/en/stop_filter_en.go new file mode 100644 index 0000000..0015ad6 --- /dev/null +++ b/analysis/lang/en/stop_filter_en.go @@ -0,0 +1,36 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package en + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/en/stop_words_en.go b/analysis/lang/en/stop_words_en.go new file mode 100644 index 0000000..d6ff496 --- /dev/null +++ b/analysis/lang/en/stop_words_en.go @@ -0,0 +1,347 @@ +package en + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_en" + +// EnglishStopWords is the built-in list of stopwords used by the "stop_en" TokenFilter. +// +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ +// ` was changed to ' to allow for literal string +var EnglishStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/english/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | An English stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | Many of the forms below are quite rare (e.g. "yourselves") but included for + | completeness. + + | PRONOUNS FORMS + | 1st person sing + +i | subject, always in upper case of course + +me | object +my | possessive adjective + | the possessive pronoun 'mine' is best suppressed, because of the + | sense of coal-mine etc. +myself | reflexive + | 1st person plural +we | subject + +| us | object + | care is required here because US = United States. It is usually + | safe to remove it if it is in lower case. +our | possessive adjective +ours | possessive pronoun +ourselves | reflexive + | second person (archaic 'thou' forms not included) +you | subject and object +your | possessive adjective +yours | possessive pronoun +yourself | reflexive (singular) +yourselves | reflexive (plural) + | third person singular +he | subject +him | object +his | possessive adjective and pronoun +himself | reflexive + +she | subject +her | object and possessive adjective +hers | possessive pronoun +herself | reflexive + +it | subject and object +its | possessive adjective +itself | reflexive + | third person plural +they | subject +them | object +their | possessive adjective +theirs | possessive pronoun +themselves | reflexive + | other forms (demonstratives, interrogatives) +what +which +who +whom +this +that +these +those + + | VERB FORMS (using F.R. Palmer's nomenclature) + | BE +am | 1st person, present +is | -s form (3rd person, present) +are | present +was | 1st person, past +were | past +be | infinitive +been | past participle +being | -ing form + | HAVE +have | simple +has | -s form +had | past +having | -ing form + | DO +do | simple +does | -s form +did | past +doing | -ing form + + | The forms below are, I believe, best omitted, because of the significant + | homonym forms: + + | He made a WILL + | old tin CAN + | merry month of MAY + | a smell of MUST + | fight the good fight with all thy MIGHT + + | would, could, should, ought might however be included + + | | AUXILIARIES + | | WILL + |will + +would + + | | SHALL + |shall + +should + + | | CAN + |can + +could + + | | MAY + |may + |might + | | MUST + |must + | | OUGHT + +ought + + | COMPOUND FORMS, increasingly encountered nowadays in 'formal' writing + | pronoun + verb + +i'm +you're +he's +she's +it's +we're +they're +i've +you've +we've +they've +i'd +you'd +he'd +she'd +we'd +they'd +i'll +you'll +he'll +she'll +we'll +they'll + + | verb + negation + +isn't +aren't +wasn't +weren't +hasn't +haven't +hadn't +doesn't +don't +didn't + + | auxiliary + negation + +won't +wouldn't +shan't +shouldn't +can't +cannot +couldn't +mustn't + + | miscellaneous forms + +let's +that's +who's +what's +here's +there's +when's +where's +why's +how's + + | rarer forms + + | daren't needn't + + | doubtful forms + + | oughtn't mightn't + + | ARTICLES +a +an +the + + | THE REST (Overlap among prepositions, conjunctions, adverbs etc is so + | high, that classification is pointless.) +and +but +if +or +because +as +until +while + +of +at +by +for +with +about +against +between +into +through +during +before +after +above +below +to +from +up +down +in +out +on +off +over +under + +again +further +then +once + +here +there +when +where +why +how + +all +any +both +each +few +more +most +other +some +such + +no +nor +not +only +own +same +so +than +too +very + + | Just for the record, the following words are among the commonest in English + + | one + | every + | least + | less + | many + | now + | ever + | never + | say + | says + | said + | also + | get + | go + | goes + | just + | made + | make + | put + | see + | seen + | whether + | like + | well + | back + | even + | still + | way + | take + | since + | another + | however + | two + | three + | four + | five + | first + | second + | new + | old + | high + | long +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(EnglishStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/es/analyzer_es.go b/analysis/lang/es/analyzer_es.go new file mode 100644 index 0000000..eb4bca3 --- /dev/null +++ b/analysis/lang/es/analyzer_es.go @@ -0,0 +1,66 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package es + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +) + +const AnalyzerName = "es" + +func AnalyzerConstructor(config map[string]interface{}, + cache *registry.Cache) (analysis.Analyzer, error) { + unicodeTokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + normalizeEsFilter, err := cache.TokenFilterNamed(NormalizeName) + if err != nil { + return nil, err + } + stopEsFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + lightStemmerEsFilter, err := cache.TokenFilterNamed(LightStemmerName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: unicodeTokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + stopEsFilter, + normalizeEsFilter, + lightStemmerEsFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/es/analyzer_es_test.go b/analysis/lang/es/analyzer_es_test.go new file mode 100644 index 0000000..ad3b1f6 --- /dev/null +++ b/analysis/lang/es/analyzer_es_test.go @@ -0,0 +1,122 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package es + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestSpanishAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // stemming + { + input: []byte("chicana"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chican"), + Position: 1, + Start: 0, + End: 7, + }, + }, + }, + { + input: []byte("chicano"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chican"), + Position: 1, + Start: 0, + End: 7, + }, + }, + }, + // added by marty for better coverage + { + input: []byte("yeses"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("yes"), + Position: 1, + Start: 0, + End: 5, + }, + }, + }, + { + input: []byte("jaeces"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("jaez"), + Position: 1, + Start: 0, + End: 6, + }, + }, + }, + { + input: []byte("arcos"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("arc"), + Position: 1, + Start: 0, + End: 5, + }, + }, + }, + { + input: []byte("caos"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("caos"), + Position: 1, + Start: 0, + End: 4, + }, + }, + }, + { + input: []byte("parecer"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("parecer"), + Position: 1, + Start: 0, + End: 7, + }, + }, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %v, got %v", test.output, actual) + } + } +} diff --git a/analysis/lang/es/light_stemmer_es.go b/analysis/lang/es/light_stemmer_es.go new file mode 100644 index 0000000..22ed97f --- /dev/null +++ b/analysis/lang/es/light_stemmer_es.go @@ -0,0 +1,78 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package es + +import ( + "bytes" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const LightStemmerName = "stemmer_es_light" + +type SpanishLightStemmerFilter struct { +} + +func NewSpanishLightStemmerFilter() *SpanishLightStemmerFilter { + return &SpanishLightStemmerFilter{} +} + +func (s *SpanishLightStemmerFilter) Filter( + input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + runes := bytes.Runes(token.Term) + runes = stem(runes) + token.Term = analysis.BuildTermFromRunes(runes) + } + return input +} + +func stem(input []rune) []rune { + l := len(input) + if l < 5 { + return input + } + + switch input[l-1] { + case 'o', 'a', 'e': + return input[:l-1] + case 's': + if input[l-2] == 'e' && input[l-3] == 's' && input[l-4] == 'e' { + return input[:l-2] + } + if input[l-2] == 'e' && input[l-3] == 'c' { + input[l-3] = 'z' + return input[:l-2] + } + if input[l-2] == 'o' || input[l-2] == 'a' || input[l-2] == 'e' { + return input[:l-2] + } + } + + return input +} + +func SpanishLightStemmerFilterConstructor(config map[string]interface{}, + cache *registry.Cache) (analysis.TokenFilter, error) { + return NewSpanishLightStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(LightStemmerName, SpanishLightStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/es/spanish_normalize.go b/analysis/lang/es/spanish_normalize.go new file mode 100644 index 0000000..8ed42a0 --- /dev/null +++ b/analysis/lang/es/spanish_normalize.go @@ -0,0 +1,70 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package es + +import ( + "bytes" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const NormalizeName = "normalize_es" + +type SpanishNormalizeFilter struct { +} + +func NewSpanishNormalizeFilter() *SpanishNormalizeFilter { + return &SpanishNormalizeFilter{} +} + +func (s *SpanishNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + term := normalize(token.Term) + token.Term = term + } + return input +} + +func normalize(input []byte) []byte { + runes := bytes.Runes(input) + for i := 0; i < len(runes); i++ { + switch runes[i] { + case 'à', 'á', 'â', 'ä': + runes[i] = 'a' + case 'ò', 'ó', 'ô', 'ö': + runes[i] = 'o' + case 'è', 'é', 'ê', 'ë': + runes[i] = 'e' + case 'ù', 'ú', 'û', 'ü': + runes[i] = 'u' + case 'ì', 'í', 'î', 'ï': + runes[i] = 'i' + } + } + + return analysis.BuildTermFromRunes(runes) +} + +func NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewSpanishNormalizeFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/es/spanish_normalize_test.go b/analysis/lang/es/spanish_normalize_test.go new file mode 100644 index 0000000..b2f9df5 --- /dev/null +++ b/analysis/lang/es/spanish_normalize_test.go @@ -0,0 +1,112 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package es + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestSpanishNormalizeFilter(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Guía"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Guia"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Belcebú"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Belcebu"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Limón"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Limon"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("agüero"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("aguero"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("laúd"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("laud"), + }, + }, + }, + // empty + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + } + + spanishNormalizeFilter := NewSpanishNormalizeFilter() + for _, test := range tests { + actual := spanishNormalizeFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %#v, got %#v", test.output, actual) + t.Errorf("expected %s(% x), got %s(% x)", test.output[0].Term, test.output[0].Term, actual[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/es/stemmer_es_snowball.go b/analysis/lang/es/stemmer_es_snowball.go new file mode 100644 index 0000000..d09a11b --- /dev/null +++ b/analysis/lang/es/stemmer_es_snowball.go @@ -0,0 +1,52 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package es + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/snowballstem" + "github.com/blevesearch/snowballstem/spanish" +) + +const SnowballStemmerName = "stemmer_es_snowball" + +type SpanishStemmerFilter struct { +} + +func NewSpanishStemmerFilter() *SpanishStemmerFilter { + return &SpanishStemmerFilter{} +} + +func (s *SpanishStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + env := snowballstem.NewEnv(string(token.Term)) + spanish.Stem(env) + token.Term = []byte(env.Current()) + } + return input +} + +func SpanishStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewSpanishStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(SnowballStemmerName, SpanishStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/es/stemmer_es_snowball_test.go b/analysis/lang/es/stemmer_es_snowball_test.go new file mode 100644 index 0000000..0531865 --- /dev/null +++ b/analysis/lang/es/stemmer_es_snowball_test.go @@ -0,0 +1,79 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package es + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestSnowballSpanishStemmer(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("agresivos"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("agres"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("agresivamente"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("agres"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("agresividad"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("agres"), + }, + }, + }, + } + + cache := registry.NewCache() + filter, err := cache.TokenFilterNamed(SnowballStemmerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := filter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/es/stop_filter_es.go b/analysis/lang/es/stop_filter_es.go new file mode 100644 index 0000000..f0a6d20 --- /dev/null +++ b/analysis/lang/es/stop_filter_es.go @@ -0,0 +1,36 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package es + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, + cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/es/stop_words_es.go b/analysis/lang/es/stop_words_es.go new file mode 100644 index 0000000..4c2105b --- /dev/null +++ b/analysis/lang/es/stop_words_es.go @@ -0,0 +1,383 @@ +package es + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_es" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ +// ` was changed to ' to allow for literal string + +var SpanishStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/spanish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A Spanish stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + + | The following is a ranked list (commonest to rarest) of stopwords + | deriving from a large sample of text. + + | Extra words have been added at the end. + +de | from, of +la | the, her +que | who, that +el | the +en | in +y | and +a | to +los | the, them +del | de + el +se | himself, from him etc +las | the, them +por | for, by, etc +un | a +para | for +con | with +no | no +una | a +su | his, her +al | a + el + | es from SER +lo | him +como | how +más | more +pero | pero +sus | su plural +le | to him, her +ya | already +o | or + | fue from SER +este | this + | ha from HABER +sí | himself etc +porque | because +esta | this + | son from SER +entre | between + | está from ESTAR +cuando | when +muy | very +sin | without +sobre | on + | ser from SER + | tiene from TENER +también | also +me | me +hasta | until +hay | there is/are +donde | where + | han from HABER +quien | whom, that + | están from ESTAR + | estado from ESTAR +desde | from +todo | all +nos | us +durante | during + | estados from ESTAR +todos | all +uno | a +les | to them +ni | nor +contra | against +otros | other + | fueron from SER +ese | that +eso | that + | había from HABER +ante | before +ellos | they +e | and (variant of y) +esto | this +mí | me +antes | before +algunos | some +qué | what? +unos | a +yo | I +otro | other +otras | other +otra | other +él | he +tanto | so much, many +esa | that +estos | these +mucho | much, many +quienes | who +nada | nothing +muchos | many +cual | who + | sea from SER +poco | few +ella | she +estar | to be + | haber from HABER +estas | these + | estaba from ESTAR + | estamos from ESTAR +algunas | some +algo | something +nosotros | we + + | other forms + +mi | me +mis | mi plural +tú | thou +te | thee +ti | thee +tu | thy +tus | tu plural +ellas | they +nosotras | we +vosotros | you +vosotras | you +os | you +mío | mine +mía | +míos | +mías | +tuyo | thine +tuya | +tuyos | +tuyas | +suyo | his, hers, theirs +suya | +suyos | +suyas | +nuestro | ours +nuestra | +nuestros | +nuestras | +vuestro | yours +vuestra | +vuestros | +vuestras | +esos | those +esas | those + + | forms of estar, to be (not including the infinitive): +estoy +estás +está +estamos +estáis +están +esté +estés +estemos +estéis +estén +estaré +estarás +estará +estaremos +estaréis +estarán +estaría +estarías +estaríamos +estaríais +estarían +estaba +estabas +estábamos +estabais +estaban +estuve +estuviste +estuvo +estuvimos +estuvisteis +estuvieron +estuviera +estuvieras +estuviéramos +estuvierais +estuvieran +estuviese +estuvieses +estuviésemos +estuvieseis +estuviesen +estando +estado +estada +estados +estadas +estad + + | forms of haber, to have (not including the infinitive): +he +has +ha +hemos +habéis +han +haya +hayas +hayamos +hayáis +hayan +habré +habrás +habrá +habremos +habréis +habrán +habría +habrías +habríamos +habríais +habrían +había +habías +habíamos +habíais +habían +hube +hubiste +hubo +hubimos +hubisteis +hubieron +hubiera +hubieras +hubiéramos +hubierais +hubieran +hubiese +hubieses +hubiésemos +hubieseis +hubiesen +habiendo +habido +habida +habidos +habidas + + | forms of ser, to be (not including the infinitive): +soy +eres +es +somos +sois +son +sea +seas +seamos +seáis +sean +seré +serás +será +seremos +seréis +serán +sería +serías +seríamos +seríais +serían +era +eras +éramos +erais +eran +fui +fuiste +fue +fuimos +fuisteis +fueron +fuera +fueras +fuéramos +fuerais +fueran +fuese +fueses +fuésemos +fueseis +fuesen +siendo +sido + | sed also means 'thirst' + + | forms of tener, to have (not including the infinitive): +tengo +tienes +tiene +tenemos +tenéis +tienen +tenga +tengas +tengamos +tengáis +tengan +tendré +tendrás +tendrá +tendremos +tendréis +tendrán +tendría +tendrías +tendríamos +tendríais +tendrían +tenía +tenías +teníamos +teníais +tenían +tuve +tuviste +tuvo +tuvimos +tuvisteis +tuvieron +tuviera +tuvieras +tuviéramos +tuvierais +tuvieran +tuviese +tuvieses +tuviésemos +tuvieseis +tuviesen +teniendo +tenido +tenida +tenidos +tenidas +tened + +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(SpanishStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/eu/stop_filter_eu.go b/analysis/lang/eu/stop_filter_eu.go new file mode 100644 index 0000000..12d5150 --- /dev/null +++ b/analysis/lang/eu/stop_filter_eu.go @@ -0,0 +1,36 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eu + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/eu/stop_words_eu.go b/analysis/lang/eu/stop_words_eu.go new file mode 100644 index 0000000..372914c --- /dev/null +++ b/analysis/lang/eu/stop_words_eu.go @@ -0,0 +1,126 @@ +package eu + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_eu" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ +// ` was changed to ' to allow for literal string + +var BasqueStopWords = []byte(`# example set of basque stopwords +al +anitz +arabera +asko +baina +bat +batean +batek +bati +batzuei +batzuek +batzuetan +batzuk +bera +beraiek +berau +berauek +bere +berori +beroriek +beste +bezala +da +dago +dira +ditu +du +dute +edo +egin +ere +eta +eurak +ez +gainera +gu +gutxi +guzti +haiei +haiek +haietan +hainbeste +hala +han +handik +hango +hara +hari +hark +hartan +hau +hauei +hauek +hauetan +hemen +hemendik +hemengo +hi +hona +honek +honela +honetan +honi +hor +hori +horiei +horiek +horietan +horko +horra +horrek +horrela +horretan +horri +hortik +hura +izan +ni +noiz +nola +non +nondik +nongo +nor +nora +ze +zein +zen +zenbait +zenbat +zer +zergatik +ziren +zituen +zu +zuek +zuen +zuten +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(BasqueStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/fa/analyzer_fa.go b/analysis/lang/fa/analyzer_fa.go new file mode 100644 index 0000000..da2cb28 --- /dev/null +++ b/analysis/lang/fa/analyzer_fa.go @@ -0,0 +1,74 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fa + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/bleve/v2/analysis/char/zerowidthnonjoiner" + "github.com/blevesearch/bleve/v2/analysis/lang/ar" + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +) + +const AnalyzerName = "fa" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + zFilter, err := cache.CharFilterNamed(zerowidthnonjoiner.Name) + if err != nil { + return nil, err + } + unicodeTokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + normArFilter, err := cache.TokenFilterNamed(ar.NormalizeName) + if err != nil { + return nil, err + } + normFaFilter, err := cache.TokenFilterNamed(NormalizeName) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopFaFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + CharFilters: []analysis.CharFilter{ + zFilter, + }, + Tokenizer: unicodeTokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + normArFilter, + normFaFilter, + stopFaFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/fa/analyzer_fa_test.go b/analysis/lang/fa/analyzer_fa_test.go new file mode 100644 index 0000000..f648261 --- /dev/null +++ b/analysis/lang/fa/analyzer_fa_test.go @@ -0,0 +1,684 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fa + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestPersianAnalyzerVerbs(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // active present indicative + { + input: []byte("می‌خورد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورد"), + }, + }, + }, + // active preterite indicative + { + input: []byte("خورد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورد"), + }, + }, + }, + // active imperfective preterite indicative + { + input: []byte("می‌خورد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورد"), + }, + }, + }, + // active future indicative + { + input: []byte("خواهد خورد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورد"), + }, + }, + }, + // active present progressive indicative + { + input: []byte("دارد می‌خورد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورد"), + }, + }, + }, + // active preterite progressive indicative + { + input: []byte("داشت می‌خورد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورد"), + }, + }, + }, + // active perfect indicative + { + input: []byte("خورده‌است"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // active imperfective perfect indicative + { + input: []byte("می‌خورده‌است"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // active pluperfect indicative + { + input: []byte("خورده بود"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // active imperfective pluperfect indicative + { + input: []byte("می‌خورده بود"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // active preterite subjunctive + { + input: []byte("خورده باشد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // active imperfective preterite subjunctive + { + input: []byte("می‌خورده باشد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // active pluperfect subjunctive + { + input: []byte("خورده بوده باشد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // active imperfective pluperfect subjunctive + { + input: []byte("می‌خورده بوده باشد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive present indicative + { + input: []byte("خورده می‌شود"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive preterite indicative + { + input: []byte("خورده شد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive imperfective preterite indicative + { + input: []byte("خورده می‌شد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive perfect indicative + { + input: []byte("خورده شده‌است"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive imperfective perfect indicative + { + input: []byte("خورده می‌شده‌است"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive pluperfect indicative + { + input: []byte("خورده شده بود"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive imperfective pluperfect indicative + { + input: []byte("خورده می‌شده بود"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive future indicative + { + input: []byte("خورده خواهد شد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive present progressive indicative + { + input: []byte("دارد خورده می‌شود"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive preterite progressive indicative + { + input: []byte("داشت خورده می‌شد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive present subjunctive + { + input: []byte("خورده شود"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive preterite subjunctive + { + input: []byte("خورده شده باشد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive imperfective preterite subjunctive + { + input: []byte("خورده می‌شده باشد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive pluperfect subjunctive + { + input: []byte("خورده شده بوده باشد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive imperfective pluperfect subjunctive + { + input: []byte("خورده می‌شده بوده باشد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // active present subjunctive + { + input: []byte("بخورد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("بخورد"), + }, + }, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if len(actual) != len(test.output) { + t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) + } + for i, tok := range actual { + if !reflect.DeepEqual(tok.Term, test.output[i].Term) { + t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) + } + } + } +} + +func TestPersianAnalyzerVerbsDefective(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // active present indicative + { + input: []byte("مي خورد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورد"), + }, + }, + }, + // active preterite indicative + { + input: []byte("خورد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورد"), + }, + }, + }, + // active imperfective preterite indicative + { + input: []byte("مي خورد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورد"), + }, + }, + }, + // active future indicative + { + input: []byte("خواهد خورد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورد"), + }, + }, + }, + // active present progressive indicative + { + input: []byte("دارد مي خورد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورد"), + }, + }, + }, + // active preterite progressive indicative + { + input: []byte("داشت مي خورد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورد"), + }, + }, + }, + // active perfect indicative + { + input: []byte("خورده است"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // active imperfective perfect indicative + { + input: []byte("مي خورده است"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // active pluperfect indicative + { + input: []byte("خورده بود"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // active imperfective pluperfect indicative + { + input: []byte("مي خورده بود"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // active preterite subjunctive + { + input: []byte("خورده باشد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // active imperfective preterite subjunctive + { + input: []byte("مي خورده باشد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // active pluperfect subjunctive + { + input: []byte("خورده بوده باشد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // active imperfective pluperfect subjunctive + { + input: []byte("مي خورده بوده باشد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive present indicative + { + input: []byte("خورده مي شود"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive preterite indicative + { + input: []byte("خورده شد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive imperfective preterite indicative + { + input: []byte("خورده مي شد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive perfect indicative + { + input: []byte("خورده شده است"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive imperfective perfect indicative + { + input: []byte("خورده مي شده است"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive pluperfect indicative + { + input: []byte("خورده شده بود"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive imperfective pluperfect indicative + { + input: []byte("خورده مي شده بود"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive future indicative + { + input: []byte("خورده خواهد شد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive present progressive indicative + { + input: []byte("دارد خورده مي شود"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive preterite progressive indicative + { + input: []byte("داشت خورده مي شد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive present subjunctive + { + input: []byte("خورده شود"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive preterite subjunctive + { + input: []byte("خورده شده باشد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive imperfective preterite subjunctive + { + input: []byte("خورده مي شده باشد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive pluperfect subjunctive + { + input: []byte("خورده شده بوده باشد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // passive imperfective pluperfect subjunctive + { + input: []byte("خورده مي شده بوده باشد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + // active present subjunctive + { + input: []byte("بخورد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("بخورد"), + }, + }, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if len(actual) != len(test.output) { + t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) + } + for i, tok := range actual { + if !reflect.DeepEqual(tok.Term, test.output[i].Term) { + t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) + } + } + } +} + +func TestPersianAnalyzerOthers(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // nouns + { + input: []byte("برگ ها"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("برگ"), + }, + }, + }, + { + input: []byte("برگ‌ها"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("برگ"), + }, + }, + }, + // non persian + { + input: []byte("English test."), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("english"), + }, + &analysis.Token{ + Term: []byte("test"), + }, + }, + }, + // others + { + input: []byte("خورده مي شده بوده باشد"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("خورده"), + }, + }, + }, + { + input: []byte("برگ‌ها"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("برگ"), + }, + }, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if len(actual) != len(test.output) { + t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) + } + for i, tok := range actual { + if !reflect.DeepEqual(tok.Term, test.output[i].Term) { + t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) + } + } + } +} diff --git a/analysis/lang/fa/persian_normalize.go b/analysis/lang/fa/persian_normalize.go new file mode 100644 index 0000000..7d73ded --- /dev/null +++ b/analysis/lang/fa/persian_normalize.go @@ -0,0 +1,80 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fa + +import ( + "bytes" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const NormalizeName = "normalize_fa" + +const ( + Yeh = '\u064A' + FarsiYeh = '\u06CC' + YehBarree = '\u06D2' + Keheh = '\u06A9' + Kaf = '\u0643' + HamzaAbove = '\u0654' + HehYeh = '\u06C0' + HehGoal = '\u06C1' + Heh = '\u0647' +) + +type PersianNormalizeFilter struct { +} + +func NewPersianNormalizeFilter() *PersianNormalizeFilter { + return &PersianNormalizeFilter{} +} + +func (s *PersianNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + term := normalize(token.Term) + token.Term = term + } + return input +} + +func normalize(input []byte) []byte { + runes := bytes.Runes(input) + for i := 0; i < len(runes); i++ { + switch runes[i] { + case FarsiYeh, YehBarree: + runes[i] = Yeh + case Keheh: + runes[i] = Kaf + case HehYeh, HehGoal: + runes[i] = Heh + case HamzaAbove: // necessary for HEH + HAMZA + runes = analysis.DeleteRune(runes, i) + i-- + } + } + return analysis.BuildTermFromRunes(runes) +} + +func NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewPersianNormalizeFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/fa/persian_normalize_test.go b/analysis/lang/fa/persian_normalize_test.go new file mode 100644 index 0000000..4511cba --- /dev/null +++ b/analysis/lang/fa/persian_normalize_test.go @@ -0,0 +1,130 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fa + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestPersianNormalizeFilter(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + // FarsiYeh + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("های"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("هاي"), + }, + }, + }, + // YehBarree + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("هاے"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("هاي"), + }, + }, + }, + // Keheh + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("کشاندن"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("كشاندن"), + }, + }, + }, + // HehYeh + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("كتابۀ"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("كتابه"), + }, + }, + }, + // HehHamzaAbove + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("كتابهٔ"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("كتابه"), + }, + }, + }, + // HehGoal + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("زادہ"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("زاده"), + }, + }, + }, + // empty + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + } + + persianNormalizeFilter := NewPersianNormalizeFilter() + for _, test := range tests { + actual := persianNormalizeFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %#v, got %#v", test.output, actual) + t.Errorf("expected % x, got % x", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/fa/stop_filter_fa.go b/analysis/lang/fa/stop_filter_fa.go new file mode 100644 index 0000000..bcc237b --- /dev/null +++ b/analysis/lang/fa/stop_filter_fa.go @@ -0,0 +1,36 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fa + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/fa/stop_words_fa.go b/analysis/lang/fa/stop_words_fa.go new file mode 100644 index 0000000..4183004 --- /dev/null +++ b/analysis/lang/fa/stop_words_fa.go @@ -0,0 +1,340 @@ +package fa + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_fa" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ +// ` was changed to ' to allow for literal string + +var PersianStopWords = []byte(`# This file was created by Jacques Savoy and is distributed under the BSD license. +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# Also see http://www.opensource.org/licenses/bsd-license.html +# Note: by default this file is used after normalization, so when adding entries +# to this file, use the arabic 'ي' instead of 'ی' +انان +نداشته +سراسر +خياه +ايشان +وي +تاكنون +بيشتري +دوم +پس +ناشي +وگو +يا +داشتند +سپس +هنگام +هرگز +پنج +نشان +امسال +ديگر +گروهي +شدند +چطور +ده +و +دو +نخستين +ولي +چرا +چه +وسط +ه +كدام +قابل +يك +رفت +هفت +همچنين +در +هزار +بله +بلي +شايد +اما +شناسي +گرفته +دهد +داشته +دانست +داشتن +خواهيم +ميليارد +وقتيكه +امد +خواهد +جز +اورده +شده +بلكه +خدمات +شدن +برخي +نبود +بسياري +جلوگيري +حق +كردند +نوعي +بعري +نكرده +نظير +نبايد +بوده +بودن +داد +اورد +هست +جايي +شود +دنبال +داده +بايد +سابق +هيچ +همان +انجا +كمتر +كجاست +گردد +كسي +تر +مردم +تان +دادن +بودند +سري +جدا +ندارند +مگر +يكديگر +دارد +دهند +بنابراين +هنگامي +سمت +جا +انچه +خود +دادند +زياد +دارند +اثر +بدون +بهترين +بيشتر +البته +به +براساس +بيرون +كرد +بعضي +گرفت +توي +اي +ميليون +او +جريان +تول +بر +مانند +برابر +باشيم +مدتي +گويند +اكنون +تا +تنها +جديد +چند +بي +نشده +كردن +كردم +گويد +كرده +كنيم +نمي +نزد +روي +قصد +فقط +بالاي +ديگران +اين +ديروز +توسط +سوم +ايم +دانند +سوي +استفاده +شما +كنار +داريم +ساخته +طور +امده +رفته +نخست +بيست +نزديك +طي +كنيد +از +انها +تمامي +داشت +يكي +طريق +اش +چيست +روب +نمايد +گفت +چندين +چيزي +تواند +ام +ايا +با +ان +ايد +ترين +اينكه +ديگري +راه +هايي +بروز +همچنان +پاعين +كس +حدود +مختلف +مقابل +چيز +گيرد +ندارد +ضد +همچون +سازي +شان +مورد +باره +مرسي +خويش +برخوردار +چون +خارج +شش +هنوز +تحت +ضمن +هستيم +گفته +فكر +بسيار +پيش +براي +روزهاي +انكه +نخواهد +بالا +كل +وقتي +كي +چنين +كه +گيري +نيست +است +كجا +كند +نيز +يابد +بندي +حتي +توانند +عقب +خواست +كنند +بين +تمام +همه +ما +باشند +مثل +شد +اري +باشد +اره +طبق +بعد +اگر +صورت +غير +جاي +بيش +ريزي +اند +زيرا +چگونه +بار +لطفا +مي +درباره +من +ديده +همين +گذاري +برداري +علت +گذاشته +هم +فوق +نه +ها +شوند +اباد +همواره +هر +اول +خواهند +چهار +نام +امروز +مان +هاي +قبل +كنم +سعي +تازه +را +هستند +زير +جلوي +عنوان +بود +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(PersianStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/fi/analyzer_fi.go b/analysis/lang/fi/analyzer_fi.go new file mode 100644 index 0000000..b2d8f1d --- /dev/null +++ b/analysis/lang/fi/analyzer_fi.go @@ -0,0 +1,60 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fi + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +) + +const AnalyzerName = "fi" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + unicodeTokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopFiFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + stemmerFiFilter, err := cache.TokenFilterNamed(SnowballStemmerName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: unicodeTokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + stopFiFilter, + stemmerFiFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/fi/analyzer_fi_test.go b/analysis/lang/fi/analyzer_fi_test.go new file mode 100644 index 0000000..45aa242 --- /dev/null +++ b/analysis/lang/fi/analyzer_fi_test.go @@ -0,0 +1,70 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fi + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestFinishAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // stemming + { + input: []byte("edeltäjiinsä"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("edeltäj"), + }, + }, + }, + { + input: []byte("edeltäjistään"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("edeltäj"), + }, + }, + }, + // stop word + { + input: []byte("olla"), + output: analysis.TokenStream{}, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if len(actual) != len(test.output) { + t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) + } + for i, tok := range actual { + if !reflect.DeepEqual(tok.Term, test.output[i].Term) { + t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) + } + } + } +} diff --git a/analysis/lang/fi/stemmer_fi.go b/analysis/lang/fi/stemmer_fi.go new file mode 100644 index 0000000..9b956c7 --- /dev/null +++ b/analysis/lang/fi/stemmer_fi.go @@ -0,0 +1,52 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fi + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/snowballstem" + "github.com/blevesearch/snowballstem/finnish" +) + +const SnowballStemmerName = "stemmer_fi_snowball" + +type FinnishStemmerFilter struct { +} + +func NewFinnishStemmerFilter() *FinnishStemmerFilter { + return &FinnishStemmerFilter{} +} + +func (s *FinnishStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + env := snowballstem.NewEnv(string(token.Term)) + finnish.Stem(env) + token.Term = []byte(env.Current()) + } + return input +} + +func FinnishStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewFinnishStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(SnowballStemmerName, FinnishStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/fi/stop_filter_fi.go b/analysis/lang/fi/stop_filter_fi.go new file mode 100644 index 0000000..55e463b --- /dev/null +++ b/analysis/lang/fi/stop_filter_fi.go @@ -0,0 +1,36 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fi + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/fi/stop_words_fi.go b/analysis/lang/fi/stop_words_fi.go new file mode 100644 index 0000000..e7fabf9 --- /dev/null +++ b/analysis/lang/fi/stop_words_fi.go @@ -0,0 +1,124 @@ +package fi + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_fi" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ +// ` was changed to ' to allow for literal string + +var FinnishStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/finnish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + +| forms of BE + +olla +olen +olet +on +olemme +olette +ovat +ole | negative form + +oli +olisi +olisit +olisin +olisimme +olisitte +olisivat +olit +olin +olimme +olitte +olivat +ollut +olleet + +en | negation +et +ei +emme +ette +eivät + +|Nom Gen Acc Part Iness Elat Illat Adess Ablat Allat Ess Trans +minä minun minut minua minussa minusta minuun minulla minulta minulle | I +sinä sinun sinut sinua sinussa sinusta sinuun sinulla sinulta sinulle | you +hän hänen hänet häntä hänessä hänestä häneen hänellä häneltä hänelle | he she +me meidän meidät meitä meissä meistä meihin meillä meiltä meille | we +te teidän teidät teitä teissä teistä teihin teillä teiltä teille | you +he heidän heidät heitä heissä heistä heihin heillä heiltä heille | they + +tämä tämän tätä tässä tästä tähän tallä tältä tälle tänä täksi | this +tuo tuon tuotä tuossa tuosta tuohon tuolla tuolta tuolle tuona tuoksi | that +se sen sitä siinä siitä siihen sillä siltä sille sinä siksi | it +nämä näiden näitä näissä näistä näihin näillä näiltä näille näinä näiksi | these +nuo noiden noita noissa noista noihin noilla noilta noille noina noiksi | those +ne niiden niitä niissä niistä niihin niillä niiltä niille niinä niiksi | they + +kuka kenen kenet ketä kenessä kenestä keneen kenellä keneltä kenelle kenenä keneksi| who +ketkä keiden ketkä keitä keissä keistä keihin keillä keiltä keille keinä keiksi | (pl) +mikä minkä minkä mitä missä mistä mihin millä miltä mille minä miksi | which what +mitkä | (pl) + +joka jonka jota jossa josta johon jolla jolta jolle jona joksi | who which +jotka joiden joita joissa joista joihin joilla joilta joille joina joiksi | (pl) + +| conjunctions + +että | that +ja | and +jos | if +koska | because +kuin | than +mutta | but +niin | so +sekä | and +sillä | for +tai | or +vaan | but +vai | or +vaikka | although + + +| prepositions + +kanssa | with +mukaan | according to +noin | about +poikki | across +yli | over, across + +| other + +kun | when +niin | so +nyt | now +itse | self + +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(FinnishStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/fr/analyzer_fr.go b/analysis/lang/fr/analyzer_fr.go new file mode 100644 index 0000000..0808da0 --- /dev/null +++ b/analysis/lang/fr/analyzer_fr.go @@ -0,0 +1,65 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fr + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +) + +const AnalyzerName = "fr" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + tokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + elisionFilter, err := cache.TokenFilterNamed(ElisionName) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopFrFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + stemmerFrFilter, err := cache.TokenFilterNamed(LightStemmerName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: tokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + elisionFilter, + stopFrFilter, + stemmerFrFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/fr/analyzer_fr_test.go b/analysis/lang/fr/analyzer_fr_test.go new file mode 100644 index 0000000..38f89e0 --- /dev/null +++ b/analysis/lang/fr/analyzer_fr_test.go @@ -0,0 +1,209 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fr + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestFrenchAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + { + input: []byte(""), + output: analysis.TokenStream{}, + }, + { + input: []byte("chien chat cheval"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chien"), + }, + &analysis.Token{ + Term: []byte("chat"), + }, + &analysis.Token{ + Term: []byte("cheval"), + }, + }, + }, + { + input: []byte("chien CHAT CHEVAL"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chien"), + }, + &analysis.Token{ + Term: []byte("chat"), + }, + &analysis.Token{ + Term: []byte("cheval"), + }, + }, + }, + { + input: []byte(" chien ,? + = - CHAT /: > CHEVAL"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chien"), + }, + &analysis.Token{ + Term: []byte("chat"), + }, + &analysis.Token{ + Term: []byte("cheval"), + }, + }, + }, + { + input: []byte("chien++"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chien"), + }, + }, + }, + { + input: []byte("mot \"entreguillemet\""), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("mot"), + }, + &analysis.Token{ + Term: []byte("entreguilemet"), + }, + }, + }, + { + input: []byte("Jean-François"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("jean"), + }, + &analysis.Token{ + Term: []byte("francoi"), + }, + }, + }, + // stop words + { + input: []byte("le la chien les aux chat du des à cheval"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chien"), + }, + &analysis.Token{ + Term: []byte("chat"), + }, + &analysis.Token{ + Term: []byte("cheval"), + }, + }, + }, + // nouns and adjectives + { + input: []byte("lances chismes habitable chiste éléments captifs"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("lanc"), + }, + &analysis.Token{ + Term: []byte("chism"), + }, + &analysis.Token{ + Term: []byte("habitabl"), + }, + &analysis.Token{ + Term: []byte("chist"), + }, + &analysis.Token{ + Term: []byte("element"), + }, + &analysis.Token{ + Term: []byte("captif"), + }, + }, + }, + // verbs + { + input: []byte("finissions souffrirent rugissante"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("finision"), + }, + &analysis.Token{ + Term: []byte("soufrirent"), + }, + &analysis.Token{ + Term: []byte("rugisant"), + }, + }, + }, + { + input: []byte("C3PO aujourd'hui oeuf ïâöûàä anticonstitutionnellement Java++ "), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("c3po"), + }, + &analysis.Token{ + Term: []byte("aujourd'hui"), + }, + &analysis.Token{ + Term: []byte("oeuf"), + }, + &analysis.Token{ + Term: []byte("ïaöuaä"), + }, + &analysis.Token{ + Term: []byte("anticonstitutionel"), + }, + &analysis.Token{ + Term: []byte("java"), + }, + }, + }, + { + input: []byte("propriétaire"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("proprietair"), + }, + }, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if len(actual) != len(test.output) { + t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) + } + for i, tok := range actual { + if !reflect.DeepEqual(tok.Term, test.output[i].Term) { + t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) + } + } + } +} diff --git a/analysis/lang/fr/articles_fr.go b/analysis/lang/fr/articles_fr.go new file mode 100644 index 0000000..540ab91 --- /dev/null +++ b/analysis/lang/fr/articles_fr.go @@ -0,0 +1,40 @@ +package fr + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const ArticlesName = "articles_fr" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis + +var FrenchArticles = []byte(` +l +m +t +qu +n +s +j +d +c +jusqu +quoiqu +lorsqu +puisqu +`) + +func ArticlesTokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(FrenchArticles) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(ArticlesName, ArticlesTokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/fr/elision_fr.go b/analysis/lang/fr/elision_fr.go new file mode 100644 index 0000000..7bd3ac9 --- /dev/null +++ b/analysis/lang/fr/elision_fr.go @@ -0,0 +1,40 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fr + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/elision" + "github.com/blevesearch/bleve/v2/registry" +) + +const ElisionName = "elision_fr" + +func ElisionFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + articlesTokenMap, err := cache.TokenMapNamed(ArticlesName) + if err != nil { + return nil, fmt.Errorf("error building elision filter: %v", err) + } + return elision.NewElisionFilter(articlesTokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(ElisionName, ElisionFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/fr/elision_fr_test.go b/analysis/lang/fr/elision_fr_test.go new file mode 100644 index 0000000..19689ec --- /dev/null +++ b/analysis/lang/fr/elision_fr_test.go @@ -0,0 +1,55 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fr + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestFrenchElision(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("l'avion"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("avion"), + }, + }, + }, + } + + cache := registry.NewCache() + elisionFilter, err := cache.TokenFilterNamed(ElisionName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := elisionFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/fr/light_stemmer_fr.go b/analysis/lang/fr/light_stemmer_fr.go new file mode 100644 index 0000000..74f9200 --- /dev/null +++ b/analysis/lang/fr/light_stemmer_fr.go @@ -0,0 +1,309 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fr + +import ( + "bytes" + "unicode" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const LightStemmerName = "stemmer_fr_light" + +type FrenchLightStemmerFilter struct { +} + +func NewFrenchLightStemmerFilter() *FrenchLightStemmerFilter { + return &FrenchLightStemmerFilter{} +} + +func (s *FrenchLightStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + runes := bytes.Runes(token.Term) + runes = stem(runes) + token.Term = analysis.BuildTermFromRunes(runes) + } + return input +} + +func stem(input []rune) []rune { + + inputLen := len(input) + + if inputLen > 5 && input[inputLen-1] == 'x' { + if input[inputLen-3] == 'a' && input[inputLen-2] == 'u' && input[inputLen-4] != 'e' { + input[inputLen-2] = 'l' + } + input = input[0 : inputLen-1] + inputLen = len(input) + } + + if inputLen > 3 && input[inputLen-1] == 'x' { + input = input[0 : inputLen-1] + inputLen = len(input) + } + + if inputLen > 3 && input[inputLen-1] == 's' { + input = input[0 : inputLen-1] + inputLen = len(input) + } + + if inputLen > 9 && analysis.RunesEndsWith(input, "issement") { + input = input[0 : inputLen-6] + inputLen = len(input) + input[inputLen-1] = 'r' + return norm(input) + } + + if inputLen > 8 && analysis.RunesEndsWith(input, "issant") { + input = input[0 : inputLen-4] + inputLen = len(input) + input[inputLen-1] = 'r' + return norm(input) + } + + if inputLen > 6 && analysis.RunesEndsWith(input, "ement") { + input = input[0 : inputLen-4] + inputLen = len(input) + if inputLen > 3 && analysis.RunesEndsWith(input, "ive") { + input = input[0 : inputLen-1] + inputLen = len(input) + input[inputLen-1] = 'f' + } + return norm(input) + } + + if inputLen > 11 && analysis.RunesEndsWith(input, "ficatrice") { + input = input[0 : inputLen-5] + inputLen = len(input) + input[inputLen-2] = 'e' + input[inputLen-1] = 'r' + return norm(input) + } + + if inputLen > 10 && analysis.RunesEndsWith(input, "ficateur") { + input = input[0 : inputLen-4] + inputLen = len(input) + input[inputLen-2] = 'e' + input[inputLen-1] = 'r' + return norm(input) + } + + if inputLen > 9 && analysis.RunesEndsWith(input, "catrice") { + input = input[0 : inputLen-3] + inputLen = len(input) + input[inputLen-4] = 'q' + input[inputLen-3] = 'u' + input[inputLen-2] = 'e' + //s[len-1] = 'r' <-- unnecessary, already 'r'. + return norm(input) + } + + if inputLen > 8 && analysis.RunesEndsWith(input, "cateur") { + input = input[0 : inputLen-2] + inputLen = len(input) + input[inputLen-4] = 'q' + input[inputLen-3] = 'u' + input[inputLen-2] = 'e' + input[inputLen-1] = 'r' + return norm(input) + } + + if inputLen > 8 && analysis.RunesEndsWith(input, "atrice") { + input = input[0 : inputLen-4] + inputLen = len(input) + input[inputLen-2] = 'e' + input[inputLen-1] = 'r' + return norm(input) + } + + if inputLen > 7 && analysis.RunesEndsWith(input, "ateur") { + input = input[0 : inputLen-3] + inputLen = len(input) + input[inputLen-2] = 'e' + input[inputLen-1] = 'r' + return norm(input) + } + + if inputLen > 6 && analysis.RunesEndsWith(input, "trice") { + input = input[0 : inputLen-1] + inputLen = len(input) + input[inputLen-3] = 'e' + input[inputLen-2] = 'u' + input[inputLen-1] = 'r' + } + + if inputLen > 5 && analysis.RunesEndsWith(input, "ième") { + return norm(input[0 : inputLen-4]) + } + + if inputLen > 7 && analysis.RunesEndsWith(input, "teuse") { + input = input[0 : inputLen-2] + inputLen = len(input) + input[inputLen-1] = 'r' + return norm(input) + } + + if inputLen > 6 && analysis.RunesEndsWith(input, "teur") { + input = input[0 : inputLen-1] + inputLen = len(input) + input[inputLen-1] = 'r' + return norm(input) + } + + if inputLen > 5 && analysis.RunesEndsWith(input, "euse") { + return norm(input[0 : inputLen-2]) + } + + if inputLen > 8 && analysis.RunesEndsWith(input, "ère") { + input = input[0 : inputLen-1] + inputLen = len(input) + input[inputLen-2] = 'e' + return norm(input) + } + + if inputLen > 7 && analysis.RunesEndsWith(input, "ive") { + input = input[0 : inputLen-1] + inputLen = len(input) + input[inputLen-1] = 'f' + return norm(input) + } + + if inputLen > 4 && + (analysis.RunesEndsWith(input, "folle") || + analysis.RunesEndsWith(input, "molle")) { + input = input[0 : inputLen-2] + inputLen = len(input) + input[inputLen-1] = 'u' + return norm(input) + } + + if inputLen > 9 && analysis.RunesEndsWith(input, "nnelle") { + return norm(input[0 : inputLen-5]) + } + + if inputLen > 9 && analysis.RunesEndsWith(input, "nnel") { + return norm(input[0 : inputLen-3]) + } + + if inputLen > 4 && analysis.RunesEndsWith(input, "ète") { + input = input[0 : inputLen-1] + inputLen = len(input) + input[inputLen-2] = 'e' + } + + if inputLen > 8 && analysis.RunesEndsWith(input, "ique") { + input = input[0 : inputLen-4] + inputLen = len(input) + } + + if inputLen > 8 && analysis.RunesEndsWith(input, "esse") { + return norm(input[0 : inputLen-3]) + } + + if inputLen > 7 && analysis.RunesEndsWith(input, "inage") { + return norm(input[0 : inputLen-3]) + } + + if inputLen > 9 && analysis.RunesEndsWith(input, "isation") { + input = input[0 : inputLen-7] + inputLen = len(input) + if inputLen > 5 && analysis.RunesEndsWith(input, "ual") { + input[inputLen-2] = 'e' + } + return norm(input) + } + + if inputLen > 9 && analysis.RunesEndsWith(input, "isateur") { + return norm(input[0 : inputLen-7]) + } + + if inputLen > 8 && analysis.RunesEndsWith(input, "ation") { + return norm(input[0 : inputLen-5]) + } + + if inputLen > 8 && analysis.RunesEndsWith(input, "ition") { + return norm(input[0 : inputLen-5]) + } + + return norm(input) + +} + +func norm(input []rune) []rune { + + if len(input) > 4 { + for i := 0; i < len(input); i++ { + switch input[i] { + case 'à', 'á', 'â': + input[i] = 'a' + case 'ô': + input[i] = 'o' + case 'è', 'é', 'ê': + input[i] = 'e' + case 'ù', 'û': + input[i] = 'u' + case 'î': + input[i] = 'i' + case 'ç': + input[i] = 'c' + } + + ch := input[0] + for i := 1; i < len(input); i++ { + if input[i] == ch && unicode.IsLetter(ch) { + input = analysis.DeleteRune(input, i) + i -= 1 + } else { + ch = input[i] + } + } + } + } + + if len(input) > 4 && analysis.RunesEndsWith(input, "ie") { + input = input[0 : len(input)-2] + } + + if len(input) > 4 { + if input[len(input)-1] == 'r' { + input = input[0 : len(input)-1] + } + if input[len(input)-1] == 'e' { + input = input[0 : len(input)-1] + } + if input[len(input)-1] == 'e' { + input = input[0 : len(input)-1] + } + if input[len(input)-1] == input[len(input)-2] && unicode.IsLetter(input[len(input)-1]) { + input = input[0 : len(input)-1] + } + } + + return input +} + +func FrenchLightStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewFrenchLightStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(LightStemmerName, FrenchLightStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/fr/light_stemmer_fr_test.go b/analysis/lang/fr/light_stemmer_fr_test.go new file mode 100644 index 0000000..a098b97 --- /dev/null +++ b/analysis/lang/fr/light_stemmer_fr_test.go @@ -0,0 +1,1015 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fr + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestFrenchLightStemmer(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chevaux"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("cheval"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("cheval"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("cheval"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("hiboux"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("hibou"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("hibou"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("hibou"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chantés"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chant"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chanter"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chant"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chante"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chant"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chant"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chant"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("baronnes"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("baron"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("barons"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("baron"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("baron"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("baron"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("peaux"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("peau"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("peau"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("peau"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("anneaux"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("aneau"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("anneau"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("aneau"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("neveux"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("neveu"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("neveu"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("neveu"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("affreux"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("afreu"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("affreuse"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("afreu"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("investissement"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("investi"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("investir"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("investi"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("assourdissant"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("asourdi"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("assourdir"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("asourdi"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("pratiquement"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("pratiqu"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("pratique"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("pratiqu"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("administrativement"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("administratif"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("administratif"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("administratif"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("justificatrice"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("justifi"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("justificateur"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("justifi"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("justifier"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("justifi"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("educatrice"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("eduqu"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("eduquer"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("eduqu"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("communicateur"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("comuniqu"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("communiquer"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("comuniqu"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("accompagnatrice"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("acompagn"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("accompagnateur"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("acompagn"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("administrateur"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("administr"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("administrer"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("administr"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("productrice"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("product"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("producteur"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("product"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("acheteuse"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("achet"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("acheteur"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("achet"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("planteur"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("plant"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("plante"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("plant"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("poreuse"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("poreu"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("poreux"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("poreu"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("plieuse"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("plieu"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("bijoutière"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("bijouti"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("bijoutier"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("bijouti"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("caissière"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("caisi"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("caissier"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("caisi"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abrasive"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abrasif"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abrasif"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abrasif"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("folle"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("fou"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("fou"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("fou"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("personnelle"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("person"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("personne"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("person"), + }, + }, + }, + // algo bug: too short length + // { + // input: analysis.TokenStream{ + // &analysis.Token{ + // Term: []byte("personnel"), + // }, + // }, + // output: analysis.TokenStream{ + // &analysis.Token{ + // Term: []byte("person"), + // }, + // }, + // }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("complète"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("complet"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("complet"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("complet"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("aromatique"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("aromat"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("faiblesse"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("faibl"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("faible"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("faibl"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("patinage"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("patin"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("patin"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("patin"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("sonorisation"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("sono"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ritualisation"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("rituel"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("rituel"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("rituel"), + }, + }, + }, + // algo bug: masked by rules above + // { + // input: analysis.TokenStream{ + // &analysis.Token{ + // Term: []byte("colonisateur"), + // }, + // }, + // output: analysis.TokenStream{ + // &analysis.Token{ + // Term: []byte("colon"), + // }, + // }, + // }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("nomination"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("nomin"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("disposition"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("dispos"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("dispose"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("dispos"), + }, + }, + }, + // SOLR-3463 : abusive compression of repeated characters in numbers + // Trailing repeated char elision : + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("1234555"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("1234555"), + }, + }, + }, + // Repeated char within numbers with more than 4 characters : + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("12333345"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("12333345"), + }, + }, + }, + // Short numbers weren't affected already: + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("1234"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("1234"), + }, + }, + }, + // Ensure behaviour is preserved for words! + // Trailing repeated char elision : + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abcdeff"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abcdef"), + }, + }, + }, + // Repeated char within words with more than 4 characters : + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abcccddeef"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abcdef"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("créées"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("cre"), + }, + }, + }, + // Combined letter and digit repetition + // 10:00pm + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("22hh00"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("22h00"), + }, + }, + }, + // bug #214 + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("propriétaire"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("proprietair"), + }, + }, + }, + } + + cache := registry.NewCache() + filter, err := cache.TokenFilterNamed(LightStemmerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := filter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/fr/minimal_stemmer_fr.go b/analysis/lang/fr/minimal_stemmer_fr.go new file mode 100644 index 0000000..ac6bb47 --- /dev/null +++ b/analysis/lang/fr/minimal_stemmer_fr.go @@ -0,0 +1,82 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fr + +import ( + "bytes" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const MinimalStemmerName = "stemmer_fr_min" + +type FrenchMinimalStemmerFilter struct { +} + +func NewFrenchMinimalStemmerFilter() *FrenchMinimalStemmerFilter { + return &FrenchMinimalStemmerFilter{} +} + +func (s *FrenchMinimalStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + runes := bytes.Runes(token.Term) + runes = minstem(runes) + token.Term = analysis.BuildTermFromRunes(runes) + } + return input +} + +func minstem(input []rune) []rune { + + if len(input) < 6 { + return input + } + + if input[len(input)-1] == 'x' { + if input[len(input)-3] == 'a' && input[len(input)-2] == 'u' { + input[len(input)-2] = 'l' + } + return input[0 : len(input)-1] + } + + if input[len(input)-1] == 's' { + input = input[0 : len(input)-1] + } + if input[len(input)-1] == 'r' { + input = input[0 : len(input)-1] + } + if input[len(input)-1] == 'e' { + input = input[0 : len(input)-1] + } + if input[len(input)-1] == 'é' { + input = input[0 : len(input)-1] + } + if input[len(input)-1] == input[len(input)-2] { + input = input[0 : len(input)-1] + } + return input +} + +func FrenchMinimalStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewFrenchMinimalStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(MinimalStemmerName, FrenchMinimalStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/fr/minimal_stemmer_fr_test.go b/analysis/lang/fr/minimal_stemmer_fr_test.go new file mode 100644 index 0000000..fa17528 --- /dev/null +++ b/analysis/lang/fr/minimal_stemmer_fr_test.go @@ -0,0 +1,139 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fr + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestFrenchMinimalStemmer(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chevaux"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("cheval"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("hiboux"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("hibou"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chantés"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chant"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chanter"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chant"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chante"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("chant"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("baronnes"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("baron"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("barons"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("baron"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("baron"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("baron"), + }, + }, + }, + } + + cache := registry.NewCache() + filter, err := cache.TokenFilterNamed(MinimalStemmerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := filter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/fr/stemmer_fr_snowball.go b/analysis/lang/fr/stemmer_fr_snowball.go new file mode 100644 index 0000000..5b542a3 --- /dev/null +++ b/analysis/lang/fr/stemmer_fr_snowball.go @@ -0,0 +1,52 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fr + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/snowballstem" + "github.com/blevesearch/snowballstem/french" +) + +const SnowballStemmerName = "stemmer_fr_snowball" + +type FrenchStemmerFilter struct { +} + +func NewFrenchStemmerFilter() *FrenchStemmerFilter { + return &FrenchStemmerFilter{} +} + +func (s *FrenchStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + env := snowballstem.NewEnv(string(token.Term)) + french.Stem(env) + token.Term = []byte(env.Current()) + } + return input +} + +func FrenchStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewFrenchStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(SnowballStemmerName, FrenchStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/fr/stemmer_fr_snowball_test.go b/analysis/lang/fr/stemmer_fr_snowball_test.go new file mode 100644 index 0000000..69e9f26 --- /dev/null +++ b/analysis/lang/fr/stemmer_fr_snowball_test.go @@ -0,0 +1,79 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fr + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestSnowballFrenchStemmer(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("antagoniste"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("antagon"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("barbouillait"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("barbouill"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("calculateur"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("calcul"), + }, + }, + }, + } + + cache := registry.NewCache() + filter, err := cache.TokenFilterNamed(SnowballStemmerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := filter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/fr/stop_filter_fr.go b/analysis/lang/fr/stop_filter_fr.go new file mode 100644 index 0000000..e2d4612 --- /dev/null +++ b/analysis/lang/fr/stop_filter_fr.go @@ -0,0 +1,36 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fr + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/fr/stop_words_fr.go b/analysis/lang/fr/stop_words_fr.go new file mode 100644 index 0000000..6a767d6 --- /dev/null +++ b/analysis/lang/fr/stop_words_fr.go @@ -0,0 +1,213 @@ +package fr + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_fr" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ +// ` was changed to ' to allow for literal string + +var FrenchStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/french/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A French stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + +au | a + le +aux | a + les +avec | with +ce | this +ces | these +dans | with +de | of +des | de + les +du | de + le +elle | she +en | 'of them' etc +et | and +eux | them +il | he +je | I +la | the +le | the +leur | their +lui | him +ma | my (fem) +mais | but +me | me +même | same; as in moi-même (myself) etc +mes | me (pl) +moi | me +mon | my (masc) +ne | not +nos | our (pl) +notre | our +nous | we +on | one +ou | where +par | by +pas | not +pour | for +qu | que before vowel +que | that +qui | who +sa | his, her (fem) +se | oneself +ses | his (pl) +son | his, her (masc) +sur | on +ta | thy (fem) +te | thee +tes | thy (pl) +toi | thee +ton | thy (masc) +tu | thou +un | a +une | a +vos | your (pl) +votre | your +vous | you + + | single letter forms + +c | c' +d | d' +j | j' +l | l' +à | to, at +m | m' +n | n' +s | s' +t | t' +y | there + + | forms of être (not including the infinitive): +été +étée +étées +étés +étant +suis +es +est +sommes +êtes +sont +serai +seras +sera +serons +serez +seront +serais +serait +serions +seriez +seraient +étais +était +étions +étiez +étaient +fus +fut +fûmes +fûtes +furent +sois +soit +soyons +soyez +soient +fusse +fusses +fût +fussions +fussiez +fussent + + | forms of avoir (not including the infinitive): +ayant +eu +eue +eues +eus +ai +as +avons +avez +ont +aurai +auras +aura +aurons +aurez +auront +aurais +aurait +aurions +auriez +auraient +avais +avait +avions +aviez +avaient +eut +eûmes +eûtes +eurent +aie +aies +ait +ayons +ayez +aient +eusse +eusses +eût +eussions +eussiez +eussent + + | Later additions (from Jean-Christophe Deschamps) +ceci | this +cela | that +celà | that +cet | this +cette | this +ici | here +ils | they +les | the (pl) +leurs | their (pl) +quel | which +quels | which +quelle | which +quelles | which +sans | without +soi | oneself + +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(FrenchStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ga/articles_ga.go b/analysis/lang/ga/articles_ga.go new file mode 100644 index 0000000..2ccaa36 --- /dev/null +++ b/analysis/lang/ga/articles_ga.go @@ -0,0 +1,30 @@ +package ga + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const ArticlesName = "articles_ga" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis + +var IrishArticles = []byte(` +d +m +b +`) + +func ArticlesTokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(IrishArticles) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(ArticlesName, ArticlesTokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ga/elision_ga.go b/analysis/lang/ga/elision_ga.go new file mode 100644 index 0000000..988ec50 --- /dev/null +++ b/analysis/lang/ga/elision_ga.go @@ -0,0 +1,40 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ga + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/elision" + "github.com/blevesearch/bleve/v2/registry" +) + +const ElisionName = "elision_ga" + +func ElisionFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + articlesTokenMap, err := cache.TokenMapNamed(ArticlesName) + if err != nil { + return nil, fmt.Errorf("error building elision filter: %v", err) + } + return elision.NewElisionFilter(articlesTokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(ElisionName, ElisionFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ga/elision_ga_test.go b/analysis/lang/ga/elision_ga_test.go new file mode 100644 index 0000000..809a7c3 --- /dev/null +++ b/analysis/lang/ga/elision_ga_test.go @@ -0,0 +1,55 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ga + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestFrenchElision(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("b'fhearr"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("fhearr"), + }, + }, + }, + } + + cache := registry.NewCache() + elisionFilter, err := cache.TokenFilterNamed(ElisionName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := elisionFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/ga/stop_filter_ga.go b/analysis/lang/ga/stop_filter_ga.go new file mode 100644 index 0000000..9c34924 --- /dev/null +++ b/analysis/lang/ga/stop_filter_ga.go @@ -0,0 +1,36 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ga + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ga/stop_words_ga.go b/analysis/lang/ga/stop_words_ga.go new file mode 100644 index 0000000..e49e671 --- /dev/null +++ b/analysis/lang/ga/stop_words_ga.go @@ -0,0 +1,137 @@ +package ga + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_ga" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ +// ` was changed to ' to allow for literal string + +var IrishStopWords = []byte(` +a +ach +ag +agus +an +aon +ar +arna +as +b' +ba +beirt +bhúr +caoga +ceathair +ceathrar +chomh +chtó +chuig +chun +cois +céad +cúig +cúigear +d' +daichead +dar +de +deich +deichniúr +den +dhá +do +don +dtí +dá +dár +dó +faoi +faoin +faoina +faoinár +fara +fiche +gach +gan +go +gur +haon +hocht +i +iad +idir +in +ina +ins +inár +is +le +leis +lena +lenár +m' +mar +mo +mé +na +nach +naoi +naonúr +ná +ní +níor +nó +nócha +ocht +ochtar +os +roimh +sa +seacht +seachtar +seachtó +seasca +seisear +siad +sibh +sinn +sna +sé +sí +tar +thar +thú +triúr +trí +trína +trínár +tríocha +tú +um +ár +é +éis +í +ó +ón +óna +ónár +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(IrishStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/gl/stop_filter_gl.go b/analysis/lang/gl/stop_filter_gl.go new file mode 100644 index 0000000..5c93816 --- /dev/null +++ b/analysis/lang/gl/stop_filter_gl.go @@ -0,0 +1,36 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gl + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/gl/stop_words_gl.go b/analysis/lang/gl/stop_words_gl.go new file mode 100644 index 0000000..766c5ec --- /dev/null +++ b/analysis/lang/gl/stop_words_gl.go @@ -0,0 +1,188 @@ +package gl + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_gl" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ +// ` was changed to ' to allow for literal string + +var GalicianStopWords = []byte(`# galican stopwords +a +aínda +alí +aquel +aquela +aquelas +aqueles +aquilo +aquí +ao +aos +as +así +á +ben +cando +che +co +coa +comigo +con +connosco +contigo +convosco +coas +cos +cun +cuns +cunha +cunhas +da +dalgunha +dalgunhas +dalgún +dalgúns +das +de +del +dela +delas +deles +desde +deste +do +dos +dun +duns +dunha +dunhas +e +el +ela +elas +eles +en +era +eran +esa +esas +ese +eses +esta +estar +estaba +está +están +este +estes +estiven +estou +eu +é +facer +foi +foron +fun +había +hai +iso +isto +la +las +lle +lles +lo +los +mais +me +meu +meus +min +miña +miñas +moi +na +nas +neste +nin +no +non +nos +nosa +nosas +noso +nosos +nós +nun +nunha +nuns +nunhas +o +os +ou +ó +ós +para +pero +pode +pois +pola +polas +polo +polos +por +que +se +senón +ser +seu +seus +sexa +sido +sobre +súa +súas +tamén +tan +te +ten +teñen +teño +ter +teu +teus +ti +tido +tiña +tiven +túa +túas +un +unha +unhas +uns +vos +vosa +vosas +voso +vosos +vós +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(GalicianStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/hi/analyzer_hi.go b/analysis/lang/hi/analyzer_hi.go new file mode 100644 index 0000000..4d22cf0 --- /dev/null +++ b/analysis/lang/hi/analyzer_hi.go @@ -0,0 +1,71 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hi + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/bleve/v2/analysis/lang/in" + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +) + +const AnalyzerName = "hi" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + tokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + indicNormalizeFilter, err := cache.TokenFilterNamed(in.NormalizeName) + if err != nil { + return nil, err + } + hindiNormalizeFilter, err := cache.TokenFilterNamed(NormalizeName) + if err != nil { + return nil, err + } + stopHiFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + stemmerHiFilter, err := cache.TokenFilterNamed(StemmerName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: tokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + indicNormalizeFilter, + hindiNormalizeFilter, + stopHiFilter, + stemmerHiFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/hi/analyzer_hi_test.go b/analysis/lang/hi/analyzer_hi_test.go new file mode 100644 index 0000000..a86aeef --- /dev/null +++ b/analysis/lang/hi/analyzer_hi_test.go @@ -0,0 +1,66 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hi + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestHindiAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // two ways to write 'hindi' itself + { + input: []byte("हिन्दी"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("हिंद"), + Position: 1, + Start: 0, + End: 18, + }, + }, + }, + { + input: []byte("हिंदी"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("हिंद"), + Position: 1, + Start: 0, + End: 15, + }, + }, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %v, got %v", test.output, actual) + } + } +} diff --git a/analysis/lang/hi/hindi_normalize.go b/analysis/lang/hi/hindi_normalize.go new file mode 100644 index 0000000..3ba5ead --- /dev/null +++ b/analysis/lang/hi/hindi_normalize.go @@ -0,0 +1,141 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hi + +import ( + "bytes" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const NormalizeName = "normalize_hi" + +type HindiNormalizeFilter struct { +} + +func NewHindiNormalizeFilter() *HindiNormalizeFilter { + return &HindiNormalizeFilter{} +} + +func (s *HindiNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + term := normalize(token.Term) + token.Term = term + } + return input +} + +func normalize(input []byte) []byte { + runes := bytes.Runes(input) + for i := 0; i < len(runes); i++ { + switch runes[i] { + // dead n -> bindu + case '\u0928': + if i+1 < len(runes) && runes[i+1] == '\u094D' { + runes[i] = '\u0902' + runes = analysis.DeleteRune(runes, i+1) + } + // candrabindu -> bindu + case '\u0901': + runes[i] = '\u0902' + // nukta deletions + case '\u093C': + runes = analysis.DeleteRune(runes, i) + i-- + case '\u0929': + runes[i] = '\u0928' + case '\u0931': + runes[i] = '\u0930' + case '\u0934': + runes[i] = '\u0933' + case '\u0958': + runes[i] = '\u0915' + case '\u0959': + runes[i] = '\u0916' + case '\u095A': + runes[i] = '\u0917' + case '\u095B': + runes[i] = '\u091C' + case '\u095C': + runes[i] = '\u0921' + case '\u095D': + runes[i] = '\u0922' + case '\u095E': + runes[i] = '\u092B' + case '\u095F': + runes[i] = '\u092F' + // zwj/zwnj -> delete + case '\u200D', '\u200C': + runes = analysis.DeleteRune(runes, i) + i-- + // virama -> delete + case '\u094D': + runes = analysis.DeleteRune(runes, i) + i-- + // chandra/short -> replace + case '\u0945', '\u0946': + runes[i] = '\u0947' + case '\u0949', '\u094A': + runes[i] = '\u094B' + case '\u090D', '\u090E': + runes[i] = '\u090F' + case '\u0911', '\u0912': + runes[i] = '\u0913' + case '\u0972': + runes[i] = '\u0905' + // long -> short ind. vowels + case '\u0906': + runes[i] = '\u0905' + case '\u0908': + runes[i] = '\u0907' + case '\u090A': + runes[i] = '\u0909' + case '\u0960': + runes[i] = '\u090B' + case '\u0961': + runes[i] = '\u090C' + case '\u0910': + runes[i] = '\u090F' + case '\u0914': + runes[i] = '\u0913' + // long -> short dep. vowels + case '\u0940': + runes[i] = '\u093F' + case '\u0942': + runes[i] = '\u0941' + case '\u0944': + runes[i] = '\u0943' + case '\u0963': + runes[i] = '\u0962' + case '\u0948': + runes[i] = '\u0947' + case '\u094C': + runes[i] = '\u094B' + } + } + return analysis.BuildTermFromRunes(runes) +} + +func NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewHindiNormalizeFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/hi/hindi_normalize_test.go b/analysis/lang/hi/hindi_normalize_test.go new file mode 100644 index 0000000..242008d --- /dev/null +++ b/analysis/lang/hi/hindi_normalize_test.go @@ -0,0 +1,251 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hi + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestHindiNormalizeFilter(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + // basics + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अँगरेज़ी"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अंगरेजि"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अँगरेजी"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अंगरेजि"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अँग्रेज़ी"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अंगरेजि"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अँग्रेजी"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अंगरेजि"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अंगरेज़ी"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अंगरेजि"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अंगरेजी"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अंगरेजि"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अंग्रेज़ी"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अंगरेजि"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अंग्रेजी"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अंगरेजि"), + }, + }, + }, + // test decompositions + // removing nukta dot + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("क़िताब"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("किताब"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("फ़र्ज़"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("फरज"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("क़र्ज़"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("करज"), + }, + }, + }, + // some other composed nukta forms + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ऱऴख़ग़ड़ढ़य़"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("रळखगडढय"), + }, + }, + }, + // removal of format (ZWJ/ZWNJ) + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("शार्‍मा"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("शारमा"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("शार्‌मा"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("शारमा"), + }, + }, + }, + // removal of chandra + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ॅॆॉॊऍऎऑऒ\u0972"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ेेोोएएओओअ"), + }, + }, + }, + // vowel shortening + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("आईऊॠॡऐऔीूॄॣैौ"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अइउऋऌएओिुृॢेो"), + }, + }, + }, + // empty + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + } + + hindiNormalizeFilter := NewHindiNormalizeFilter() + for _, test := range tests { + actual := hindiNormalizeFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %#v, got %#v", test.output, actual) + t.Errorf("expected % x, got % x", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/hi/hindi_stemmer_filter.go b/analysis/lang/hi/hindi_stemmer_filter.go new file mode 100644 index 0000000..f54cc57 --- /dev/null +++ b/analysis/lang/hi/hindi_stemmer_filter.go @@ -0,0 +1,152 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hi + +import ( + "bytes" + "unicode/utf8" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StemmerName = "stemmer_hi" + +type HindiStemmerFilter struct { +} + +func NewHindiStemmerFilter() *HindiStemmerFilter { + return &HindiStemmerFilter{} +} + +func (s *HindiStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + // if not protected keyword, stem it + if !token.KeyWord { + stemmed := stem(token.Term) + token.Term = stemmed + } + } + return input +} + +func stem(input []byte) []byte { + inputLen := utf8.RuneCount(input) + + // 5 + if inputLen > 6 && + (bytes.HasSuffix(input, []byte("ाएंगी")) || + bytes.HasSuffix(input, []byte("ाएंगे")) || + bytes.HasSuffix(input, []byte("ाऊंगी")) || + bytes.HasSuffix(input, []byte("ाऊंगा")) || + bytes.HasSuffix(input, []byte("ाइयाँ")) || + bytes.HasSuffix(input, []byte("ाइयों")) || + bytes.HasSuffix(input, []byte("ाइयां"))) { + return analysis.TruncateRunes(input, 5) + } + + // 4 + if inputLen > 5 && + (bytes.HasSuffix(input, []byte("ाएगी")) || + bytes.HasSuffix(input, []byte("ाएगा")) || + bytes.HasSuffix(input, []byte("ाओगी")) || + bytes.HasSuffix(input, []byte("ाओगे")) || + bytes.HasSuffix(input, []byte("एंगी")) || + bytes.HasSuffix(input, []byte("ेंगी")) || + bytes.HasSuffix(input, []byte("एंगे")) || + bytes.HasSuffix(input, []byte("ेंगे")) || + bytes.HasSuffix(input, []byte("ूंगी")) || + bytes.HasSuffix(input, []byte("ूंगा")) || + bytes.HasSuffix(input, []byte("ातीं")) || + bytes.HasSuffix(input, []byte("नाओं")) || + bytes.HasSuffix(input, []byte("नाएं")) || + bytes.HasSuffix(input, []byte("ताओं")) || + bytes.HasSuffix(input, []byte("ताएं")) || + bytes.HasSuffix(input, []byte("ियाँ")) || + bytes.HasSuffix(input, []byte("ियों")) || + bytes.HasSuffix(input, []byte("ियां"))) { + return analysis.TruncateRunes(input, 4) + } + + // 3 + if inputLen > 4 && + (bytes.HasSuffix(input, []byte("ाकर")) || + bytes.HasSuffix(input, []byte("ाइए")) || + bytes.HasSuffix(input, []byte("ाईं")) || + bytes.HasSuffix(input, []byte("ाया")) || + bytes.HasSuffix(input, []byte("ेगी")) || + bytes.HasSuffix(input, []byte("ेगा")) || + bytes.HasSuffix(input, []byte("ोगी")) || + bytes.HasSuffix(input, []byte("ोगे")) || + bytes.HasSuffix(input, []byte("ाने")) || + bytes.HasSuffix(input, []byte("ाना")) || + bytes.HasSuffix(input, []byte("ाते")) || + bytes.HasSuffix(input, []byte("ाती")) || + bytes.HasSuffix(input, []byte("ाता")) || + bytes.HasSuffix(input, []byte("तीं")) || + bytes.HasSuffix(input, []byte("ाओं")) || + bytes.HasSuffix(input, []byte("ाएं")) || + bytes.HasSuffix(input, []byte("ुओं")) || + bytes.HasSuffix(input, []byte("ुएं")) || + bytes.HasSuffix(input, []byte("ुआं"))) { + return analysis.TruncateRunes(input, 3) + } + + // 2 + if inputLen > 3 && + (bytes.HasSuffix(input, []byte("कर")) || + bytes.HasSuffix(input, []byte("ाओ")) || + bytes.HasSuffix(input, []byte("िए")) || + bytes.HasSuffix(input, []byte("ाई")) || + bytes.HasSuffix(input, []byte("ाए")) || + bytes.HasSuffix(input, []byte("ने")) || + bytes.HasSuffix(input, []byte("नी")) || + bytes.HasSuffix(input, []byte("ना")) || + bytes.HasSuffix(input, []byte("ते")) || + bytes.HasSuffix(input, []byte("ीं")) || + bytes.HasSuffix(input, []byte("ती")) || + bytes.HasSuffix(input, []byte("ता")) || + bytes.HasSuffix(input, []byte("ाँ")) || + bytes.HasSuffix(input, []byte("ां")) || + bytes.HasSuffix(input, []byte("ों")) || + bytes.HasSuffix(input, []byte("ें"))) { + return analysis.TruncateRunes(input, 2) + } + + // 1 + if inputLen > 2 && + (bytes.HasSuffix(input, []byte("ो")) || + bytes.HasSuffix(input, []byte("े")) || + bytes.HasSuffix(input, []byte("ू")) || + bytes.HasSuffix(input, []byte("ु")) || + bytes.HasSuffix(input, []byte("ी")) || + bytes.HasSuffix(input, []byte("ि")) || + bytes.HasSuffix(input, []byte("ा"))) { + return analysis.TruncateRunes(input, 1) + } + + return input +} + +func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewHindiStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/hi/hindi_stemmer_filter_test.go b/analysis/lang/hi/hindi_stemmer_filter_test.go new file mode 100644 index 0000000..bf5d92a --- /dev/null +++ b/analysis/lang/hi/hindi_stemmer_filter_test.go @@ -0,0 +1,308 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hi + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestHindiStemmerFilter(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + // masc noun inflections + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("लडका"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("लडक"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("लडके"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("लडक"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("लडकों"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("लडक"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("गुरु"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("गुर"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("गुरुओं"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("गुर"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("दोस्त"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("दोस्त"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("दोस्तों"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("दोस्त"), + }, + }, + }, + // feminine noun inflections + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("लडकी"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("लडक"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("लडकियों"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("लडक"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("किताब"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("किताब"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("किताबें"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("किताब"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("किताबों"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("किताब"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("आध्यापीका"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("आध्यापीक"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("आध्यापीकाएं"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("आध्यापीक"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("आध्यापीकाओं"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("आध्यापीक"), + }, + }, + }, + // some verb forms + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("खाना"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("खा"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("खाता"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("खा"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("खाती"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("खा"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("खा"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("खा"), + }, + }, + }, + // exceptions + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("कठिनाइयां"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("कठिन"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("कठिन"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("कठिन"), + }, + }, + }, + // empty + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + } + + hindiStemmerFilter := NewHindiStemmerFilter() + for _, test := range tests { + actual := hindiStemmerFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %#v, got %#v", test.output, actual) + t.Errorf("expected % x, got % x", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/hi/stop_filter_hi.go b/analysis/lang/hi/stop_filter_hi.go new file mode 100644 index 0000000..8122b4f --- /dev/null +++ b/analysis/lang/hi/stop_filter_hi.go @@ -0,0 +1,36 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hi + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/hi/stop_words_hi.go b/analysis/lang/hi/stop_words_hi.go new file mode 100644 index 0000000..844c774 --- /dev/null +++ b/analysis/lang/hi/stop_words_hi.go @@ -0,0 +1,262 @@ +package hi + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_hi" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ +// ` was changed to ' to allow for literal string + +var HindiStopWords = []byte(`# Also see http://www.opensource.org/licenses/bsd-license.html +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# This file was created by Jacques Savoy and is distributed under the BSD license. +# Note: by default this file also contains forms normalized by HindiNormalizer +# for spelling variation (see section below), such that it can be used whether or +# not you enable that feature. When adding additional entries to this list, +# please add the normalized form as well. +अंदर +अत +अपना +अपनी +अपने +अभी +आदि +आप +इत्यादि +इन +इनका +इन्हीं +इन्हें +इन्हों +इस +इसका +इसकी +इसके +इसमें +इसी +इसे +उन +उनका +उनकी +उनके +उनको +उन्हीं +उन्हें +उन्हों +उस +उसके +उसी +उसे +एक +एवं +एस +ऐसे +और +कई +कर +करता +करते +करना +करने +करें +कहते +कहा +का +काफ़ी +कि +कितना +किन्हें +किन्हों +किया +किर +किस +किसी +किसे +की +कुछ +कुल +के +को +कोई +कौन +कौनसा +गया +घर +जब +जहाँ +जा +जितना +जिन +जिन्हें +जिन्हों +जिस +जिसे +जीधर +जैसा +जैसे +जो +तक +तब +तरह +तिन +तिन्हें +तिन्हों +तिस +तिसे +तो +था +थी +थे +दबारा +दिया +दुसरा +दूसरे +दो +द्वारा +न +नहीं +ना +निहायत +नीचे +ने +पर +पर +पहले +पूरा +पे +फिर +बनी +बही +बहुत +बाद +बाला +बिलकुल +भी +भीतर +मगर +मानो +मे +में +यदि +यह +यहाँ +यही +या +यिह +ये +रखें +रहा +रहे +ऱ्वासा +लिए +लिये +लेकिन +व +वर्ग +वह +वह +वहाँ +वहीं +वाले +वुह +वे +वग़ैरह +संग +सकता +सकते +सबसे +सभी +साथ +साबुत +साभ +सारा +से +सो +ही +हुआ +हुई +हुए +है +हैं +हो +होता +होती +होते +होना +होने +# additional normalized forms of the above +अपनि +जेसे +होति +सभि +तिंहों +इंहों +दवारा +इसि +किंहें +थि +उंहों +ओर +जिंहें +वहिं +अभि +बनि +हि +उंहिं +उंहें +हें +वगेरह +एसे +रवासा +कोन +निचे +काफि +उसि +पुरा +भितर +हे +बहि +वहां +कोइ +यहां +जिंहों +तिंहें +किसि +कइ +यहि +इंहिं +जिधर +इंहें +अदि +इतयादि +हुइ +कोनसा +इसकि +दुसरे +जहां +अप +किंहों +उनकि +भि +वरग +हुअ +जेसा +नहिं +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(HindiStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/hr/analyzer_hr.go b/analysis/lang/hr/analyzer_hr.go new file mode 100644 index 0000000..7deaeda --- /dev/null +++ b/analysis/lang/hr/analyzer_hr.go @@ -0,0 +1,67 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hr + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +) + +// Originated from: http://nlp.ffzg.hr/resources/tools/stemmer-for-croatian/ + +const AnalyzerName = "hr" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + unicodeTokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + suffixFilter, err := cache.TokenFilterNamed(SuffixTransformationFilterName) + if err != nil { + return nil, err + } + stemmerFilter, err := cache.TokenFilterNamed(StemmerName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: unicodeTokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + stopFilter, + suffixFilter, + stemmerFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/hr/analyzer_hr_test.go b/analysis/lang/hr/analyzer_hr_test.go new file mode 100644 index 0000000..e1ab35a --- /dev/null +++ b/analysis/lang/hr/analyzer_hr_test.go @@ -0,0 +1,97 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hr + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestCroatianAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // stemming + { + input: []byte("Hrvatska"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("hrvatsk"), + }, + }, + }, + { + input: []byte("Hrvatski"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("hrvatsk"), + }, + }, + }, + // uppercase letters + { + input: []byte("KOMARAC"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("komarc"), + }, + }, + }, + // vowelR + { + input: []byte("crvi"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("crv"), + }, + }, + }, + // stop word + { + input: []byte("biti"), + output: analysis.TokenStream{}, + }, + // suffix transformation + { + input: []byte("zaključcima"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("zaključk"), + }, + }, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if len(actual) != len(test.output) { + t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) + } + for i, tok := range actual { + if !reflect.DeepEqual(tok.Term, test.output[i].Term) { + t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) + } + } + } +} diff --git a/analysis/lang/hr/stemmer_hr.go b/analysis/lang/hr/stemmer_hr.go new file mode 100644 index 0000000..e6e41f5 --- /dev/null +++ b/analysis/lang/hr/stemmer_hr.go @@ -0,0 +1,156 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hr + +import ( + "regexp" + "strings" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StemmerName = "stemmer_hr" + +// These regular expressions rules originated from: +// http://nlp.ffzg.hr/resources/tools/stemmer-for-croatian/ + +var stemmingRules = []*regexp.Regexp{ + regexp.MustCompile(`^(.+(s|š)k)(ijima|ijega|ijemu|ijem|ijim|ijih|ijoj|ijeg|iji|ije|ija|oga|ome|omu|ima|og|om|im|ih|oj|i|e|o|a|u)$`), + regexp.MustCompile(`^(.+(s|š)tv)(ima|om|o|a|u)$`), + regexp.MustCompile(`^(.+(t|m|p|r|g)anij)(ama|ima|om|a|u|e|i|)$`), + regexp.MustCompile(`^(.+an)(inom|ina|inu|ine|ima|in|om|u|i|a|e|)$`), + regexp.MustCompile(`^(.+in)(ima|ama|om|a|e|i|u|o|)$`), + regexp.MustCompile(`^(.+on)(ovima|ova|ove|ovi|ima|om|a|e|i|u|)$`), + regexp.MustCompile(`^(.+n)(ijima|ijega|ijemu|ijeg|ijem|ijim|ijih|ijoj|iji|ije|ija|iju|ima|ome|omu|oga|oj|om|ih|im|og|o|e|a|u|i|)$`), + regexp.MustCompile(`^(.+(a|e|u)ć)(oga|ome|omu|ega|emu|ima|oj|ih|om|eg|em|og|uh|im|e|a)$`), + regexp.MustCompile(`^(.+ugov)(ima|i|e|a)$`), + regexp.MustCompile(`^(.+ug)(ama|om|a|e|i|u|o)$`), + regexp.MustCompile(`^(.+log)(ama|om|a|u|e|)$`), + regexp.MustCompile(`^(.+[^eo]g)(ovima|ama|ovi|ove|ova|om|a|e|i|u|o|)$`), + regexp.MustCompile(`^(.+(rrar|ott|ss|ll)i)(jem|ja|ju|o|)$`), + regexp.MustCompile(`^(.+uj)(ući|emo|ete|mo|em|eš|e|u|)$`), + regexp.MustCompile(`^(.+(c|č|ć|đ|l|r)aj)(evima|evi|eva|eve|ama|ima|em|a|e|i|u|)$`), + regexp.MustCompile(`^(.+(b|c|d|l|n|m|ž|g|f|p|r|s|t|z)ij)(ima|ama|om|a|e|i|u|o|)$`), + regexp.MustCompile(`^(.+[^z]nal)(ima|ama|om|a|e|i|u|o|)$`), + regexp.MustCompile(`^(.+ijal)(ima|ama|om|a|e|i|u|o|)$`), + regexp.MustCompile(`^(.+ozil)(ima|om|a|e|u|i|)$`), + regexp.MustCompile(`^(.+olov)(ima|i|a|e)$`), + regexp.MustCompile(`^(.+ol)(ima|om|a|u|e|i|)$`), + regexp.MustCompile(`^(.+lem)(ama|ima|om|a|e|i|u|o|)$`), + regexp.MustCompile(`^(.+ram)(ama|om|a|e|i|u|o)$`), + regexp.MustCompile(`^(.+(a|d|e|o)r)(ama|ima|om|u|a|e|i|)$`), + regexp.MustCompile(`^(.+(e|i)s)(ima|om|e|a|u)$`), + regexp.MustCompile(`^(.+(t|n|j|k|j|t|b|g|v)aš)(ama|ima|om|em|a|u|i|e|)$`), + regexp.MustCompile(`^(.+(e|i)š)(ima|ama|om|em|i|e|a|u|)$`), + regexp.MustCompile(`^(.+ikat)(ima|om|a|e|i|u|o|)$`), + regexp.MustCompile(`^(.+lat)(ima|om|a|e|i|u|o|)$`), + regexp.MustCompile(`^(.+et)(ama|ima|om|a|e|i|u|o|)$`), + regexp.MustCompile(`^(.+(e|i|k|o)st)(ima|ama|om|a|e|i|u|o|)$`), + regexp.MustCompile(`^(.+išt)(ima|em|a|e|u)$`), + regexp.MustCompile(`^(.+ova)(smo|ste|hu|ti|še|li|la|le|lo|t|h|o)$`), + regexp.MustCompile(`^(.+(a|e|i)v)(ijemu|ijima|ijega|ijeg|ijem|ijim|ijih|ijoj|oga|ome|omu|ima|ama|iji|ije|ija|iju|im|ih|oj|om|og|i|a|u|e|o|)$`), + regexp.MustCompile(`^(.+[^dkml]ov)(ijemu|ijima|ijega|ijeg|ijem|ijim|ijih|ijoj|oga|ome|omu|ima|iji|ije|ija|iju|im|ih|oj|om|og|i|a|u|e|o|)$`), + regexp.MustCompile(`^(.+(m|l)ov)(ima|om|a|u|e|i|)$`), + regexp.MustCompile(`^(.+el)(ijemu|ijima|ijega|ijeg|ijem|ijim|ijih|ijoj|oga|ome|omu|ima|iji|ije|ija|iju|im|ih|oj|om|og|i|a|u|e|o|)$`), + regexp.MustCompile(`^(.+(a|e|š)nj)(ijemu|ijima|ijega|ijeg|ijem|ijim|ijih|ijoj|oga|ome|omu|ima|iji|ije|ija|iju|ega|emu|eg|em|im|ih|oj|om|og|a|e|i|o|u)$`), + regexp.MustCompile(`^(.+čin)(ama|ome|omu|oga|ima|og|om|im|ih|oj|a|u|i|o|e|)$`), + regexp.MustCompile(`^(.+roši)(vši|smo|ste|še|mo|te|ti|li|la|lo|le|m|š|t|h|o)$`), + regexp.MustCompile(`^(.+oš)(ijemu|ijima|ijega|ijeg|ijem|ijim|ijih|ijoj|oga|ome|omu|ima|iji|ije|ija|iju|im|ih|oj|om|og|i|a|u|e|)$`), + regexp.MustCompile(`^(.+(e|o)vit)(ijima|ijega|ijemu|ijem|ijim|ijih|ijoj|ijeg|iji|ije|ija|oga|ome|omu|ima|og|om|im|ih|oj|i|e|o|a|u|)$`), + regexp.MustCompile(`^(.+ast)(ijima|ijega|ijemu|ijem|ijim|ijih|ijoj|ijeg|iji|ije|ija|oga|ome|omu|ima|og|om|im|ih|oj|i|e|o|a|u|)$`), + regexp.MustCompile(`^(.+k)(ijemu|ijima|ijega|ijeg|ijem|ijim|ijih|ijoj|oga|ome|omu|ima|iji|ije|ija|iju|im|ih|oj|om|og|i|a|u|e|o|)$`), + regexp.MustCompile(`^(.+(e|a|i|u)va)(jući|smo|ste|jmo|jte|ju|la|le|li|lo|mo|na|ne|ni|no|te|ti|še|hu|h|j|m|n|o|t|v|š|)$`), + regexp.MustCompile(`^(.+ir)(ujemo|ujete|ujući|ajući|ivat|ujem|uješ|ujmo|ujte|avši|asmo|aste|ati|amo|ate|aju|aše|ahu|ala|alo|ali|ale|uje|uju|uj|al|an|am|aš|at|ah|ao)$`), + regexp.MustCompile(`^(.+ač)(ismo|iste|iti|imo|ite|iše|eći|ila|ilo|ili|ile|ena|eno|eni|ene|io|im|iš|it|ih|en|i|e)$`), + regexp.MustCompile(`^(.+ača)(vši|smo|ste|smo|ste|hu|ti|mo|te|še|la|lo|li|le|ju|na|no|ni|ne|o|m|š|t|h|n)$`), + regexp.MustCompile(`^(.+n)(uvši|usmo|uste|ući|imo|ite|emo|ete|ula|ulo|ule|uli|uto|uti|uta|em|eš|uo|ut|e|u|i)$`), + regexp.MustCompile(`^(.+ni)(vši|smo|ste|ti|mo|te|mo|te|la|lo|le|li|m|š|o)$`), + regexp.MustCompile(`^(.+((a|r|i|p|e|u)st|[^o]g|ik|uc|oj|aj|lj|ak|ck|čk|šk|uk|nj|im|ar|at|et|št|it|ot|ut|zn|zv)a)(jući|vši|smo|ste|jmo|jte|jem|mo|te|je|ju|ti|še|hu|la|li|le|lo|na|no|ni|ne|t|h|o|j|n|m|š)$`), + regexp.MustCompile(`^(.+ur)(ajući|asmo|aste|ajmo|ajte|amo|ate|aju|ati|aše|ahu|ala|ali|ale|alo|ana|ano|ani|ane|al|at|ah|ao|aj|an|am|aš)$`), + regexp.MustCompile(`^(.+(a|i|o)staj)(asmo|aste|ahu|ati|emo|ete|aše|ali|ući|ala|alo|ale|mo|ao|em|eš|at|ah|te|e|u|)$`), + regexp.MustCompile(`^(.+(b|c|č|ć|d|e|f|g|j|k|n|r|t|u|v)a)(lama|lima|lom|lu|li|la|le|lo|l)$`), + regexp.MustCompile(`^(.+(t|č|j|ž|š)aj)(evima|evi|eva|eve|ama|ima|em|a|e|i|u|)$`), + regexp.MustCompile(`^(.+([^o]m|ič|nč|uč|b|c|ć|d|đ|h|j|k|l|n|p|r|s|š|v|z|ž)a)(jući|vši|smo|ste|jmo|jte|mo|te|ju|ti|še|hu|la|li|le|lo|na|no|ni|ne|t|h|o|j|n|m|š)$`), + regexp.MustCompile(`^(.+(a|i|o)sta)(dosmo|doste|doše|nemo|demo|nete|dete|nimo|nite|nila|vši|nem|dem|neš|deš|doh|de|ti|ne|nu|du|la|li|lo|le|t|o)$`), + regexp.MustCompile(`^(.+ta)(smo|ste|jmo|jte|vši|ti|mo|te|ju|še|la|lo|le|li|na|no|ni|ne|n|j|o|m|š|t|h)$`), + regexp.MustCompile(`^(.+inj)(asmo|aste|ati|emo|ete|ali|ala|alo|ale|aše|ahu|em|eš|at|ah|ao)$`), + regexp.MustCompile(`^(.+as)(temo|tete|timo|tite|tući|tem|teš|tao|te|li|ti|la|lo|le)$`), + regexp.MustCompile(`^(.+(elj|ulj|tit|ac|ič|od|oj|et|av|ov)i)(vši|eći|smo|ste|še|mo|te|ti|li|la|lo|le|m|š|t|h|o)$`), + regexp.MustCompile(`^(.+(tit|jeb|ar|ed|uš|ič)i)(jemo|jete|jem|ješ|smo|ste|jmo|jte|vši|mo|še|te|ti|ju|je|la|lo|li|le|t|m|š|h|j|o)$`), + regexp.MustCompile(`^(.+(b|č|d|l|m|p|r|s|š|ž)i)(jemo|jete|jem|ješ|smo|ste|jmo|jte|vši|mo|lu|še|te|ti|ju|je|la|lo|li|le|t|m|š|h|j|o)$`), + regexp.MustCompile(`^(.+luč)(ujete|ujući|ujemo|ujem|uješ|ismo|iste|ujmo|ujte|uje|uju|iše|iti|imo|ite|ila|ilo|ili|ile|ena|eno|eni|ene|uj|io|en|im|iš|it|ih|e|i)$`), + regexp.MustCompile(`^(.+jeti)(smo|ste|še|mo|te|ti|li|la|lo|le|m|š|t|h|o)$`), + regexp.MustCompile(`^(.+e)(lama|lima|lom|lu|li|la|le|lo|l)$`), + regexp.MustCompile(`^(.+i)(lama|lima|lom|lu|li|la|le|lo|l)$`), + regexp.MustCompile(`^(.+at)(ijega|ijemu|ijima|ijeg|ijem|ijih|ijim|ima|oga|ome|omu|iji|ije|ija|iju|oj|og|om|im|ih|a|u|i|e|o|)$`), + regexp.MustCompile(`^(.+et)(avši|ući|emo|imo|em|eš|e|u|i)$`), + regexp.MustCompile(`^(.+)(ajući|alima|alom|avši|asmo|aste|ajmo|ajte|ivši|amo|ate|aju|ati|aše|ahu|ali|ala|ale|alo|ana|ano|ani|ane|am|aš|at|ah|ao|aj|an)$`), + regexp.MustCompile(`^(.+)(anje|enje|anja|enja|enom|enoj|enog|enim|enih|anom|anoj|anog|anim|anih|eno|ovi|ova|oga|ima|ove|enu|anu|ena|ama)$`), + regexp.MustCompile(`^(.+)(nijega|nijemu|nijima|nijeg|nijem|nijim|nijih|nima|niji|nije|nija|niju|noj|nom|nog|nim|nih|an|na|nu|ni|ne|no)$`), + regexp.MustCompile(`^(.+)(om|og|im|ih|em|oj|an|u|o|i|e|a)$`), +} + +var highlightVowelRRegex = regexp.MustCompile(`(^|[^aeiou])r($|[^aeiou])`) + +func highlightVowelR(term string) string { + return highlightVowelRRegex.ReplaceAllString(term, `${1}R${2}`) +} + +func hasVowel(term string) bool { + term = highlightVowelR(term) + return strings.ContainsAny(term, "aeiouR") +} + +func stem(term string) string { + for _, rule := range stemmingRules { + results := rule.FindStringSubmatch(term) + if len(results) == 0 { + continue + } + + root := results[1] + if hasVowel(root) && root != "" { + return root + } + } + + return term +} + +type CroatianStemmerFilter struct{} + +func NewCroatianStemmerFilter() *CroatianStemmerFilter { + return &CroatianStemmerFilter{} +} + +func (s *CroatianStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + token.Term = []byte(stem(string(token.Term))) + } + + return input +} + +func CroatianStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewCroatianStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(StemmerName, CroatianStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/hr/stop_filter_hr.go b/analysis/lang/hr/stop_filter_hr.go new file mode 100644 index 0000000..c79015f --- /dev/null +++ b/analysis/lang/hr/stop_filter_hr.go @@ -0,0 +1,36 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hr + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/hr/stop_words_hr.go b/analysis/lang/hr/stop_words_hr.go new file mode 100644 index 0000000..3bd3845 --- /dev/null +++ b/analysis/lang/hr/stop_words_hr.go @@ -0,0 +1,111 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hr + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_hr" + +var CroatianStopWords = []byte(`biti +jesam +budem +sam +jesi +budeš +si +jesmo +budemo +smo +jeste +budete +ste +jesu +budu +su +bih +bijah +bjeh +bijaše +bi +bje +bješe +bijasmo +bismo +bjesmo +bijaste +biste +bjeste +bijahu +biste +bjeste +bijahu +bi +biše +bjehu +bješe +bio +bili +budimo +budite +bila +bilo +bile +ću +ćeš +će +ćemo +ćete +želim +želiš +želi +želimo +želite +žele +moram +moraš +mora +moramo +morate +moraju +trebam +trebaš +treba +trebamo +trebate +trebaju +mogu +možeš +može +možemo +možete +za +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(CroatianStopWords) + + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/hr/suffix_transformation_hr.go b/analysis/lang/hr/suffix_transformation_hr.go new file mode 100644 index 0000000..3223ee4 --- /dev/null +++ b/analysis/lang/hr/suffix_transformation_hr.go @@ -0,0 +1,189 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hr + +import ( + "strings" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const SuffixTransformationFilterName = "hr_suffix_transformation_filter" + +var SuffixTransformations = map[string]string{ + "lozi": "loga", + "lozima": "loga", + "pjesi": "pjeh", + "pjesima": "pjeh", + "vojci": "vojka", + "bojci": "bojka", + "jaci": "jak", + "jacima": "jak", + "čajan": "čajni", + "ijeran": "ijerni", + "laran": "larni", + "ijesan": "ijesni", + "anjac": "anjca", + "ajac": "ajca", + "ajaca": "ajca", + "ljaca": "ljca", + "ljac": "ljca", + "ejac": "ejca", + "ejaca": "ejca", + "ojac": "ojca", + "ojaca": "ojca", + "ajaka": "ajka", + "ojaka": "ojka", + "šaca": "šca", + "šac": "šca", + "inzima": "ing", + "inzi": "ing", + "tvenici": "tvenik", + "tetici": "tetika", + "teticima": "tetika", + "nstava": "nstva", + "nicima": "nik", + "ticima": "tik", + "zicima": "zik", + "snici": "snik", + "kuse": "kusi", + "kusan": "kusni", + "kustava": "kustva", + "dušan": "dušni", + "antan": "antni", + "bilan": "bilni", + "tilan": "tilni", + "avilan": "avilni", + "silan": "silni", + "gilan": "gilni", + "rilan": "rilni", + "nilan": "nilni", + "alan": "alni", + "ozan": "ozni", + "rave": "ravi", + "stavan": "stavni", + "pravan": "pravni", + "tivan": "tivni", + "sivan": "sivni", + "atan": "atni", + "cenata": "centa", + "denata": "denta", + "genata": "genta", + "lenata": "lenta", + "menata": "menta", + "jenata": "jenta", + "venata": "venta", + "tetan": "tetni", + "pletan": "pletni", + "šave": "šavi", + "manata": "manta", + "tanata": "tanta", + "lanata": "lanta", + "sanata": "santa", + "ačak": "ačka", + "ačaka": "ačka", + "ušak": "uška", + "atak": "atka", + "ataka": "atka", + "atci": "atka", + "atcima": "atka", + "etak": "etka", + "etaka": "etka", + "itak": "itka", + "itaka": "itka", + "itci": "itka", + "otak": "otka", + "otaka": "otka", + "utak": "utka", + "utaka": "utka", + "utci": "utka", + "utcima": "utka", + "eskan": "eskna", + "tičan": "tični", + "ojsci": "ojska", + "esama": "esma", + "metara": "metra", + "centar": "centra", + "centara": "centra", + "istara": "istra", + "istar": "istra", + "ošću": "osti", + "daba": "dba", + "čcima": "čka", + "čci": "čka", + "mac": "mca", + "maca": "mca", + "voljan": "voljni", + "anaka": "anki", + "vac": "vca", + "vaca": "vca", + "saca": "sca", + "sac": "sca", + "naca": "nca", + "nac": "nca", + "raca": "rca", + "rac": "rca", + "aoca": "alca", + "alaca": "alca", + "alac": "alca", + "elaca": "elca", + "elac": "elca", + "olaca": "olca", + "olac": "olca", + "olce": "olca", + "njac": "njca", + "njaca": "njca", + "ekata": "ekta", + "ekat": "ekta", + "izam": "izma", + "izama": "izma", + "jebe": "jebi", + "ašan": "ašni", +} + +type SuffixTransformationFilter struct{} + +func NewSuffixTransformationFilter() *SuffixTransformationFilter { + return &SuffixTransformationFilter{} +} + +func (s *SuffixTransformationFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + term := string(token.Term) + + for suffix, newSuffix := range SuffixTransformations { + if strings.HasSuffix(term, suffix) { + term = term[:len(term)-len(suffix)] + newSuffix + break + } + } + + token.Term = []byte(term) + } + + return input +} + +func SuffixTransformationFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewSuffixTransformationFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(SuffixTransformationFilterName, SuffixTransformationFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/hu/analyzer_hu.go b/analysis/lang/hu/analyzer_hu.go new file mode 100644 index 0000000..09ba091 --- /dev/null +++ b/analysis/lang/hu/analyzer_hu.go @@ -0,0 +1,60 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hu + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +) + +const AnalyzerName = "hu" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + unicodeTokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopHuFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + stemmerHuFilter, err := cache.TokenFilterNamed(SnowballStemmerName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: unicodeTokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + stopHuFilter, + stemmerHuFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/hu/analyzer_hu_test.go b/analysis/lang/hu/analyzer_hu_test.go new file mode 100644 index 0000000..8745668 --- /dev/null +++ b/analysis/lang/hu/analyzer_hu_test.go @@ -0,0 +1,70 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hu + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestHungarianAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // stemming + { + input: []byte("babakocsi"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("babakocs"), + }, + }, + }, + { + input: []byte("babakocsijáért"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("babakocs"), + }, + }, + }, + // stop word + { + input: []byte("által"), + output: analysis.TokenStream{}, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if len(actual) != len(test.output) { + t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) + } + for i, tok := range actual { + if !reflect.DeepEqual(tok.Term, test.output[i].Term) { + t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) + } + } + } +} diff --git a/analysis/lang/hu/stemmer_hu.go b/analysis/lang/hu/stemmer_hu.go new file mode 100644 index 0000000..3aadd6a --- /dev/null +++ b/analysis/lang/hu/stemmer_hu.go @@ -0,0 +1,52 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hu + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/snowballstem" + "github.com/blevesearch/snowballstem/hungarian" +) + +const SnowballStemmerName = "stemmer_hu_snowball" + +type HungarianStemmerFilter struct { +} + +func NewHungarianStemmerFilter() *HungarianStemmerFilter { + return &HungarianStemmerFilter{} +} + +func (s *HungarianStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + env := snowballstem.NewEnv(string(token.Term)) + hungarian.Stem(env) + token.Term = []byte(env.Current()) + } + return input +} + +func HungarianStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewHungarianStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(SnowballStemmerName, HungarianStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/hu/stop_filter_hu.go b/analysis/lang/hu/stop_filter_hu.go new file mode 100644 index 0000000..ebc6894 --- /dev/null +++ b/analysis/lang/hu/stop_filter_hu.go @@ -0,0 +1,36 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hu + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/hu/stop_words_hu.go b/analysis/lang/hu/stop_words_hu.go new file mode 100644 index 0000000..effdc04 --- /dev/null +++ b/analysis/lang/hu/stop_words_hu.go @@ -0,0 +1,238 @@ +package hu + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_hu" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ +// ` was changed to ' to allow for literal string + +var HungarianStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/hungarian/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + +| Hungarian stop word list +| prepared by Anna Tordai + +a +ahogy +ahol +aki +akik +akkor +alatt +által +általában +amely +amelyek +amelyekben +amelyeket +amelyet +amelynek +ami +amit +amolyan +amíg +amikor +át +abban +ahhoz +annak +arra +arról +az +azok +azon +azt +azzal +azért +aztán +azután +azonban +bár +be +belül +benne +cikk +cikkek +cikkeket +csak +de +e +eddig +egész +egy +egyes +egyetlen +egyéb +egyik +egyre +ekkor +el +elég +ellen +elő +először +előtt +első +én +éppen +ebben +ehhez +emilyen +ennek +erre +ez +ezt +ezek +ezen +ezzel +ezért +és +fel +felé +hanem +hiszen +hogy +hogyan +igen +így +illetve +ill. +ill +ilyen +ilyenkor +ison +ismét +itt +jó +jól +jobban +kell +kellett +keresztül +keressünk +ki +kívül +között +közül +legalább +lehet +lehetett +legyen +lenne +lenni +lesz +lett +maga +magát +majd +majd +már +más +másik +meg +még +mellett +mert +mely +melyek +mi +mit +míg +miért +milyen +mikor +minden +mindent +mindenki +mindig +mint +mintha +mivel +most +nagy +nagyobb +nagyon +ne +néha +nekem +neki +nem +néhány +nélkül +nincs +olyan +ott +össze +ő +ők +őket +pedig +persze +rá +s +saját +sem +semmi +sok +sokat +sokkal +számára +szemben +szerint +szinte +talán +tehát +teljes +tovább +továbbá +több +úgy +ugyanis +új +újabb +újra +után +utána +utolsó +vagy +vagyis +valaki +valami +valamint +való +vagyok +van +vannak +volt +voltam +voltak +voltunk +vissza +vele +viszont +volna +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(HungarianStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/hy/stop_filter_hy.go b/analysis/lang/hy/stop_filter_hy.go new file mode 100644 index 0000000..eb69789 --- /dev/null +++ b/analysis/lang/hy/stop_filter_hy.go @@ -0,0 +1,36 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hy + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/hy/stop_words_hy.go b/analysis/lang/hy/stop_words_hy.go new file mode 100644 index 0000000..dcb6603 --- /dev/null +++ b/analysis/lang/hy/stop_words_hy.go @@ -0,0 +1,73 @@ +package hy + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_hy" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ +// ` was changed to ' to allow for literal string + +var ArmenianStopWords = []byte(`# example set of Armenian stopwords. +այդ +այլ +այն +այս +դու +դուք +եմ +են +ենք +ես +եք +է +էի +էին +էինք +էիր +էիք +էր +ըստ +թ +ի +ին +իսկ +իր +կամ +համար +հետ +հետո +մենք +մեջ +մի +ն +նա +նաև +նրա +նրանք +որ +որը +որոնք +որպես +ու +ում +պիտի +վրա +և +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(ArmenianStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/id/stop_filter_id.go b/analysis/lang/id/stop_filter_id.go new file mode 100644 index 0000000..252f131 --- /dev/null +++ b/analysis/lang/id/stop_filter_id.go @@ -0,0 +1,36 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package id + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/id/stop_words_id.go b/analysis/lang/id/stop_words_id.go new file mode 100644 index 0000000..b85a816 --- /dev/null +++ b/analysis/lang/id/stop_words_id.go @@ -0,0 +1,386 @@ +package id + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_id" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ +// ` was changed to ' to allow for literal string + +var IndonesianStopWords = []byte(`# from appendix D of: A Study of Stemming Effects on Information +# Retrieval in Bahasa Indonesia +ada +adanya +adalah +adapun +agak +agaknya +agar +akan +akankah +akhirnya +aku +akulah +amat +amatlah +anda +andalah +antar +diantaranya +antara +antaranya +diantara +apa +apaan +mengapa +apabila +apakah +apalagi +apatah +atau +ataukah +ataupun +bagai +bagaikan +sebagai +sebagainya +bagaimana +bagaimanapun +sebagaimana +bagaimanakah +bagi +bahkan +bahwa +bahwasanya +sebaliknya +banyak +sebanyak +beberapa +seberapa +begini +beginian +beginikah +beginilah +sebegini +begitu +begitukah +begitulah +begitupun +sebegitu +belum +belumlah +sebelum +sebelumnya +sebenarnya +berapa +berapakah +berapalah +berapapun +betulkah +sebetulnya +biasa +biasanya +bila +bilakah +bisa +bisakah +sebisanya +boleh +bolehkah +bolehlah +buat +bukan +bukankah +bukanlah +bukannya +cuma +percuma +dahulu +dalam +dan +dapat +dari +daripada +dekat +demi +demikian +demikianlah +sedemikian +dengan +depan +di +dia +dialah +dini +diri +dirinya +terdiri +dong +dulu +enggak +enggaknya +entah +entahlah +terhadap +terhadapnya +hal +hampir +hanya +hanyalah +harus +haruslah +harusnya +seharusnya +hendak +hendaklah +hendaknya +hingga +sehingga +ia +ialah +ibarat +ingin +inginkah +inginkan +ini +inikah +inilah +itu +itukah +itulah +jangan +jangankan +janganlah +jika +jikalau +juga +justru +kala +kalau +kalaulah +kalaupun +kalian +kami +kamilah +kamu +kamulah +kan +kapan +kapankah +kapanpun +dikarenakan +karena +karenanya +ke +kecil +kemudian +kenapa +kepada +kepadanya +ketika +seketika +khususnya +kini +kinilah +kiranya +sekiranya +kita +kitalah +kok +lagi +lagian +selagi +lah +lain +lainnya +melainkan +selaku +lalu +melalui +terlalu +lama +lamanya +selama +selama +selamanya +lebih +terlebih +bermacam +macam +semacam +maka +makanya +makin +malah +malahan +mampu +mampukah +mana +manakala +manalagi +masih +masihkah +semasih +masing +mau +maupun +semaunya +memang +mereka +merekalah +meski +meskipun +semula +mungkin +mungkinkah +nah +namun +nanti +nantinya +nyaris +oleh +olehnya +seorang +seseorang +pada +padanya +padahal +paling +sepanjang +pantas +sepantasnya +sepantasnyalah +para +pasti +pastilah +per +pernah +pula +pun +merupakan +rupanya +serupa +saat +saatnya +sesaat +saja +sajalah +saling +bersama +sama +sesama +sambil +sampai +sana +sangat +sangatlah +saya +sayalah +se +sebab +sebabnya +sebuah +tersebut +tersebutlah +sedang +sedangkan +sedikit +sedikitnya +segala +segalanya +segera +sesegera +sejak +sejenak +sekali +sekalian +sekalipun +sesekali +sekaligus +sekarang +sekarang +sekitar +sekitarnya +sela +selain +selalu +seluruh +seluruhnya +semakin +sementara +sempat +semua +semuanya +sendiri +sendirinya +seolah +seperti +sepertinya +sering +seringnya +serta +siapa +siapakah +siapapun +disini +disinilah +sini +sinilah +sesuatu +sesuatunya +suatu +sesudah +sesudahnya +sudah +sudahkah +sudahlah +supaya +tadi +tadinya +tak +tanpa +setelah +telah +tentang +tentu +tentulah +tentunya +tertentu +seterusnya +tapi +tetapi +setiap +tiap +setidaknya +tidak +tidakkah +tidaklah +toh +waduh +wah +wahai +sewaktu +walau +walaupun +wong +yaitu +yakni +yang +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(IndonesianStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/in/indic_normalize.go b/analysis/lang/in/indic_normalize.go new file mode 100644 index 0000000..f3ad7d9 --- /dev/null +++ b/analysis/lang/in/indic_normalize.go @@ -0,0 +1,51 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package in + +import ( + "bytes" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const NormalizeName = "normalize_in" + +type IndicNormalizeFilter struct { +} + +func NewIndicNormalizeFilter() *IndicNormalizeFilter { + return &IndicNormalizeFilter{} +} + +func (s *IndicNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + runes := bytes.Runes(token.Term) + runes = normalize(runes) + token.Term = analysis.BuildTermFromRunes(runes) + } + return input +} + +func NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewIndicNormalizeFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/in/indic_normalize_test.go b/analysis/lang/in/indic_normalize_test.go new file mode 100644 index 0000000..5789eb7 --- /dev/null +++ b/analysis/lang/in/indic_normalize_test.go @@ -0,0 +1,138 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package in + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestIndicNormalizeFilter(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + // basics + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अाॅअाॅ"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ऑऑ"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अाॆअाॆ"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ऒऒ"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अाेअाे"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ओओ"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अाैअाै"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("औऔ"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अाअा"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("आआ"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("अाैर"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("और"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ত্‍"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ৎ"), + }, + }, + }, + // empty term + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte(""), + }, + }, + }, + } + + indicNormalizeFilter := NewIndicNormalizeFilter() + for _, test := range tests { + actual := indicNormalizeFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %#v, got %#v", test.output, actual) + t.Errorf("expected % x, got % x for % x", test.output[0].Term, actual[0].Term, test.input[0].Term) + t.Errorf("expected %s, got %s for %s", test.output[0].Term, actual[0].Term, test.input[0].Term) + } + } +} diff --git a/analysis/lang/in/scripts.go b/analysis/lang/in/scripts.go new file mode 100644 index 0000000..cc32f10 --- /dev/null +++ b/analysis/lang/in/scripts.go @@ -0,0 +1,296 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package in + +import ( + "unicode" + + "github.com/bits-and-blooms/bitset" + "github.com/blevesearch/bleve/v2/analysis" +) + +type ScriptData struct { + flag rune + base rune + decompMask *bitset.BitSet +} + +var scripts = map[*unicode.RangeTable]*ScriptData{ + unicode.Devanagari: { + flag: 1, + base: 0x0900, + }, + unicode.Bengali: { + flag: 2, + base: 0x0980, + }, + unicode.Gurmukhi: { + flag: 4, + base: 0x0A00, + }, + unicode.Gujarati: { + flag: 8, + base: 0x0A80, + }, + unicode.Oriya: { + flag: 16, + base: 0x0B00, + }, + unicode.Tamil: { + flag: 32, + base: 0x0B80, + }, + unicode.Telugu: { + flag: 64, + base: 0x0C00, + }, + unicode.Kannada: { + flag: 128, + base: 0x0C80, + }, + unicode.Malayalam: { + flag: 256, + base: 0x0D00, + }, +} + +func flag(ub *unicode.RangeTable) rune { + return scripts[ub].flag +} + +var decompositions = [][]rune{ + /* devanagari, gujarati vowel candra O */ + {0x05, 0x3E, 0x45, 0x11, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, + /* devanagari short O */ + {0x05, 0x3E, 0x46, 0x12, flag(unicode.Devanagari)}, + /* devanagari, gujarati letter O */ + {0x05, 0x3E, 0x47, 0x13, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, + /* devanagari letter AI, gujarati letter AU */ + {0x05, 0x3E, 0x48, 0x14, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, + /* devanagari, bengali, gurmukhi, gujarati, oriya AA */ + {0x05, 0x3E, -1, 0x06, flag(unicode.Devanagari) | flag(unicode.Bengali) | flag(unicode.Gurmukhi) | flag(unicode.Gujarati) | flag(unicode.Oriya)}, + /* devanagari letter candra A */ + {0x05, 0x45, -1, 0x72, flag(unicode.Devanagari)}, + /* gujarati vowel candra E */ + {0x05, 0x45, -1, 0x0D, flag(unicode.Gujarati)}, + /* devanagari letter short A */ + {0x05, 0x46, -1, 0x04, flag(unicode.Devanagari)}, + /* gujarati letter E */ + {0x05, 0x47, -1, 0x0F, flag(unicode.Gujarati)}, + /* gurmukhi, gujarati letter AI */ + {0x05, 0x48, -1, 0x10, flag(unicode.Gurmukhi) | flag(unicode.Gujarati)}, + /* devanagari, gujarati vowel candra O */ + {0x05, 0x49, -1, 0x11, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, + /* devanagari short O */ + {0x05, 0x4A, -1, 0x12, flag(unicode.Devanagari)}, + /* devanagari, gujarati letter O */ + {0x05, 0x4B, -1, 0x13, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, + /* devanagari letter AI, gurmukhi letter AU, gujarati letter AU */ + {0x05, 0x4C, -1, 0x14, flag(unicode.Devanagari) | flag(unicode.Gurmukhi) | flag(unicode.Gujarati)}, + /* devanagari, gujarati vowel candra O */ + {0x06, 0x45, -1, 0x11, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, + /* devanagari short O */ + {0x06, 0x46, -1, 0x12, flag(unicode.Devanagari)}, + /* devanagari, gujarati letter O */ + {0x06, 0x47, -1, 0x13, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, + /* devanagari letter AI, gujarati letter AU */ + {0x06, 0x48, -1, 0x14, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, + /* malayalam letter II */ + {0x07, 0x57, -1, 0x08, flag(unicode.Malayalam)}, + /* devanagari letter UU */ + {0x09, 0x41, -1, 0x0A, flag(unicode.Devanagari)}, + /* tamil, malayalam letter UU (some styles) */ + {0x09, 0x57, -1, 0x0A, flag(unicode.Tamil) | flag(unicode.Malayalam)}, + /* malayalam letter AI */ + {0x0E, 0x46, -1, 0x10, flag(unicode.Malayalam)}, + /* devanagari candra E */ + {0x0F, 0x45, -1, 0x0D, flag(unicode.Devanagari)}, + /* devanagari short E */ + {0x0F, 0x46, -1, 0x0E, flag(unicode.Devanagari)}, + /* devanagari AI */ + {0x0F, 0x47, -1, 0x10, flag(unicode.Devanagari)}, + /* oriya AI */ + {0x0F, 0x57, -1, 0x10, flag(unicode.Oriya)}, + /* malayalam letter OO */ + {0x12, 0x3E, -1, 0x13, flag(unicode.Malayalam)}, + /* telugu, kannada letter AU */ + {0x12, 0x4C, -1, 0x14, flag(unicode.Telugu) | flag(unicode.Kannada)}, + /* telugu letter OO */ + {0x12, 0x55, -1, 0x13, flag(unicode.Telugu)}, + /* tamil, malayalam letter AU */ + {0x12, 0x57, -1, 0x14, flag(unicode.Tamil) | flag(unicode.Malayalam)}, + /* oriya letter AU */ + {0x13, 0x57, -1, 0x14, flag(unicode.Oriya)}, + /* devanagari qa */ + {0x15, 0x3C, -1, 0x58, flag(unicode.Devanagari)}, + /* devanagari, gurmukhi khha */ + {0x16, 0x3C, -1, 0x59, flag(unicode.Devanagari) | flag(unicode.Gurmukhi)}, + /* devanagari, gurmukhi ghha */ + {0x17, 0x3C, -1, 0x5A, flag(unicode.Devanagari) | flag(unicode.Gurmukhi)}, + /* devanagari, gurmukhi za */ + {0x1C, 0x3C, -1, 0x5B, flag(unicode.Devanagari) | flag(unicode.Gurmukhi)}, + /* devanagari dddha, bengali, oriya rra */ + {0x21, 0x3C, -1, 0x5C, flag(unicode.Devanagari) | flag(unicode.Bengali) | flag(unicode.Oriya)}, + /* devanagari, bengali, oriya rha */ + {0x22, 0x3C, -1, 0x5D, flag(unicode.Devanagari) | flag(unicode.Bengali) | flag(unicode.Oriya)}, + /* malayalam chillu nn */ + {0x23, 0x4D, 0xFF, 0x7A, flag(unicode.Malayalam)}, + /* bengali khanda ta */ + {0x24, 0x4D, 0xFF, 0x4E, flag(unicode.Bengali)}, + /* devanagari nnna */ + {0x28, 0x3C, -1, 0x29, flag(unicode.Devanagari)}, + /* malayalam chillu n */ + {0x28, 0x4D, 0xFF, 0x7B, flag(unicode.Malayalam)}, + /* devanagari, gurmukhi fa */ + {0x2B, 0x3C, -1, 0x5E, flag(unicode.Devanagari) | flag(unicode.Gurmukhi)}, + /* devanagari, bengali yya */ + {0x2F, 0x3C, -1, 0x5F, flag(unicode.Devanagari) | flag(unicode.Bengali)}, + /* telugu letter vocalic R */ + {0x2C, 0x41, 0x41, 0x0B, flag(unicode.Telugu)}, + /* devanagari rra */ + {0x30, 0x3C, -1, 0x31, flag(unicode.Devanagari)}, + /* malayalam chillu rr */ + {0x30, 0x4D, 0xFF, 0x7C, flag(unicode.Malayalam)}, + /* malayalam chillu l */ + {0x32, 0x4D, 0xFF, 0x7D, flag(unicode.Malayalam)}, + /* devanagari llla */ + {0x33, 0x3C, -1, 0x34, flag(unicode.Devanagari)}, + /* malayalam chillu ll */ + {0x33, 0x4D, 0xFF, 0x7E, flag(unicode.Malayalam)}, + /* telugu letter MA */ + {0x35, 0x41, -1, 0x2E, flag(unicode.Telugu)}, + /* devanagari, gujarati vowel sign candra O */ + {0x3E, 0x45, -1, 0x49, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, + /* devanagari vowel sign short O */ + {0x3E, 0x46, -1, 0x4A, flag(unicode.Devanagari)}, + /* devanagari, gujarati vowel sign O */ + {0x3E, 0x47, -1, 0x4B, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, + /* devanagari, gujarati vowel sign AU */ + {0x3E, 0x48, -1, 0x4C, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, + /* kannada vowel sign II */ + {0x3F, 0x55, -1, 0x40, flag(unicode.Kannada)}, + /* gurmukhi vowel sign UU (when stacking) */ + {0x41, 0x41, -1, 0x42, flag(unicode.Gurmukhi)}, + /* tamil, malayalam vowel sign O */ + {0x46, 0x3E, -1, 0x4A, flag(unicode.Tamil) | flag(unicode.Malayalam)}, + /* kannada vowel sign OO */ + {0x46, 0x42, 0x55, 0x4B, flag(unicode.Kannada)}, + /* kannada vowel sign O */ + {0x46, 0x42, -1, 0x4A, flag(unicode.Kannada)}, + /* malayalam vowel sign AI (if reordered twice) */ + {0x46, 0x46, -1, 0x48, flag(unicode.Malayalam)}, + /* telugu, kannada vowel sign EE */ + {0x46, 0x55, -1, 0x47, flag(unicode.Telugu) | flag(unicode.Kannada)}, + /* telugu, kannada vowel sign AI */ + {0x46, 0x56, -1, 0x48, flag(unicode.Telugu) | flag(unicode.Kannada)}, + /* tamil, malayalam vowel sign AU */ + {0x46, 0x57, -1, 0x4C, flag(unicode.Tamil) | flag(unicode.Malayalam)}, + /* bengali, oriya vowel sign O, tamil, malayalam vowel sign OO */ + {0x47, 0x3E, -1, 0x4B, flag(unicode.Bengali) | flag(unicode.Oriya) | flag(unicode.Tamil) | flag(unicode.Malayalam)}, + /* bengali, oriya vowel sign AU */ + {0x47, 0x57, -1, 0x4C, flag(unicode.Bengali) | flag(unicode.Oriya)}, + /* kannada vowel sign OO */ + {0x4A, 0x55, -1, 0x4B, flag(unicode.Kannada)}, + /* gurmukhi letter I */ + {0x72, 0x3F, -1, 0x07, flag(unicode.Gurmukhi)}, + /* gurmukhi letter II */ + {0x72, 0x40, -1, 0x08, flag(unicode.Gurmukhi)}, + /* gurmukhi letter EE */ + {0x72, 0x47, -1, 0x0F, flag(unicode.Gurmukhi)}, + /* gurmukhi letter U */ + {0x73, 0x41, -1, 0x09, flag(unicode.Gurmukhi)}, + /* gurmukhi letter UU */ + {0x73, 0x42, -1, 0x0A, flag(unicode.Gurmukhi)}, + /* gurmukhi letter OO */ + {0x73, 0x4B, -1, 0x13, flag(unicode.Gurmukhi)}, +} + +func init() { + for _, scriptData := range scripts { + scriptData.decompMask = bitset.New(0x7d) + for _, decomposition := range decompositions { + ch := decomposition[0] + flags := decomposition[4] + if (flags & scriptData.flag) != 0 { + scriptData.decompMask.Set(uint(ch)) + } + } + } +} + +func lookupScript(r rune) *unicode.RangeTable { + for script := range scripts { + if unicode.Is(script, r) { + return script + } + } + return nil +} + +func normalize(input []rune) []rune { + inputLen := len(input) + for i := 0; i < inputLen; i++ { + r := input[i] + script := lookupScript(r) + if script != nil { + scriptData := scripts[script] + ch := r - scriptData.base + if scriptData.decompMask.Test(uint(ch)) { + input = compose(ch, script, scriptData, input, i, inputLen) + inputLen = len(input) + } + } + } + return input[0:inputLen] +} + +func compose(ch0 rune, script0 *unicode.RangeTable, scriptData *ScriptData, input []rune, pos int, inputLen int) []rune { + if pos+1 >= inputLen { + return input // need at least 2 characters + } + + ch1 := input[pos+1] - scriptData.base + script1 := lookupScript(input[pos+1]) + if script0 != script1 { + return input // need to be same script + } + + ch2 := rune(-1) + if pos+2 < inputLen { + ch2 = input[pos+2] - scriptData.base + script2 := lookupScript(input[pos+2]) + if input[pos+2] == '\u200D' { + ch2 = 0xff // zero width joiner + } else if script2 != script1 { + ch2 = -1 // still allow 2 character match + } + } + + for _, decomposition := range decompositions { + if decomposition[0] == ch0 && + (decomposition[4]&scriptData.flag) != 0 { + if decomposition[1] == ch1 && + (decomposition[2] < 0 || decomposition[2] == ch2) { + input[pos] = scriptData.base + decomposition[3] + input = analysis.DeleteRune(input, pos+1) + if decomposition[2] >= 0 { + input = analysis.DeleteRune(input, pos+1) + } + return input + } + } + } + return input +} diff --git a/analysis/lang/it/analyzer_it.go b/analysis/lang/it/analyzer_it.go new file mode 100644 index 0000000..8531166 --- /dev/null +++ b/analysis/lang/it/analyzer_it.go @@ -0,0 +1,65 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package it + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +) + +const AnalyzerName = "it" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + tokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + elisionFilter, err := cache.TokenFilterNamed(ElisionName) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopItFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + stemmerItFilter, err := cache.TokenFilterNamed(LightStemmerName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: tokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + elisionFilter, + stopItFilter, + stemmerItFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/it/analyzer_it_test.go b/analysis/lang/it/analyzer_it_test.go new file mode 100644 index 0000000..19b9d4d --- /dev/null +++ b/analysis/lang/it/analyzer_it_test.go @@ -0,0 +1,96 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package it + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestItalianAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // stemming + { + input: []byte("abbandonata"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abbandonat"), + }, + }, + }, + { + input: []byte("abbandonati"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abbandonat"), + }, + }, + }, + // stop word + { + input: []byte("dallo"), + output: analysis.TokenStream{}, + }, + // contractions + { + input: []byte("dell'Italia"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ital"), + }, + }, + }, + { + input: []byte("l'Italiano"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("italian"), + }, + }, + }, + // test for bug #218 + { + input: []byte("Nell'anfora"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("anfor"), + }, + }, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if len(actual) != len(test.output) { + t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) + } + for i, tok := range actual { + if !reflect.DeepEqual(tok.Term, test.output[i].Term) { + t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) + } + } + } +} diff --git a/analysis/lang/it/articles_it.go b/analysis/lang/it/articles_it.go new file mode 100644 index 0000000..08d3c01 --- /dev/null +++ b/analysis/lang/it/articles_it.go @@ -0,0 +1,48 @@ +package it + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const ArticlesName = "articles_it" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis + +var ItalianArticles = []byte(` +c +l +all +dall +dell +nell +sull +coll +pell +gl +agl +dagl +degl +negl +sugl +un +m +t +s +v +d +`) + +func ArticlesTokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(ItalianArticles) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(ArticlesName, ArticlesTokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/it/elision_it.go b/analysis/lang/it/elision_it.go new file mode 100644 index 0000000..022e9c6 --- /dev/null +++ b/analysis/lang/it/elision_it.go @@ -0,0 +1,40 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package it + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/elision" + "github.com/blevesearch/bleve/v2/registry" +) + +const ElisionName = "elision_it" + +func ElisionFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + articlesTokenMap, err := cache.TokenMapNamed(ArticlesName) + if err != nil { + return nil, fmt.Errorf("error building elision filter: %v", err) + } + return elision.NewElisionFilter(articlesTokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(ElisionName, ElisionFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/it/elision_it_test.go b/analysis/lang/it/elision_it_test.go new file mode 100644 index 0000000..b79610d --- /dev/null +++ b/analysis/lang/it/elision_it_test.go @@ -0,0 +1,55 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package it + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestItalianElision(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("dell'Italia"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Italia"), + }, + }, + }, + } + + cache := registry.NewCache() + elisionFilter, err := cache.TokenFilterNamed(ElisionName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := elisionFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/it/light_stemmer_it.go b/analysis/lang/it/light_stemmer_it.go new file mode 100644 index 0000000..57c1564 --- /dev/null +++ b/analysis/lang/it/light_stemmer_it.go @@ -0,0 +1,104 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package it + +import ( + "bytes" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const LightStemmerName = "stemmer_it_light" + +type ItalianLightStemmerFilter struct { +} + +func NewItalianLightStemmerFilterFilter() *ItalianLightStemmerFilter { + return &ItalianLightStemmerFilter{} +} + +func (s *ItalianLightStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + runes := bytes.Runes(token.Term) + runes = stem(runes) + token.Term = analysis.BuildTermFromRunes(runes) + } + return input +} + +func stem(input []rune) []rune { + + inputLen := len(input) + + if inputLen < 6 { + return input + } + + for i := 0; i < inputLen; i++ { + switch input[i] { + case 'à', 'á', 'â', 'ä': + input[i] = 'a' + case 'ò', 'ó', 'ô', 'ö': + input[i] = 'o' + case 'è', 'é', 'ê', 'ë': + input[i] = 'e' + case 'ù', 'ú', 'û', 'ü': + input[i] = 'u' + case 'ì', 'í', 'î', 'ï': + input[i] = 'i' + } + } + + switch input[inputLen-1] { + case 'e': + if input[inputLen-2] == 'i' || input[inputLen-2] == 'h' { + return input[0 : inputLen-2] + } else { + return input[0 : inputLen-1] + } + case 'i': + if input[inputLen-2] == 'h' || input[inputLen-2] == 'i' { + return input[0 : inputLen-2] + } else { + return input[0 : inputLen-1] + } + case 'a': + if input[inputLen-2] == 'i' { + return input[0 : inputLen-2] + } else { + return input[0 : inputLen-1] + } + case 'o': + if input[inputLen-2] == 'i' { + return input[0 : inputLen-2] + } else { + return input[0 : inputLen-1] + } + } + + return input +} + +func ItalianLightStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewItalianLightStemmerFilterFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(LightStemmerName, ItalianLightStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/it/light_stemmer_it_test.go b/analysis/lang/it/light_stemmer_it_test.go new file mode 100644 index 0000000..5122f19 --- /dev/null +++ b/analysis/lang/it/light_stemmer_it_test.go @@ -0,0 +1,67 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package it + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestItalianLightStemmer(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ragazzo"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ragazz"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ragazzi"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ragazz"), + }, + }, + }, + } + + cache := registry.NewCache() + filter, err := cache.TokenFilterNamed(LightStemmerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := filter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/it/stemmer_it_snowball.go b/analysis/lang/it/stemmer_it_snowball.go new file mode 100644 index 0000000..fd51641 --- /dev/null +++ b/analysis/lang/it/stemmer_it_snowball.go @@ -0,0 +1,52 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package it + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/snowballstem" + "github.com/blevesearch/snowballstem/italian" +) + +const SnowballStemmerName = "stemmer_it_snowball" + +type ItalianStemmerFilter struct { +} + +func NewItalianStemmerFilter() *ItalianStemmerFilter { + return &ItalianStemmerFilter{} +} + +func (s *ItalianStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + env := snowballstem.NewEnv(string(token.Term)) + italian.Stem(env) + token.Term = []byte(env.Current()) + } + return input +} + +func ItalianStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewItalianStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(SnowballStemmerName, ItalianStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/it/stemmer_it_snowball_test.go b/analysis/lang/it/stemmer_it_snowball_test.go new file mode 100644 index 0000000..134f2e0 --- /dev/null +++ b/analysis/lang/it/stemmer_it_snowball_test.go @@ -0,0 +1,79 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package it + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestSnowballItalianStemmer(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("aizzata"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("aizz"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("aizzargli"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("aizz"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("aizzasse"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("aizz"), + }, + }, + }, + } + + cache := registry.NewCache() + filter, err := cache.TokenFilterNamed(SnowballStemmerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := filter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/it/stop_filter_it.go b/analysis/lang/it/stop_filter_it.go new file mode 100644 index 0000000..73e297f --- /dev/null +++ b/analysis/lang/it/stop_filter_it.go @@ -0,0 +1,36 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package it + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/it/stop_words_it.go b/analysis/lang/it/stop_words_it.go new file mode 100644 index 0000000..b167f4d --- /dev/null +++ b/analysis/lang/it/stop_words_it.go @@ -0,0 +1,330 @@ +package it + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_it" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ +// ` was changed to ' to allow for literal string + +var ItalianStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/italian/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | An Italian stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + +ad | a (to) before vowel +al | a + il +allo | a + lo +ai | a + i +agli | a + gli +all | a + l' +agl | a + gl' +alla | a + la +alle | a + le +con | with +col | con + il +coi | con + i (forms collo, cogli etc are now very rare) +da | from +dal | da + il +dallo | da + lo +dai | da + i +dagli | da + gli +dall | da + l' +dagl | da + gll' +dalla | da + la +dalle | da + le +di | of +del | di + il +dello | di + lo +dei | di + i +degli | di + gli +dell | di + l' +degl | di + gl' +della | di + la +delle | di + le +in | in +nel | in + el +nello | in + lo +nei | in + i +negli | in + gli +nell | in + l' +negl | in + gl' +nella | in + la +nelle | in + le +su | on +sul | su + il +sullo | su + lo +sui | su + i +sugli | su + gli +sull | su + l' +sugl | su + gl' +sulla | su + la +sulle | su + le +per | through, by +tra | among +contro | against +io | I +tu | thou +lui | he +lei | she +noi | we +voi | you +loro | they +mio | my +mia | +miei | +mie | +tuo | +tua | +tuoi | thy +tue | +suo | +sua | +suoi | his, her +sue | +nostro | our +nostra | +nostri | +nostre | +vostro | your +vostra | +vostri | +vostre | +mi | me +ti | thee +ci | us, there +vi | you, there +lo | him, the +la | her, the +li | them +le | them, the +gli | to him, the +ne | from there etc +il | the +un | a +uno | a +una | a +ma | but +ed | and +se | if +perché | why, because +anche | also +come | how +dov | where (as dov') +dove | where +che | who, that +chi | who +cui | whom +non | not +più | more +quale | who, that +quanto | how much +quanti | +quanta | +quante | +quello | that +quelli | +quella | +quelle | +questo | this +questi | +questa | +queste | +si | yes +tutto | all +tutti | all + + | single letter forms: + +a | at +c | as c' for ce or ci +e | and +i | the +l | as l' +o | or + + | forms of avere, to have (not including the infinitive): + +ho +hai +ha +abbiamo +avete +hanno +abbia +abbiate +abbiano +avrò +avrai +avrà +avremo +avrete +avranno +avrei +avresti +avrebbe +avremmo +avreste +avrebbero +avevo +avevi +aveva +avevamo +avevate +avevano +ebbi +avesti +ebbe +avemmo +aveste +ebbero +avessi +avesse +avessimo +avessero +avendo +avuto +avuta +avuti +avute + + | forms of essere, to be (not including the infinitive): +sono +sei +è +siamo +siete +sia +siate +siano +sarò +sarai +sarà +saremo +sarete +saranno +sarei +saresti +sarebbe +saremmo +sareste +sarebbero +ero +eri +era +eravamo +eravate +erano +fui +fosti +fu +fummo +foste +furono +fossi +fosse +fossimo +fossero +essendo + + | forms of fare, to do (not including the infinitive, fa, fat-): +faccio +fai +facciamo +fanno +faccia +facciate +facciano +farò +farai +farà +faremo +farete +faranno +farei +faresti +farebbe +faremmo +fareste +farebbero +facevo +facevi +faceva +facevamo +facevate +facevano +feci +facesti +fece +facemmo +faceste +fecero +facessi +facesse +facessimo +facessero +facendo + + | forms of stare, to be (not including the infinitive): +sto +stai +sta +stiamo +stanno +stia +stiate +stiano +starò +starai +starà +staremo +starete +staranno +starei +staresti +starebbe +staremmo +stareste +starebbero +stavo +stavi +stava +stavamo +stavate +stavano +stetti +stesti +stette +stemmo +steste +stettero +stessi +stesse +stessimo +stessero +stando +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(ItalianStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/nl/analyzer_nl.go b/analysis/lang/nl/analyzer_nl.go new file mode 100644 index 0000000..22a6fbf --- /dev/null +++ b/analysis/lang/nl/analyzer_nl.go @@ -0,0 +1,60 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nl + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +) + +const AnalyzerName = "nl" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + unicodeTokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopNlFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + stemmerNlFilter, err := cache.TokenFilterNamed(SnowballStemmerName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: unicodeTokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + stopNlFilter, + stemmerNlFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/nl/analyzer_nl_test.go b/analysis/lang/nl/analyzer_nl_test.go new file mode 100644 index 0000000..707655f --- /dev/null +++ b/analysis/lang/nl/analyzer_nl_test.go @@ -0,0 +1,70 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nl + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestDutchAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // stemming + { + input: []byte("lichamelijk"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("licham"), + }, + }, + }, + { + input: []byte("lichamelijke"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("licham"), + }, + }, + }, + // stop word + { + input: []byte("van"), + output: analysis.TokenStream{}, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if len(actual) != len(test.output) { + t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) + } + for i, tok := range actual { + if !reflect.DeepEqual(tok.Term, test.output[i].Term) { + t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) + } + } + } +} diff --git a/analysis/lang/nl/stemmer_nl.go b/analysis/lang/nl/stemmer_nl.go new file mode 100644 index 0000000..5612965 --- /dev/null +++ b/analysis/lang/nl/stemmer_nl.go @@ -0,0 +1,52 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nl + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/snowballstem" + "github.com/blevesearch/snowballstem/dutch" +) + +const SnowballStemmerName = "stemmer_nl_snowball" + +type DutchStemmerFilter struct { +} + +func NewDutchStemmerFilter() *DutchStemmerFilter { + return &DutchStemmerFilter{} +} + +func (s *DutchStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + env := snowballstem.NewEnv(string(token.Term)) + dutch.Stem(env) + token.Term = []byte(env.Current()) + } + return input +} + +func DutchStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewDutchStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(SnowballStemmerName, DutchStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/nl/stop_filter_nl.go b/analysis/lang/nl/stop_filter_nl.go new file mode 100644 index 0000000..31fa91e --- /dev/null +++ b/analysis/lang/nl/stop_filter_nl.go @@ -0,0 +1,36 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nl + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/nl/stop_words_nl.go b/analysis/lang/nl/stop_words_nl.go new file mode 100644 index 0000000..bb7e6de --- /dev/null +++ b/analysis/lang/nl/stop_words_nl.go @@ -0,0 +1,146 @@ +package nl + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_nl" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ +// ` was changed to ' to allow for literal string + +var DutchStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/dutch/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A Dutch stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This is a ranked list (commonest to rarest) of stopwords derived from + | a large sample of Dutch text. + + | Dutch stop words frequently exhibit homonym clashes. These are indicated + | clearly below. + +de | the +en | and +van | of, from +ik | I, the ego +te | (1) chez, at etc, (2) to, (3) too +dat | that, which +die | that, those, who, which +in | in, inside +een | a, an, one +hij | he +het | the, it +niet | not, nothing, naught +zijn | (1) to be, being, (2) his, one's, its +is | is +was | (1) was, past tense of all persons sing. of 'zijn' (to be) (2) wax, (3) the washing, (4) rise of river +op | on, upon, at, in, up, used up +aan | on, upon, to (as dative) +met | with, by +als | like, such as, when +voor | (1) before, in front of, (2) furrow +had | had, past tense all persons sing. of 'hebben' (have) +er | there +maar | but, only +om | round, about, for etc +hem | him +dan | then +zou | should/would, past tense all persons sing. of 'zullen' +of | or, whether, if +wat | what, something, anything +mijn | possessive and noun 'mine' +men | people, 'one' +dit | this +zo | so, thus, in this way +door | through by +over | over, across +ze | she, her, they, them +zich | oneself +bij | (1) a bee, (2) by, near, at +ook | also, too +tot | till, until +je | you +mij | me +uit | out of, from +der | Old Dutch form of 'van der' still found in surnames +daar | (1) there, (2) because +haar | (1) her, their, them, (2) hair +naar | (1) unpleasant, unwell etc, (2) towards, (3) as +heb | present first person sing. of 'to have' +hoe | how, why +heeft | present third person sing. of 'to have' +hebben | 'to have' and various parts thereof +deze | this +u | you +want | (1) for, (2) mitten, (3) rigging +nog | yet, still +zal | 'shall', first and third person sing. of verb 'zullen' (will) +me | me +zij | she, they +nu | now +ge | 'thou', still used in Belgium and south Netherlands +geen | none +omdat | because +iets | something, somewhat +worden | to become, grow, get +toch | yet, still +al | all, every, each +waren | (1) 'were' (2) to wander, (3) wares, (3) +veel | much, many +meer | (1) more, (2) lake +doen | to do, to make +toen | then, when +moet | noun 'spot/mote' and present form of 'to must' +ben | (1) am, (2) 'are' in interrogative second person singular of 'to be' +zonder | without +kan | noun 'can' and present form of 'to be able' +hun | their, them +dus | so, consequently +alles | all, everything, anything +onder | under, beneath +ja | yes, of course +eens | once, one day +hier | here +wie | who +werd | imperfect third person sing. of 'become' +altijd | always +doch | yet, but etc +wordt | present third person sing. of 'become' +wezen | (1) to be, (2) 'been' as in 'been fishing', (3) orphans +kunnen | to be able +ons | us/our +zelf | self +tegen | against, towards, at +na | after, near +reeds | already +wil | (1) present tense of 'want', (2) 'will', noun, (3) fender +kon | could; past tense of 'to be able' +niets | nothing +uw | your +iemand | somebody +geweest | been; past participle of 'be' +andere | other +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(DutchStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/no/analyzer_no.go b/analysis/lang/no/analyzer_no.go new file mode 100644 index 0000000..64bf71c --- /dev/null +++ b/analysis/lang/no/analyzer_no.go @@ -0,0 +1,60 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package no + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +) + +const AnalyzerName = "no" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + unicodeTokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopNoFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + stemmerNoFilter, err := cache.TokenFilterNamed(SnowballStemmerName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: unicodeTokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + stopNoFilter, + stemmerNoFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/no/analyzer_no_test.go b/analysis/lang/no/analyzer_no_test.go new file mode 100644 index 0000000..b37cb4d --- /dev/null +++ b/analysis/lang/no/analyzer_no_test.go @@ -0,0 +1,70 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package no + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestNorwegianAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // stemming + { + input: []byte("havnedistriktene"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("havnedistrikt"), + }, + }, + }, + { + input: []byte("havnedistrikter"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("havnedistrikt"), + }, + }, + }, + // stop word + { + input: []byte("det"), + output: analysis.TokenStream{}, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if len(actual) != len(test.output) { + t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) + } + for i, tok := range actual { + if !reflect.DeepEqual(tok.Term, test.output[i].Term) { + t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) + } + } + } +} diff --git a/analysis/lang/no/stemmer_no.go b/analysis/lang/no/stemmer_no.go new file mode 100644 index 0000000..97e06d4 --- /dev/null +++ b/analysis/lang/no/stemmer_no.go @@ -0,0 +1,52 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package no + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/snowballstem" + "github.com/blevesearch/snowballstem/norwegian" +) + +const SnowballStemmerName = "stemmer_no_snowball" + +type NorwegianStemmerFilter struct { +} + +func NewNorwegianStemmerFilter() *NorwegianStemmerFilter { + return &NorwegianStemmerFilter{} +} + +func (s *NorwegianStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + env := snowballstem.NewEnv(string(token.Term)) + norwegian.Stem(env) + token.Term = []byte(env.Current()) + } + return input +} + +func NorwegianStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewNorwegianStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(SnowballStemmerName, NorwegianStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/no/stop_filter_no.go b/analysis/lang/no/stop_filter_no.go new file mode 100644 index 0000000..a698c62 --- /dev/null +++ b/analysis/lang/no/stop_filter_no.go @@ -0,0 +1,36 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package no + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/no/stop_words_no.go b/analysis/lang/no/stop_words_no.go new file mode 100644 index 0000000..4b58ab9 --- /dev/null +++ b/analysis/lang/no/stop_words_no.go @@ -0,0 +1,221 @@ +package no + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_no" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ +// ` was changed to ' to allow for literal string + +var NorwegianStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/norwegian/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A Norwegian stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This stop word list is for the dominant bokmål dialect. Words unique + | to nynorsk are marked *. + + | Revised by Jan Bruusgaard , Jan 2005 + +og | and +i | in +jeg | I +det | it/this/that +at | to (w. inf.) +en | a/an +et | a/an +den | it/this/that +til | to +er | is/am/are +som | who/that +på | on +de | they / you(formal) +med | with +han | he +av | of +ikke | not +ikkje | not * +der | there +så | so +var | was/were +meg | me +seg | you +men | but +ett | one +har | have +om | about +vi | we +min | my +mitt | my +ha | have +hadde | had +hun | she +nå | now +over | over +da | when/as +ved | by/know +fra | from +du | you +ut | out +sin | your +dem | them +oss | us +opp | up +man | you/one +kan | can +hans | his +hvor | where +eller | or +hva | what +skal | shall/must +selv | self (reflective) +sjøl | self (reflective) +her | here +alle | all +vil | will +bli | become +ble | became +blei | became * +blitt | have become +kunne | could +inn | in +når | when +være | be +kom | come +noen | some +noe | some +ville | would +dere | you +som | who/which/that +deres | their/theirs +kun | only/just +ja | yes +etter | after +ned | down +skulle | should +denne | this +for | for/because +deg | you +si | hers/his +sine | hers/his +sitt | hers/his +mot | against +å | to +meget | much +hvorfor | why +dette | this +disse | these/those +uten | without +hvordan | how +ingen | none +din | your +ditt | your +blir | become +samme | same +hvilken | which +hvilke | which (plural) +sånn | such a +inni | inside/within +mellom | between +vår | our +hver | each +hvem | who +vors | us/ours +hvis | whose +både | both +bare | only/just +enn | than +fordi | as/because +før | before +mange | many +også | also +slik | just +vært | been +være | to be +båe | both * +begge | both +siden | since +dykk | your * +dykkar | yours * +dei | they * +deira | them * +deires | theirs * +deim | them * +di | your (fem.) * +då | as/when * +eg | I * +ein | a/an * +eit | a/an * +eitt | a/an * +elles | or * +honom | he * +hjå | at * +ho | she * +hoe | she * +henne | her +hennar | her/hers +hennes | hers +hoss | how * +hossen | how * +ikkje | not * +ingi | noone * +inkje | noone * +korleis | how * +korso | how * +kva | what/which * +kvar | where * +kvarhelst | where * +kven | who/whom * +kvi | why * +kvifor | why * +me | we * +medan | while * +mi | my * +mine | my * +mykje | much * +no | now * +nokon | some (masc./neut.) * +noka | some (fem.) * +nokor | some * +noko | some * +nokre | some * +si | his/hers * +sia | since * +sidan | since * +so | so * +somt | some * +somme | some * +um | about* +upp | up * +vere | be * +vore | was * +verte | become * +vort | become * +varte | became * +vart | became * + +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(NorwegianStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/pl/analyzer_pl.go b/analysis/lang/pl/analyzer_pl.go new file mode 100644 index 0000000..0e202e8 --- /dev/null +++ b/analysis/lang/pl/analyzer_pl.go @@ -0,0 +1,60 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pl + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +) + +const AnalyzerName = "pl" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + tokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopPlFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + stemmerPlFilter, err := cache.TokenFilterNamed(SnowballStemmerName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: tokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + stopPlFilter, + stemmerPlFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/pl/analyzer_pl_test.go b/analysis/lang/pl/analyzer_pl_test.go new file mode 100644 index 0000000..073a28f --- /dev/null +++ b/analysis/lang/pl/analyzer_pl_test.go @@ -0,0 +1,149 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pl + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestPolishAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // stemming + { + input: []byte("śmiało"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("śmieć"), + }, + }, + }, + { + input: []byte("przypadku"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("przypadek"), + }, + }, + }, + // stop word + { + input: []byte("według"), + output: analysis.TokenStream{}, + }, + // digits safe + { + input: []byte("text 1000"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("text"), + }, + &analysis.Token{ + Term: []byte("1000"), + }, + }, + }, + { + input: []byte("badawczego było opracowanie kompendium które przystępny sposób prezentowało niespecjalistom zakresu kryptografii kwantowej wykorzystanie technik kwantowych do bezpiecznego przesyłu przetwarzania informacji"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("badawczy"), + }, + &analysis.Token{ + Term: []byte("opracować"), + }, + &analysis.Token{ + Term: []byte("kompendium"), + }, + &analysis.Token{ + Term: []byte("przystyć"), + }, + &analysis.Token{ + Term: []byte("prezentować"), + }, + &analysis.Token{ + Term: []byte("niespecjalista"), + }, + &analysis.Token{ + Term: []byte("zakres"), + }, + &analysis.Token{ + Term: []byte("kryptografia"), + }, + &analysis.Token{ + Term: []byte("kwantowy"), + }, + &analysis.Token{ + Term: []byte("wykorzyseć"), + }, + &analysis.Token{ + Term: []byte("technika"), + }, + &analysis.Token{ + Term: []byte("kwantowy"), + }, + &analysis.Token{ + Term: []byte("bezpieczny"), + }, + &analysis.Token{ + Term: []byte("przesył"), + }, + &analysis.Token{ + Term: []byte("przetwarzać"), + }, + &analysis.Token{ + Term: []byte("informacja"), + }, + }, + }, + { + input: []byte("Ale ta wiedza była utrzymywana w tajemnicy"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("wiedza"), + }, + &analysis.Token{ + Term: []byte("utrzymywać"), + }, + &analysis.Token{ + Term: []byte("tajemnik"), + }, + }, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if len(actual) != len(test.output) { + t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) + } + for i, tok := range actual { + if !reflect.DeepEqual(tok.Term, test.output[i].Term) { + t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) + } + } + } +} diff --git a/analysis/lang/pl/stemmer_pl.go b/analysis/lang/pl/stemmer_pl.go new file mode 100644 index 0000000..c997fb1 --- /dev/null +++ b/analysis/lang/pl/stemmer_pl.go @@ -0,0 +1,58 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pl + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/lang/pl/stempel" + "github.com/blevesearch/bleve/v2/registry" +) + +const SnowballStemmerName = "stemmer_pl" + +type PolishStemmerFilter struct { + trie stempel.Trie +} + +func NewPolishStemmerFilter() (*PolishStemmerFilter, error) { + trie, err := stempel.LoadTrie() + if err != nil { + return nil, err + } + return &PolishStemmerFilter{ + trie: trie, + }, nil +} + +func (s *PolishStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + buff := []rune(string(token.Term)) + diff := s.trie.GetLastOnPath(buff) + buff = stempel.Diff(buff, diff) + token.Term = []byte(string(buff)) + } + return input +} + +func PolishStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewPolishStemmerFilter() +} + +func init() { + err := registry.RegisterTokenFilter(SnowballStemmerName, PolishStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/pl/stemmer_pl_test.go b/analysis/lang/pl/stemmer_pl_test.go new file mode 100644 index 0000000..dcfed34 --- /dev/null +++ b/analysis/lang/pl/stemmer_pl_test.go @@ -0,0 +1,67 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pl + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestPolishStemmer(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("utrzymywana"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("utrzymywać"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("tajemnicy"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("tajemnik"), + }, + }, + }, + } + + cache := registry.NewCache() + filter, err := cache.TokenFilterNamed(SnowballStemmerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := filter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/pl/stempel/LICENSE b/analysis/lang/pl/stempel/LICENSE new file mode 100644 index 0000000..7a4a3ea --- /dev/null +++ b/analysis/lang/pl/stempel/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/analysis/lang/pl/stempel/cell.go b/analysis/lang/pl/stempel/cell.go new file mode 100644 index 0000000..b090571 --- /dev/null +++ b/analysis/lang/pl/stempel/cell.go @@ -0,0 +1,53 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stempel + +import ( + "fmt" + + "github.com/blevesearch/stempel/javadata" +) + +type cell struct { + ref int32 + cmd int32 +} + +func (c *cell) String() string { + return fmt.Sprintf("ref(%d) cmd(%d)", c.ref, c.cmd) +} + +func newCell(r *javadata.Reader) (*cell, error) { + cmd, err := r.ReadInt32() + if err != nil { + return nil, fmt.Errorf("error reading cell cmd: %v", err) + } + _, err = r.ReadInt32() + if err != nil { + return nil, fmt.Errorf("error reading cell cnt: %v", err) + } + ref, err := r.ReadInt32() + if err != nil { + return nil, fmt.Errorf("error reading cell ref: %v", err) + } + _, err = r.ReadInt32() + if err != nil { + return nil, fmt.Errorf("error reading cell skip: %v", err) + } + return &cell{ + cmd: cmd, + ref: ref, + }, nil +} diff --git a/analysis/lang/pl/stempel/diff.go b/analysis/lang/pl/stempel/diff.go new file mode 100644 index 0000000..5b1ce25 --- /dev/null +++ b/analysis/lang/pl/stempel/diff.go @@ -0,0 +1,64 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stempel + +// Diff transforms the dest rune slice following the rules described +// in the diff command rune slice. +func Diff(dest, diff []rune) []rune { + if len(diff) == 0 { + return dest + } + + pos := len(dest) - 1 + if pos < 0 { + return dest + } + + for i := 0; i < len(diff)/2; i++ { + cmd := diff[2*i] + param := diff[2*i+1] + parNum := int(param - 'a' + 1) + switch cmd { + case '-': + pos = pos - parNum + 1 + case 'R': + if pos < 0 || pos >= len(dest) { + // out of bounds, just return + return dest + } + dest[pos] = param + case 'D': + o := pos + pos -= parNum - 1 + if pos < 0 || pos >= len(dest) { + // out of bounds, just return + return dest + } + dest = append(dest[:pos], dest[o+1:]...) + case 'I': + pos++ + if pos < 0 || pos > len(dest) { + // out of bounds, just return + return dest + } + + dest = append(dest, 0) + copy(dest[pos+1:], dest[pos:]) + dest[pos] = param + } + pos-- + } + return dest +} diff --git a/analysis/lang/pl/stempel/diff_test.go b/analysis/lang/pl/stempel/diff_test.go new file mode 100644 index 0000000..9812202 --- /dev/null +++ b/analysis/lang/pl/stempel/diff_test.go @@ -0,0 +1,144 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stempel + +import ( + "fmt" + "reflect" + "testing" +) + +func TestDiff(t *testing.T) { + tests := []struct { + in []rune + cmd []rune + out []rune + }{ + // test delete, this command deletes N chars backwards from the current pos + // the current pos starts at the end of the string + // if you try to delete a negative number of chars or more chars than there + // are, you will get the buffer at that time + { + in: []rune{'h', 'e', 'l', 'l', 'o'}, + // delete 1 + cmd: []rune{'D', 'a'}, + out: []rune{'h', 'e', 'l', 'l'}, + }, + { + in: []rune{'h', 'e', 'l', 'l', 'o'}, + // delete 2 + cmd: []rune{'D', 'a' + 1}, + out: []rune{'h', 'e', 'l'}, + }, + { + in: []rune{'h', 'e', 'l', 'l', 'o'}, + // delete 3 + cmd: []rune{'D', 'a' + 2}, + out: []rune{'h', 'e'}, + }, + { + in: []rune{'h', 'e', 'l', 'l', 'o'}, + // delete 4 + cmd: []rune{'D', 'a' + 3}, + out: []rune{'h'}, + }, + { + in: []rune{'h', 'e', 'l', 'l', 'o'}, + // delete 5 + cmd: []rune{'D', 'a' + 4}, + out: []rune{}, + }, + { + in: []rune{'h', 'e', 'l', 'l', 'o'}, + // delete 6 (invalid, return buffer at that point) + cmd: []rune{'D', 'a' + 5}, + out: []rune{'h', 'e', 'l', 'l', 'o'}, + }, + { + in: []rune{'h', 'e', 'l', 'l', 'o'}, + // delete -1 + cmd: []rune{'D', 'a' - 1}, + out: []rune{'h', 'e', 'l', 'l', 'o'}, + }, + // delete one char twice + { + in: []rune{'h', 'e', 'l', 'l', 'o'}, + // delete 1, delete 1 + cmd: []rune{'D', 'a', 'D', 'a'}, + out: []rune{'h', 'e', 'l'}, + }, + // test insert + { + in: []rune{'h', 'e', 'l', 'l', 'o'}, + // insert 'p' + cmd: []rune{'I', 'p'}, + out: []rune{'h', 'e', 'l', 'l', 'o', 'p'}, + }, + // insert twice + { + in: []rune{'h'}, + // insert 'l', insert 'e' + // NOTE how the cursor moves backwards, so we have to insert in reverse + cmd: []rune{'I', 'l', 'I', 'e'}, + out: []rune{'h', 'e', 'l'}, + }, + // test replace + { + in: []rune{'h', 'e', 'l', 'l', 'o'}, + // replace with 'y' + cmd: []rune{'R', 'y'}, + out: []rune{'h', 'e', 'l', 'l', 'y'}, + }, + // test replace again + { + in: []rune{'h', 'e', 'l', 'l', 'o'}, + // replace with 'y', then replace with 'x' + // NOTE how the cursor moves backwards as we replace + cmd: []rune{'R', 'y', 'R', 'x'}, + out: []rune{'h', 'e', 'l', 'x', 'y'}, + }, + // test skip, then replace + { + in: []rune{'h', 'e', 'l', 'l', 'o'}, + // skip 1, then replace with 'y' + cmd: []rune{'-', 'a', 'R', 'y'}, + out: []rune{'h', 'e', 'l', 'y', 'o'}, + }, + // test skip 2, then replace + { + in: []rune{'h', 'e', 'l', 'l', 'o'}, + // skip 1, then replace with 'y' + cmd: []rune{'-', 'a' + 1, 'R', 'y'}, + out: []rune{'h', 'e', 'y', 'l', 'o'}, + }, + // test skip 2, then replace + { + in: []rune{'h', 'e', 'l', 'l', 'o'}, + // skip 5 (too far), then replace with 'y' + // get original + cmd: []rune{'-', 'a' + 4, 'R', 'y'}, + out: []rune{'h', 'e', 'l', 'l', 'o'}, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%s-'%s'", string(test.in), string(test.cmd)), func(t *testing.T) { + got := Diff(test.in, test.cmd) + if !reflect.DeepEqual(test.out, got) { + t.Errorf("expected %v, got %v", test.out, got) + } + }) + } +} diff --git a/analysis/lang/pl/stempel/file.go b/analysis/lang/pl/stempel/file.go new file mode 100644 index 0000000..dbbf798 --- /dev/null +++ b/analysis/lang/pl/stempel/file.go @@ -0,0 +1,71 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stempel + +import ( + "bytes" + _ "embed" + "github.com/blevesearch/stempel/javadata" + "io" + "os" + "strings" +) + +//go:embed pl/stemmer_20000.tbl +var stempelFile []byte + +// Trie is the external interface to work with the stempel trie +type Trie interface { + GetLastOnPath([]rune) []rune +} + +// Open attempts to open a file at the specified path, and use it to +// build a Trie +func Open(path string) (Trie, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + + return buildTrieFromReader(f) +} + +// LoadTrie load trie from embed file +func LoadTrie() (Trie, error) { + return buildTrieFromReader(bytes.NewReader(stempelFile)) +} + +// buildTrieFromReader build trie from io.Reader +func buildTrieFromReader(f io.Reader) (Trie, error) { + r := javadata.NewReader(f) + method, err := r.ReadUTF() + if err != nil { + return nil, err + } + + var rv Trie + if strings.Contains(method, "M") { + rv, err = newMultiTrie(r) + if err != nil { + return nil, err + } + } else { + rv, err = newTrie(r) + if err != nil { + return nil, err + } + } + return rv, nil +} diff --git a/analysis/lang/pl/stempel/file_test.go b/analysis/lang/pl/stempel/file_test.go new file mode 100644 index 0000000..c078e8d --- /dev/null +++ b/analysis/lang/pl/stempel/file_test.go @@ -0,0 +1,91 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stempel + +import ( + "bufio" + "compress/gzip" + "os" + "strings" + "testing" + + "golang.org/x/text/encoding/charmap" +) + +func TestEmpty(t *testing.T) { + trie, err := Open("pl/stemmer_20000.tbl") + if err != nil { + t.Fatal(err) + } + + buff := []rune("") + diff := trie.GetLastOnPath(buff) + if len(diff) > 0 { + t.Fatalf("expected empty diff, got %v", diff) + } + buff = Diff(buff, diff) + if len(buff) > 0 { + t.Fatalf("expected empty buff, got %v", buff) + } +} + +// TestStem only tests that we can successfully stem everything in the +// dictionary without crashing. It does not attempt to assert correct output. +func TestStem(t *testing.T) { + trie, err := Open("pl/stemmer_20000.tbl") + if err != nil { + t.Fatal(err) + } + + wordFileGz, err := os.Open("pl/pl_PL.dic.gz") + if err != nil { + t.Fatal(err) + } + defer func() { + cerr := wordFileGz.Close() + if cerr != nil { + t.Fatal(cerr) + } + }() + + wordFile, err := gzip.NewReader(wordFileGz) + if err != nil { + t.Fatal(err) + } + defer func() { + cerr := wordFile.Close() + if cerr != nil { + t.Fatal(cerr) + } + }() + + cr := charmap.ISO8859_2.NewDecoder().Reader(wordFile) + + scanner := bufio.NewScanner(cr) + for scanner.Scan() { + before := scanner.Text() + hasSlash := strings.Index(before, "/") + if hasSlash > 0 { + before = before[0:hasSlash] + } + buff := []rune(before) + diff := trie.GetLastOnPath(buff) + _ = Diff(buff, diff) + } + + if err := scanner.Err(); err != nil { + t.Fatal(err) + } +} diff --git a/analysis/lang/pl/stempel/fuzz.go b/analysis/lang/pl/stempel/fuzz.go new file mode 100644 index 0000000..d9721bb --- /dev/null +++ b/analysis/lang/pl/stempel/fuzz.go @@ -0,0 +1,35 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build gofuzz +// +build gofuzz + +package stempel + +var fuzzTrie Trie + +func init() { + var err error + fuzzTrie, err = Open("pl/stemmer_20000.tbl") + if err != nil { + panic(err) + } +} + +func Fuzz(data []byte) int { + inRunes := []rune(string(data)) + diff := fuzzTrie.GetLastOnPath(inRunes) + _ = Diff(inRunes, diff) + return 1 +} diff --git a/analysis/lang/pl/stempel/javadata/README.md b/analysis/lang/pl/stempel/javadata/README.md new file mode 100644 index 0000000..7c40b7b --- /dev/null +++ b/analysis/lang/pl/stempel/javadata/README.md @@ -0,0 +1,3 @@ +# javadata + +Go library to read data written with java.io.DataOutput diff --git a/analysis/lang/pl/stempel/javadata/fuzz.go b/analysis/lang/pl/stempel/javadata/fuzz.go new file mode 100644 index 0000000..5ad4c21 --- /dev/null +++ b/analysis/lang/pl/stempel/javadata/fuzz.go @@ -0,0 +1,34 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build gofuzz +// +build gofuzz + +package javadata + +import "bytes" + +func Fuzz(data []byte) int { + br := bytes.NewReader(data) + jdr := NewReader(br) + + var err error + for err == nil { + _, err = jdr.ReadUTF() + } + if err != nil { + return 0 + } + return 1 +} diff --git a/analysis/lang/pl/stempel/javadata/input.go b/analysis/lang/pl/stempel/javadata/input.go new file mode 100644 index 0000000..0da4d4e --- /dev/null +++ b/analysis/lang/pl/stempel/javadata/input.go @@ -0,0 +1,135 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package javadata + +import ( + "bufio" + "encoding/binary" + "fmt" + "io" +) + +// ErrMalformedInput returned when malformed input is encountered +var ErrMalformedInput = fmt.Errorf("malformed input") + +// Reader knows how to read java serialized data +type Reader struct { + r *bufio.Reader +} + +// NewReader creates a new java data input reader +func NewReader(r io.Reader) *Reader { + return &Reader{r: bufio.NewReader(r)} +} + +// ReadBool attempts to reads a bool from the stream +func (r *Reader) ReadBool() (bool, error) { + b, err := r.r.ReadByte() + if err != nil { + return false, err + } + return b != 0, nil +} + +// ReadInt32 attempts to reads a signed 32-bit integer from the stream +func (r *Reader) ReadInt32() (rv int32, err error) { + err = binary.Read(r.r, binary.BigEndian, &rv) + return +} + +// ReadUint16 attempts to reads a unsigned 16-bit integer from the stream +func (r *Reader) ReadUint16() (rv uint16, err error) { + err = binary.Read(r.r, binary.BigEndian, &rv) + return +} + +// ReadCharAsRune attempts to read a java two byte char and return it as a rune +func (r *Reader) ReadCharAsRune() (rv rune, err error) { + var char uint16 + err = binary.Read(r.r, binary.BigEndian, &char) + rv = rune(char) + return +} + +// ReadUTF attempts to reads a UTF-encoded string from the stream +// this method follows the specific alternate encoding desribed here: +// https://docs.oracle.com/javase/7/docs/api/java/io/DataInput.html +func (r *Reader) ReadUTF() (string, error) { + utfLen, err := r.ReadUint16() + if err != nil { + return "", err + } + bytes := make([]byte, utfLen) + runes := make([]rune, utfLen) + _, err = io.ReadFull(r.r, bytes) + if err != nil { + return "", err + } + + var count uint16 + var runeCount uint16 + + // handle simple case of all ascii + for count < utfLen { + c := bytes[count] + if bytes[count] > 127 { + break + } + count++ + runes[runeCount] = rune(c) + runeCount++ + } + + // handle rest + for count < utfLen { + c := bytes[count] + switch bytes[count] >> 4 { + case 0, 1, 2, 3, 4, 5, 6, 7, 8: + /* 0xxxxxxx*/ + count++ + runes[runeCount] = rune(c) + runeCount++ + case 12, 13: + /* 110x xxxx 10xx xxxx*/ + count += 2 + if count > utfLen { + return "", ErrMalformedInput + } + char2 := rune(bytes[count-1]) + if (char2 & 0xC0) != 0x80 { + return "", ErrMalformedInput + } + runes[runeCount] = (rune(c)&0x1F)<<6 | char2&0x3F + runeCount++ + case 14: + /* 1110 xxxx 10xx xxxx 10xx xxxx */ + count += 3 + if count > utfLen { + return "", ErrMalformedInput + } + char2 := rune(bytes[count-2]) + char3 := rune(bytes[count-1]) + if ((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80) { + return "", ErrMalformedInput + } + runes[runeCount] = ((rune(c)&0x0F)<<12 | (char2&0x3F)<<6 | (char3&0x3F)<<0) + runeCount++ + default: + /* 10xx xxxx, 1111 xxxx */ + return "", ErrMalformedInput + } + } + return string(runes[0:runeCount]), nil +} diff --git a/analysis/lang/pl/stempel/javadata/input_test.go b/analysis/lang/pl/stempel/javadata/input_test.go new file mode 100644 index 0000000..b4458c6 --- /dev/null +++ b/analysis/lang/pl/stempel/javadata/input_test.go @@ -0,0 +1,249 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package javadata + +import ( + "bytes" + "io" + "testing" +) + +func TestReadBool(t *testing.T) { + + tests := []struct { + in []byte + out bool + err error + }{ + { + in: []byte{0}, + out: false, + }, + { + in: []byte{1}, + out: true, + }, + { + in: []byte{27}, + out: true, + }, + { + in: []byte{}, + err: io.EOF, + }, + } + + for _, test := range tests { + t.Run(string(test.in), func(t *testing.T) { + sr := bytes.NewReader(test.in) + dr := NewReader(sr) + actual, err := dr.ReadBool() + if err != test.err { + t.Error(err) + } + if actual != test.out { + t.Errorf("expected %t, got %t", test.out, actual) + } + }) + } +} + +func TestReadUint16(t *testing.T) { + + tests := []struct { + in []byte + out uint16 + err error + }{ + { + in: []byte{0, 0}, + out: 0, + }, + { + in: []byte{0, 1}, + out: 1, + }, + { + in: []byte{1, 0}, + out: 256, + }, + { + in: []byte{}, + err: io.EOF, + }, + } + + for _, test := range tests { + t.Run(string(test.in), func(t *testing.T) { + sr := bytes.NewReader(test.in) + dr := NewReader(sr) + actual, err := dr.ReadUint16() + if err != test.err { + t.Error(err) + } + if actual != test.out { + t.Errorf("expected %d, got %d", test.out, actual) + } + }) + } +} + +func TestReadInt32(t *testing.T) { + + tests := []struct { + in []byte + out int32 + err error + }{ + { + in: []byte{0, 0, 0, 0}, + out: 0, + }, + { + in: []byte{0, 0, 0, 1}, + out: 1, + }, + { + in: []byte{0, 0, 1, 0}, + out: 256, + }, + { + in: []byte{0, 1, 0, 0}, + out: 65536, + }, + { + in: []byte{}, + err: io.EOF, + }, + } + + for _, test := range tests { + t.Run(string(test.in), func(t *testing.T) { + sr := bytes.NewReader(test.in) + dr := NewReader(sr) + actual, err := dr.ReadInt32() + if err != test.err { + t.Error(err) + } + if actual != test.out { + t.Errorf("expected %d, got %d", test.out, actual) + } + }) + } +} + +func TestReadUTF(t *testing.T) { + + tests := []struct { + in []byte + out string + err error + }{ + { + in: []byte{0, 3, 'c', 'a', 't'}, + out: "cat", + }, + { + in: []byte{0, 2, 0xc2, 0xa3}, + out: "£", + }, + { + in: []byte{0, 3, 0xe3, 0x85, 0x85}, + out: "ㅅ", + }, + { + in: []byte{0, 6, 0xe3, 0x85, 0x85, 'c', 'a', 't'}, + out: "ㅅcat", + }, + { + in: []byte{}, + err: io.EOF, + }, + { + in: []byte{0, 3}, + err: io.EOF, + }, + { + in: []byte{0, 1, 0xc2}, + err: ErrMalformedInput, + }, + { + in: []byte{0, 2, 0xc2, 0xc3}, + err: ErrMalformedInput, + }, + { + in: []byte{0, 2, 0xe3, 0x85}, + err: ErrMalformedInput, + }, + { + in: []byte{0, 3, 0xe3, 0xc5, 0x85}, + err: ErrMalformedInput, + }, + { + in: []byte{0, 1, 0xff}, + err: ErrMalformedInput, + }, + { + in: []byte{0x0, 0x05, 0x44, 0x61, 0x52, 0xc4, 0x87}, + out: "DaRć", + }, + } + + for _, test := range tests { + t.Run(string(test.in), func(t *testing.T) { + sr := bytes.NewReader(test.in) + dr := NewReader(sr) + actual, err := dr.ReadUTF() + if err != test.err { + t.Error(err) + } + if actual != test.out { + t.Errorf("expected %s, got %s", test.out, actual) + } + }) + } + +} + +// func TestFile(t *testing.T) { +// f, err := os.Open("stemmer_20000.tbl") +// if err != nil { +// t.Fatal(err) +// } +// r := NewReader(f) +// reversed, err := r.ReadBool() +// if err != nil { +// t.Fatal(err) +// } +// log.Printf("reversed: %t", reversed) +// root, err := r.ReadInt32() +// if err != nil { +// t.Fatal(err) +// } +// log.Printf("root: %d", root) +// n, err := r.ReadInt32() +// if err != nil { +// t.Fatal(err) +// } +// log.Printf("n is %d", n) +// // for n > 0 { +// // utf, err := r.ReadUTF() +// // if err != nil { +// // t.Error(err) +// // } +// // log.Printf("read: %s", utf) +// // n-- +// // } +// } diff --git a/analysis/lang/pl/stempel/multi_trie.go b/analysis/lang/pl/stempel/multi_trie.go new file mode 100644 index 0000000..32283ff --- /dev/null +++ b/analysis/lang/pl/stempel/multi_trie.go @@ -0,0 +1,140 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stempel + +import ( + "fmt" + + "github.com/blevesearch/stempel/javadata" +) + +// multiTrie represents a trie of tries. When using the multiTrie, each trie +// is consulted consecutively to find commands to perform on the input. Thus +// a multiTrie with seven tries might have up to seven groups of commands to +// perform on the input. +type multiTrie struct { + tries []*trie + by int32 + forward bool +} + +func newMultiTrie(r *javadata.Reader) (rv *multiTrie, err error) { + rv = &multiTrie{} + rv.forward, err = r.ReadBool() + if err != nil { + return nil, err + } + rv.by, err = r.ReadInt32() + if err != nil { + return nil, err + } + nTries, err := r.ReadInt32() + if err != nil { + return nil, err + } + for nTries > 0 { + trie, err := newTrie(r) + if err != nil { + return nil, err + } + rv.tries = append(rv.tries, trie) + nTries-- + } + return rv, nil +} + +const eom = rune('*') + +func (t *multiTrie) GetLastOnPath(key []rune) []rune { + var rv []rune + lastKey := key + p := make([][]rune, len(t.tries)) + lastR := ' ' + for i := 0; i < len(t.tries); i++ { + r := t.tries[i].GetLastOnPath(lastKey) + if len(r) == 0 || len(r) == 1 && r[0] == eom { + return rv + } + if cannotFollow(lastR, r[0]) { + return rv + } + lastR = r[len(r)-2] + p[i] = r + if p[i][0] == '-' { + if i > 0 { + var err error + key, err = t.skip(key, lengthPP(p[i-1])) + if err != nil { + return rv + } + } + var err error + key, err = t.skip(key, lengthPP(p[i])) + if err != nil { + return rv + } + } + rv = append(rv, r...) + if len(key) != 0 { + lastKey = key + } + } + return rv +} + +func cannotFollow(after, goes rune) bool { + switch after { + case '-', 'D': + return after == goes + } + return false +} + +var errIndexOutOfBounds = fmt.Errorf("index out of bounds") + +func (t *multiTrie) skip(in []rune, count int) ([]rune, error) { + if count > len(in) { + return nil, errIndexOutOfBounds + } + if t.forward { + return in[count:], nil + } + return in[0 : len(in)-count], nil +} + +func lengthPP(cmd []rune) int { + rv := 0 + for i := 0; i < len(cmd); i++ { + switch cmd[i] { + case '-', 'D': + i++ + rv += int(cmd[i] - rune('a') + 1) + case 'R': + i++ + rv++ + fallthrough + case 'I': + } + } + return rv +} + +func (t *multiTrie) String() string { + rv := "" + for i, trie := range t.tries { + rv += fmt.Sprintf("trie %d\n\n %v\n--------\n", i, trie) + } + return rv +} diff --git a/analysis/lang/pl/stempel/row.go b/analysis/lang/pl/stempel/row.go new file mode 100644 index 0000000..04576b4 --- /dev/null +++ b/analysis/lang/pl/stempel/row.go @@ -0,0 +1,80 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stempel + +import ( + "fmt" + + "github.com/blevesearch/stempel/javadata" +) + +type row struct { + cells map[rune]*cell +} + +func (r *row) String() string { + rv := "" + for k, v := range r.cells { + rv += fmt.Sprintf("[%s:%v]\n", string(k), v) + } + return rv +} + +func newRow(r *javadata.Reader) (*row, error) { + rv := &row{ + cells: make(map[rune]*cell), + } + + nCells, err := r.ReadInt32() + if err != nil { + return nil, fmt.Errorf("error reading num cells: %v", err) + } + + for nCells > 0 { + + c, err := r.ReadCharAsRune() + if err != nil { + return nil, fmt.Errorf("error reading cell char: %v", err) + } + cell, err := newCell(r) + if err != nil { + return nil, fmt.Errorf("error reading cell: %v", err) + } + + rv.cells[c] = cell + nCells-- + } + return rv, nil +} + +func (r *row) getCmd(way rune) int32 { + c := r.at(way) + if c != nil { + return c.cmd + } + return -1 +} + +func (r *row) getRef(way rune) int32 { + c := r.at(way) + if c != nil { + return c.ref + } + return -1 +} + +func (r *row) at(c rune) *cell { + return r.cells[c] +} diff --git a/analysis/lang/pl/stempel/strenum.go b/analysis/lang/pl/stempel/strenum.go new file mode 100644 index 0000000..5e00fe9 --- /dev/null +++ b/analysis/lang/pl/stempel/strenum.go @@ -0,0 +1,48 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stempel + +import ( + "io" +) + +type strEnum struct { + r []rune + from int + by int +} + +func newStrEnum(s []rune, up bool) *strEnum { + rv := &strEnum{ + r: s, + } + if up { + rv.from = 0 + rv.by = 1 + } else { + rv.from = len(s) - 1 + rv.by = -1 + } + return rv +} + +func (s *strEnum) next() (rune, error) { + if s.from < 0 || s.from >= len(s.r) { + return 0, io.EOF + } + rv := s.r[s.from] + s.from += s.by + return rv, nil +} diff --git a/analysis/lang/pl/stempel/strenum_test.go b/analysis/lang/pl/stempel/strenum_test.go new file mode 100644 index 0000000..2d9b03d --- /dev/null +++ b/analysis/lang/pl/stempel/strenum_test.go @@ -0,0 +1,61 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stempel + +import ( + "fmt" + "io" + "reflect" + "testing" +) + +func TestStrenumNext(t *testing.T) { + + tests := []struct { + in []rune + up bool + expect []rune + }{ + { + in: []rune{'h', 'e', 'l', 'l', 'o'}, + up: true, + expect: []rune{'h', 'e', 'l', 'l', 'o'}, + }, + { + in: []rune{'h', 'e', 'l', 'l', 'o'}, + up: false, + expect: []rune{'o', 'l', 'l', 'e', 'h'}, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%s-up-%t", string(test.in), test.up), func(t *testing.T) { + strenum := newStrEnum(test.in, test.up) + var got []rune + next, err := strenum.next() + for err == nil { + got = append(got, next) + next, err = strenum.next() + } + if err != io.EOF { + t.Errorf("next got err: %v", err) + } + if !reflect.DeepEqual(got, test.expect) { + t.Errorf("expected %v, got %v", test.expect, got) + } + }) + } + +} diff --git a/analysis/lang/pl/stempel/trie.go b/analysis/lang/pl/stempel/trie.go new file mode 100644 index 0000000..df423bc --- /dev/null +++ b/analysis/lang/pl/stempel/trie.go @@ -0,0 +1,132 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stempel + +import ( + "fmt" + + "github.com/blevesearch/stempel/javadata" +) + +// trie represents the internal trie structure +type trie struct { + rows []*row + cmds []string + root int32 + forward bool +} + +func newTrie(r *javadata.Reader) (rv *trie, err error) { + rv = &trie{} + rv.forward, err = r.ReadBool() + if err != nil { + return nil, fmt.Errorf("error reading trie forward: %v", err) + } + rv.root, err = r.ReadInt32() + if err != nil { + return nil, fmt.Errorf("error reading trie root: %v", err) + } + + // commands + nCommands, err := r.ReadInt32() + if err != nil { + return nil, fmt.Errorf("error reading trie num commands: %v", err) + } + for nCommands > 0 { + utfCommand, nerr := r.ReadUTF() + if nerr != nil { + return nil, fmt.Errorf("error reading trie command utf: %v", nerr) + } + rv.cmds = append(rv.cmds, utfCommand) + nCommands-- + } + + // rows + nRows, err := r.ReadInt32() + if err != nil { + return nil, fmt.Errorf("error reading trie num rows: %v", err) + } + for nRows > 0 { + row, err := newRow(r) + if err != nil { + return nil, fmt.Errorf("error reading trie row: %v", err) + } + rv.rows = append(rv.rows, row) + nRows-- + } + + return rv, nil +} + +func (t *trie) getRow(i int) *row { + if i < 0 || i >= len(t.rows) { + return nil + } + return t.rows[i] +} + +func (t *trie) GetLastOnPath(key []rune) []rune { + now := t.getRow(int(t.root)) + var last []rune + var w int32 + e := newStrEnum(key, t.forward) + + // walk over each rune + // if rune has row in the table, note the cmd (as last) + // if rune has row in table, see if it transitions to another row + // if it does, move to that row and next char on next loop itr + // if it does not, return the last cmd + // if you get to end of string and there is command in row use it + // or return last + for i := 0; i < len(key)-1; i++ { + r, err := e.next() + if err != nil { + return last + } + w = now.getCmd(r) + if w >= 0 { + last = []rune(t.cmds[w]) + } + w = now.getRef(r) + if w >= 0 { + now = t.getRow(int(w)) + } else { + return last + } + } + r, err := e.next() + if err != nil { + return last + } + w = now.getCmd(r) + if err != nil { + return last + } + if w >= 0 { + return []rune(t.cmds[w]) + } + return last +} + +func (t *trie) String() string { + rv := "" + for _, cmd := range t.cmds { + rv += fmt.Sprintf("cmd: %s\n", string(cmd)) + } + for _, row := range t.rows { + rv += fmt.Sprintf("row: %v\n", row) + } + return rv +} diff --git a/analysis/lang/pl/stop_filter_pl.go b/analysis/lang/pl/stop_filter_pl.go new file mode 100644 index 0000000..f8b2918 --- /dev/null +++ b/analysis/lang/pl/stop_filter_pl.go @@ -0,0 +1,36 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pl + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/pl/stop_words_pl.go b/analysis/lang/pl/stop_words_pl.go new file mode 100644 index 0000000..0755da8 --- /dev/null +++ b/analysis/lang/pl/stop_words_pl.go @@ -0,0 +1,368 @@ +package pl + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_pl" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ +// ` was changed to ' to allow for literal string + +var PolishStopWords = []byte(` | From https://github.com/stopwords-iso/stopwords-pl/tree/master + | The MIT License (MIT) + | See https://github.com/stopwords-iso/stopwords-pl/blob/master/LICENSE + | - Encoding was converted to UTF-8. + | - This notice was added. + | - english text is auto-translate + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + | a polish stop word list. comments begin with vertical bar. each stop + | word is at the start of a line. + +a | and +aby | to +ach | ah +acz | although +aczkolwiek | although +aj | ay +albo | or +ale | but +ależ | but +ani | or +aż | until +bardziej | more +bardzo | very +bez | without +bo | because +bowiem | because +by | by +byli | were +bym | i would +bynajmniej | not at all +być | to be +był | was +była | was +było | was +były | were +będzie | will be +będą | they will +cali | inches +cała | whole +cały | whole +chce | i want +choć | though +ci | you +ciebie | you +cię | you +co | what +cokolwiek | whatever +coraz | getting +coś | something +czasami | sometimes +czasem | sometimes +czemu | why +czy | whether +czyli | that is +często | often +daleko | far +dla | for +dlaczego | why +dlatego | which is why +do | down +dobrze | all right +dokąd | where +dość | enough +dr | dr +dużo | a lot +dwa | two +dwaj | two +dwie | two +dwoje | two +dzisiaj | today +dziś | today +gdy | when +gdyby | if +gdyż | because +gdzie | where +gdziekolwiek | wherever +gdzieś | somewhere +go | him +godz | time +hab | hab +i | and +ich | their +ii | ii +iii | iii +ile | how much +im | them +inna | different +inne | other +inny | other +innych | other +inż | eng +iv | iv +ix | ix +iż | that +ja | i +jak | how +jakaś | some +jakby | as if +jaki | what +jakichś | some +jakie | what +jakiś | some +jakiż | what +jakkolwiek | however +jako | as +jakoś | somehow +je | them +jeden | one +jedna | one +jednak | but +jednakże | however +jedno | one +jednym | one +jedynie | only +jego | his +jej | her +jemu | him +jest | is +jestem | i am +jeszcze | still +jeśli | if +jeżeli | if +już | already +ją | i +każdy | everyone +kiedy | when +kierunku | direction +kilka | several +kilku | several +kimś | someone +kto | who +ktokolwiek | anyone +ktoś | someone +która | which +które | which +którego | whose +której | which +który | which +których | which +którym | which +którzy | who +ku | to +lat | years +lecz | but +lub | or +ma | has +mają | may +mam | i have +mamy | we have +mało | little +mgr | msc +mi | to me +miał | had +mimo | despite +między | between +mnie | me +mną | me +mogą | they can +moi | my +moim | my +moja | my +moje | my +może | maybe +możliwe | that's possible +można | you can +mu | him +musi | has to +my | we +mój | my +na | on +nad | above +nam | u.s +nami | us +nas | us +nasi | our +nasz | our +nasza | our +nasze | our +naszego | our +naszych | ours +natomiast | whereas +natychmiast | immediately +nawet | even +nic | nothing +nich | them +nie | no +niech | let +niego | him +niej | her +niemu | not him +nigdy | never +nim | him +nimi | them +nią | her +niż | than +no | yeah +nowe | new +np | e.g. +nr | no +o | about +o.o. | o.o. +obok | near +od | from +ok | approx +około | about +on | he +ona | she +one | they +oni | they +ono | it +oraz | and +oto | here +owszem | yes +pan | mr +pana | mr +pani | you +pl | pl +po | after +pod | under +podczas | while +pomimo | despite +ponad | above +ponieważ | because +powinien | should +powinna | she should +powinni | they should +powinno | should +poza | apart from +prawie | almost +prof | prof +przecież | yet +przed | before +przede | above +przedtem | before +przez | by +przy | by +raz | once +razie | case +roku | year +również | also +sam | alone +sama | alone +się | myself +skąd | from where +sobie | myself +sobą | myself +sposób | way +swoje | own +są | are +ta | this +tak | yes +taka | such +taki | such +takich | such +takie | such +także | too +tam | over there +te | these +tego | this +tej | this one +tel | phone +temu | ago +ten | this +teraz | now +też | too +to | this +tobie | you +tobą | you +toteż | this as well +totobą | you +trzeba | it's necessary to +tu | here +tutaj | here +twoi | yours +twoim | yours +twoja | your +twoje | your +twym | your +twój | your +ty | you +tych | these +tylko | just +tym | this +tys | thousand +tzw | so-called +tę | these +u | at +ul | st +vi | vi +vii | vii +viii | viii +vol | vol +w | in +wam | you +wami | you +was | mustache +wasi | yours +wasz | yours +wasza | yours +wasze | yours +we | in +według | according to +wie | knows +wiele | many +wielu | many +więc | so +więcej | more +wszyscy | all +wszystkich | everyone +wszystkie | all +wszystkim | everyone +wszystko | all +wtedy | then +www | www +wy | you +właśnie | exactly +wśród | among +xi | x.x +xii | xii +xiii | xii +xiv | xiv +xv | xv +z | with +za | behind +zapewne | probably +zawsze | always +zaś | and +ze | that +zeznowu | testify +znowu | again +znów | again +został | left +zł | zloty +żaden | no +żadna | none +żadne | none +żadnych | none +że | that +żeby | to + +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(PolishStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/pt/analyzer_pt.go b/analysis/lang/pt/analyzer_pt.go new file mode 100644 index 0000000..db6b3a7 --- /dev/null +++ b/analysis/lang/pt/analyzer_pt.go @@ -0,0 +1,60 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pt + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +) + +const AnalyzerName = "pt" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + tokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopPtFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + stemmerPtFilter, err := cache.TokenFilterNamed(LightStemmerName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: tokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + stopPtFilter, + stemmerPtFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/pt/analyzer_pt_test.go b/analysis/lang/pt/analyzer_pt_test.go new file mode 100644 index 0000000..417e640 --- /dev/null +++ b/analysis/lang/pt/analyzer_pt_test.go @@ -0,0 +1,70 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pt + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestPortugueseAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // stemming + { + input: []byte("quilométricas"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("quilometric"), + }, + }, + }, + { + input: []byte("quilométricos"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("quilometric"), + }, + }, + }, + // stop word + { + input: []byte("não"), + output: analysis.TokenStream{}, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if len(actual) != len(test.output) { + t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) + } + for i, tok := range actual { + if !reflect.DeepEqual(tok.Term, test.output[i].Term) { + t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) + } + } + } +} diff --git a/analysis/lang/pt/light_stemmer_pt.go b/analysis/lang/pt/light_stemmer_pt.go new file mode 100644 index 0000000..ddb9d16 --- /dev/null +++ b/analysis/lang/pt/light_stemmer_pt.go @@ -0,0 +1,198 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pt + +import ( + "bytes" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const LightStemmerName = "stemmer_pt_light" + +type PortugueseLightStemmerFilter struct { +} + +func NewPortugueseLightStemmerFilter() *PortugueseLightStemmerFilter { + return &PortugueseLightStemmerFilter{} +} + +func (s *PortugueseLightStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + runes := bytes.Runes(token.Term) + runes = stem(runes) + token.Term = analysis.BuildTermFromRunes(runes) + } + return input +} + +func stem(input []rune) []rune { + + inputLen := len(input) + + if inputLen < 4 { + return input + } + + input = removeSuffix(input) + inputLen = len(input) + + if inputLen > 3 && input[inputLen-1] == 'a' { + input = normFeminine(input) + inputLen = len(input) + } + + if inputLen > 4 { + switch input[inputLen-1] { + case 'e', 'a', 'o': + input = input[0 : inputLen-1] + inputLen = len(input) + } + } + + for i := 0; i < inputLen; i++ { + switch input[i] { + case 'à', 'á', 'â', 'ä', 'ã': + input[i] = 'a' + case 'ò', 'ó', 'ô', 'ö', 'õ': + input[i] = 'o' + case 'è', 'é', 'ê', 'ë': + input[i] = 'e' + case 'ù', 'ú', 'û', 'ü': + input[i] = 'u' + case 'ì', 'í', 'î', 'ï': + input[i] = 'i' + case 'ç': + input[i] = 'c' + } + } + + return input +} + +func removeSuffix(input []rune) []rune { + + inputLen := len(input) + + if inputLen > 4 && analysis.RunesEndsWith(input, "es") { + switch input[inputLen-3] { + case 'r', 's', 'l', 'z': + return input[0 : inputLen-2] + } + } + + if inputLen > 3 && analysis.RunesEndsWith(input, "ns") { + input[inputLen-2] = 'm' + return input[0 : inputLen-1] + } + + if inputLen > 4 && (analysis.RunesEndsWith(input, "eis") || analysis.RunesEndsWith(input, "éis")) { + input[inputLen-3] = 'e' + input[inputLen-2] = 'l' + return input[0 : inputLen-1] + } + + if inputLen > 4 && analysis.RunesEndsWith(input, "ais") { + input[inputLen-2] = 'l' + return input[0 : inputLen-1] + } + + if inputLen > 4 && analysis.RunesEndsWith(input, "óis") { + input[inputLen-3] = 'o' + input[inputLen-2] = 'l' + return input[0 : inputLen-1] + } + + if inputLen > 4 && analysis.RunesEndsWith(input, "is") { + input[inputLen-1] = 'l' + return input + } + + if inputLen > 3 && + (analysis.RunesEndsWith(input, "ões") || + analysis.RunesEndsWith(input, "ães")) { + input = input[0 : inputLen-1] + inputLen = len(input) + input[inputLen-2] = 'ã' + input[inputLen-1] = 'o' + return input + } + + if inputLen > 6 && analysis.RunesEndsWith(input, "mente") { + return input[0 : inputLen-5] + } + + if inputLen > 3 && input[inputLen-1] == 's' { + return input[0 : inputLen-1] + } + return input +} + +func normFeminine(input []rune) []rune { + inputLen := len(input) + + if inputLen > 7 && + (analysis.RunesEndsWith(input, "inha") || + analysis.RunesEndsWith(input, "iaca") || + analysis.RunesEndsWith(input, "eira")) { + input[inputLen-1] = 'o' + return input + } + + if inputLen > 6 { + if analysis.RunesEndsWith(input, "osa") || + analysis.RunesEndsWith(input, "ica") || + analysis.RunesEndsWith(input, "ida") || + analysis.RunesEndsWith(input, "ada") || + analysis.RunesEndsWith(input, "iva") || + analysis.RunesEndsWith(input, "ama") { + input[inputLen-1] = 'o' + return input + } + + if analysis.RunesEndsWith(input, "ona") { + input[inputLen-3] = 'ã' + input[inputLen-2] = 'o' + return input[0 : inputLen-1] + } + + if analysis.RunesEndsWith(input, "ora") { + return input[0 : inputLen-1] + } + + if analysis.RunesEndsWith(input, "esa") { + input[inputLen-3] = 'ê' + return input[0 : inputLen-1] + } + + if analysis.RunesEndsWith(input, "na") { + input[inputLen-1] = 'o' + return input + } + } + return input +} + +func PortugueseLightStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewPortugueseLightStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(LightStemmerName, PortugueseLightStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/pt/light_stemmer_pt_test.go b/analysis/lang/pt/light_stemmer_pt_test.go new file mode 100644 index 0000000..76b1435 --- /dev/null +++ b/analysis/lang/pt/light_stemmer_pt_test.go @@ -0,0 +1,404 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pt + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestPortugueseLightStemmer(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("doutores"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("doutor"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("doutor"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("doutor"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("homens"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("homem"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("homem"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("homem"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("papéis"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("papel"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("papel"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("papel"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("normais"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("normal"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("normal"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("normal"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("lencóis"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("lencol"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("lencol"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("lencol"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("barris"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("barril"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("barril"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("barril"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("botões"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("bota"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("botão"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("bota"), + }, + }, + }, + // longer + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("o"), + }, + &analysis.Token{ + Term: []byte("debate"), + }, + &analysis.Token{ + Term: []byte("político"), + }, + &analysis.Token{ + Term: []byte("pelo"), + }, + &analysis.Token{ + Term: []byte("menos"), + }, + &analysis.Token{ + Term: []byte("o"), + }, + &analysis.Token{ + Term: []byte("que"), + }, + &analysis.Token{ + Term: []byte("vem"), + }, + &analysis.Token{ + Term: []byte("a"), + }, + &analysis.Token{ + Term: []byte("público"), + }, + &analysis.Token{ + Term: []byte("parece"), + }, + &analysis.Token{ + Term: []byte("de"), + }, + &analysis.Token{ + Term: []byte("modo"), + }, + &analysis.Token{ + Term: []byte("nada"), + }, + &analysis.Token{ + Term: []byte("surpreendente"), + }, + &analysis.Token{ + Term: []byte("restrito"), + }, + &analysis.Token{ + Term: []byte("a"), + }, + &analysis.Token{ + Term: []byte("temas"), + }, + &analysis.Token{ + Term: []byte("menores"), + }, + &analysis.Token{ + Term: []byte("mas"), + }, + &analysis.Token{ + Term: []byte("há"), + }, + &analysis.Token{ + Term: []byte("evidentemente"), + }, + &analysis.Token{ + Term: []byte("grandes"), + }, + &analysis.Token{ + Term: []byte("questões"), + }, + &analysis.Token{ + Term: []byte("em"), + }, + &analysis.Token{ + Term: []byte("jogo"), + }, + &analysis.Token{ + Term: []byte("nas"), + }, + &analysis.Token{ + Term: []byte("eleições"), + }, + &analysis.Token{ + Term: []byte("que"), + }, + &analysis.Token{ + Term: []byte("se"), + }, + &analysis.Token{ + Term: []byte("aproximam"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("o"), + }, + &analysis.Token{ + Term: []byte("debat"), + }, + &analysis.Token{ + Term: []byte("politic"), + }, + &analysis.Token{ + Term: []byte("pelo"), + }, + &analysis.Token{ + Term: []byte("meno"), + }, + &analysis.Token{ + Term: []byte("o"), + }, + &analysis.Token{ + Term: []byte("que"), + }, + &analysis.Token{ + Term: []byte("vem"), + }, + &analysis.Token{ + Term: []byte("a"), + }, + &analysis.Token{ + Term: []byte("public"), + }, + &analysis.Token{ + Term: []byte("parec"), + }, + &analysis.Token{ + Term: []byte("de"), + }, + &analysis.Token{ + Term: []byte("modo"), + }, + &analysis.Token{ + Term: []byte("nada"), + }, + &analysis.Token{ + Term: []byte("surpreendent"), + }, + &analysis.Token{ + Term: []byte("restrit"), + }, + &analysis.Token{ + Term: []byte("a"), + }, + &analysis.Token{ + Term: []byte("tema"), + }, + &analysis.Token{ + Term: []byte("menor"), + }, + &analysis.Token{ + Term: []byte("mas"), + }, + &analysis.Token{ + Term: []byte("há"), + }, + &analysis.Token{ + Term: []byte("evident"), + }, + &analysis.Token{ + Term: []byte("grand"), + }, + &analysis.Token{ + Term: []byte("questa"), + }, + &analysis.Token{ + Term: []byte("em"), + }, + &analysis.Token{ + Term: []byte("jogo"), + }, + &analysis.Token{ + Term: []byte("nas"), + }, + &analysis.Token{ + Term: []byte("eleica"), + }, + &analysis.Token{ + Term: []byte("que"), + }, + &analysis.Token{ + Term: []byte("se"), + }, + &analysis.Token{ + Term: []byte("aproximam"), + }, + }, + }, + } + + cache := registry.NewCache() + filter, err := cache.TokenFilterNamed(LightStemmerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := filter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/pt/stop_filter_pt.go b/analysis/lang/pt/stop_filter_pt.go new file mode 100644 index 0000000..a0b8886 --- /dev/null +++ b/analysis/lang/pt/stop_filter_pt.go @@ -0,0 +1,36 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pt + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/pt/stop_words_pt.go b/analysis/lang/pt/stop_words_pt.go new file mode 100644 index 0000000..b213389 --- /dev/null +++ b/analysis/lang/pt/stop_words_pt.go @@ -0,0 +1,280 @@ +package pt + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_pt" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ +// ` was changed to ' to allow for literal string + +var PortugueseStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/portuguese/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A Portuguese stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + + | The following is a ranked list (commonest to rarest) of stopwords + | deriving from a large sample of text. + + | Extra words have been added at the end. + +de | of, from +a | the; to, at; her +o | the; him +que | who, that +e | and +do | de + o +da | de + a +em | in +um | a +para | for + | é from SER +com | with +não | not, no +uma | a +os | the; them +no | em + o +se | himself etc +na | em + a +por | for +mais | more +as | the; them +dos | de + os +como | as, like +mas | but + | foi from SER +ao | a + o +ele | he +das | de + as + | tem from TER +à | a + a +seu | his +sua | her +ou | or + | ser from SER +quando | when +muito | much + | há from HAV +nos | em + os; us +já | already, now + | está from EST +eu | I +também | also +só | only, just +pelo | per + o +pela | per + a +até | up to +isso | that +ela | he +entre | between + | era from SER +depois | after +sem | without +mesmo | same +aos | a + os + | ter from TER +seus | his +quem | whom +nas | em + as +me | me +esse | that +eles | they + | estão from EST +você | you + | tinha from TER + | foram from SER +essa | that +num | em + um +nem | nor +suas | her +meu | my +às | a + as +minha | my + | têm from TER +numa | em + uma +pelos | per + os +elas | they + | havia from HAV + | seja from SER +qual | which + | será from SER +nós | we + | tenho from TER +lhe | to him, her +deles | of them +essas | those +esses | those +pelas | per + as +este | this + | fosse from SER +dele | of him + + | other words. There are many contractions such as naquele = em+aquele, + | mo = me+o, but they are rare. + | Indefinite article plural forms are also rare. + +tu | thou +te | thee +vocês | you (plural) +vos | you +lhes | to them +meus | my +minhas +teu | thy +tua +teus +tuas +nosso | our +nossa +nossos +nossas + +dela | of her +delas | of them + +esta | this +estes | these +estas | these +aquele | that +aquela | that +aqueles | those +aquelas | those +isto | this +aquilo | that + + | forms of estar, to be (not including the infinitive): +estou +está +estamos +estão +estive +esteve +estivemos +estiveram +estava +estávamos +estavam +estivera +estivéramos +esteja +estejamos +estejam +estivesse +estivéssemos +estivessem +estiver +estivermos +estiverem + + | forms of haver, to have (not including the infinitive): +hei +há +havemos +hão +houve +houvemos +houveram +houvera +houvéramos +haja +hajamos +hajam +houvesse +houvéssemos +houvessem +houver +houvermos +houverem +houverei +houverá +houveremos +houverão +houveria +houveríamos +houveriam + + | forms of ser, to be (not including the infinitive): +sou +somos +são +era +éramos +eram +fui +foi +fomos +foram +fora +fôramos +seja +sejamos +sejam +fosse +fôssemos +fossem +for +formos +forem +serei +será +seremos +serão +seria +seríamos +seriam + + | forms of ter, to have (not including the infinitive): +tenho +tem +temos +tém +tinha +tínhamos +tinham +tive +teve +tivemos +tiveram +tivera +tivéramos +tenha +tenhamos +tenham +tivesse +tivéssemos +tivessem +tiver +tivermos +tiverem +terei +terá +teremos +terão +teria +teríamos +teriam +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(PortugueseStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ro/analyzer_ro.go b/analysis/lang/ro/analyzer_ro.go new file mode 100644 index 0000000..f138f80 --- /dev/null +++ b/analysis/lang/ro/analyzer_ro.go @@ -0,0 +1,60 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ro + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +) + +const AnalyzerName = "ro" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + unicodeTokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopRoFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + stemmerRoFilter, err := cache.TokenFilterNamed(SnowballStemmerName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: unicodeTokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + stopRoFilter, + stemmerRoFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ro/analyzer_ro_test.go b/analysis/lang/ro/analyzer_ro_test.go new file mode 100644 index 0000000..0fe4645 --- /dev/null +++ b/analysis/lang/ro/analyzer_ro_test.go @@ -0,0 +1,70 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ro + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestRomanianAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // stemming + { + input: []byte("absenţa"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("absenţ"), + }, + }, + }, + { + input: []byte("absenţi"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("absenţ"), + }, + }, + }, + // stop word + { + input: []byte("îl"), + output: analysis.TokenStream{}, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if len(actual) != len(test.output) { + t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) + } + for i, tok := range actual { + if !reflect.DeepEqual(tok.Term, test.output[i].Term) { + t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) + } + } + } +} diff --git a/analysis/lang/ro/stemmer_ro.go b/analysis/lang/ro/stemmer_ro.go new file mode 100644 index 0000000..f7efc73 --- /dev/null +++ b/analysis/lang/ro/stemmer_ro.go @@ -0,0 +1,52 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ro + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/snowballstem" + "github.com/blevesearch/snowballstem/romanian" +) + +const SnowballStemmerName = "stemmer_ro_snowball" + +type RomanianStemmerFilter struct { +} + +func NewRomanianStemmerFilter() *RomanianStemmerFilter { + return &RomanianStemmerFilter{} +} + +func (s *RomanianStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + env := snowballstem.NewEnv(string(token.Term)) + romanian.Stem(env) + token.Term = []byte(env.Current()) + } + return input +} + +func RomanianStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewRomanianStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(SnowballStemmerName, RomanianStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ro/stop_filter_ro.go b/analysis/lang/ro/stop_filter_ro.go new file mode 100644 index 0000000..0932d33 --- /dev/null +++ b/analysis/lang/ro/stop_filter_ro.go @@ -0,0 +1,36 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ro + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ro/stop_words_ro.go b/analysis/lang/ro/stop_words_ro.go new file mode 100644 index 0000000..8c01a41 --- /dev/null +++ b/analysis/lang/ro/stop_words_ro.go @@ -0,0 +1,260 @@ +package ro + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_ro" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ +// ` was changed to ' to allow for literal string + +var RomanianStopWords = []byte(`# This file was created by Jacques Savoy and is distributed under the BSD license. +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# Also see http://www.opensource.org/licenses/bsd-license.html +acea +aceasta +această +aceea +acei +aceia +acel +acela +acele +acelea +acest +acesta +aceste +acestea +aceşti +aceştia +acolo +acum +ai +aia +aibă +aici +al +ăla +ale +alea +ălea +altceva +altcineva +am +ar +are +aş +aşadar +asemenea +asta +ăsta +astăzi +astea +ăstea +ăştia +asupra +aţi +au +avea +avem +aveţi +azi +bine +bucur +bună +ca +că +căci +când +care +cărei +căror +cărui +cât +câte +câţi +către +câtva +ce +cel +ceva +chiar +cînd +cine +cineva +cît +cîte +cîţi +cîtva +contra +cu +cum +cumva +curând +curînd +da +dă +dacă +dar +datorită +de +deci +deja +deoarece +departe +deşi +din +dinaintea +dintr +dintre +drept +după +ea +ei +el +ele +eram +este +eşti +eu +face +fără +fi +fie +fiecare +fii +fim +fiţi +iar +ieri +îi +îl +îmi +împotriva +în +înainte +înaintea +încât +încît +încotro +între +întrucât +întrucît +îţi +la +lângă +le +li +lîngă +lor +lui +mă +mâine +mea +mei +mele +mereu +meu +mi +mine +mult +multă +mulţi +ne +nicăieri +nici +nimeni +nişte +noastră +noastre +noi +noştri +nostru +nu +ori +oricând +oricare +oricât +orice +oricînd +oricine +oricît +oricum +oriunde +până +pe +pentru +peste +pînă +poate +pot +prea +prima +primul +prin +printr +sa +să +săi +sale +sau +său +se +şi +sînt +sîntem +sînteţi +spre +sub +sunt +suntem +sunteţi +ta +tăi +tale +tău +te +ţi +ţie +tine +toată +toate +tot +toţi +totuşi +tu +un +una +unde +undeva +unei +unele +uneori +unor +vă +vi +voastră +voastre +voi +voştri +vostru +vouă +vreo +vreun +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(RomanianStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ru/analyzer_ru.go b/analysis/lang/ru/analyzer_ru.go new file mode 100644 index 0000000..06092a4 --- /dev/null +++ b/analysis/lang/ru/analyzer_ru.go @@ -0,0 +1,60 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ru + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +) + +const AnalyzerName = "ru" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + tokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopRuFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + stemmerRuFilter, err := cache.TokenFilterNamed(SnowballStemmerName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: tokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + stopRuFilter, + stemmerRuFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ru/analyzer_ru_test.go b/analysis/lang/ru/analyzer_ru_test.go new file mode 100644 index 0000000..38534af --- /dev/null +++ b/analysis/lang/ru/analyzer_ru_test.go @@ -0,0 +1,122 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ru + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestRussianAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // stemming + { + input: []byte("километрах"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("километр"), + }, + }, + }, + { + input: []byte("актеров"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("актер"), + }, + }, + }, + // stop word + { + input: []byte("как"), + output: analysis.TokenStream{}, + }, + // digits safe + { + input: []byte("text 1000"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("text"), + }, + &analysis.Token{ + Term: []byte("1000"), + }, + }, + }, + { + input: []byte("Вместе с тем о силе электромагнитной энергии имели представление еще"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("вмест"), + }, + &analysis.Token{ + Term: []byte("сил"), + }, + &analysis.Token{ + Term: []byte("электромагнитн"), + }, + &analysis.Token{ + Term: []byte("энерг"), + }, + &analysis.Token{ + Term: []byte("имел"), + }, + &analysis.Token{ + Term: []byte("представлен"), + }, + }, + }, + { + input: []byte("Но знание это хранилось в тайне"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("знан"), + }, + &analysis.Token{ + Term: []byte("эт"), + }, + &analysis.Token{ + Term: []byte("хран"), + }, + &analysis.Token{ + Term: []byte("тайн"), + }, + }, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if len(actual) != len(test.output) { + t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) + } + for i, tok := range actual { + if !reflect.DeepEqual(tok.Term, test.output[i].Term) { + t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) + } + } + } +} diff --git a/analysis/lang/ru/stemmer_ru.go b/analysis/lang/ru/stemmer_ru.go new file mode 100644 index 0000000..e271bb6 --- /dev/null +++ b/analysis/lang/ru/stemmer_ru.go @@ -0,0 +1,52 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ru + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/snowballstem" + "github.com/blevesearch/snowballstem/russian" +) + +const SnowballStemmerName = "stemmer_ru_snowball" + +type RussianStemmerFilter struct { +} + +func NewRussianStemmerFilter() *RussianStemmerFilter { + return &RussianStemmerFilter{} +} + +func (s *RussianStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + env := snowballstem.NewEnv(string(token.Term)) + russian.Stem(env) + token.Term = []byte(env.Current()) + } + return input +} + +func RussianStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewRussianStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(SnowballStemmerName, RussianStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ru/stemmer_ru_test.go b/analysis/lang/ru/stemmer_ru_test.go new file mode 100644 index 0000000..8d44181 --- /dev/null +++ b/analysis/lang/ru/stemmer_ru_test.go @@ -0,0 +1,67 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ru + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestSnowballRussianStemmer(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("актеров"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("актер"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("километров"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("километр"), + }, + }, + }, + } + + cache := registry.NewCache() + filter, err := cache.TokenFilterNamed(SnowballStemmerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := filter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/ru/stop_filter_ru.go b/analysis/lang/ru/stop_filter_ru.go new file mode 100644 index 0000000..a7b9fa1 --- /dev/null +++ b/analysis/lang/ru/stop_filter_ru.go @@ -0,0 +1,36 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ru + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/ru/stop_words_ru.go b/analysis/lang/ru/stop_words_ru.go new file mode 100644 index 0000000..28ed500 --- /dev/null +++ b/analysis/lang/ru/stop_words_ru.go @@ -0,0 +1,270 @@ +package ru + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_ru" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ +// ` was changed to ' to allow for literal string + +var RussianStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/russian/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | a russian stop word list. comments begin with vertical bar. each stop + | word is at the start of a line. + + | this is a ranked list (commonest to rarest) of stopwords derived from + | a large text sample. + + | letter 'ё' is translated to 'е'. + +и | and +в | in/into +во | alternative form +не | not +что | what/that +он | he +на | on/onto +я | i +с | from +со | alternative form +как | how +а | milder form of 'no' (but) +то | conjunction and form of 'that' +все | all +она | she +так | so, thus +его | him +но | but +да | yes/and +ты | thou +к | towards, by +у | around, chez +же | intensifier particle +вы | you +за | beyond, behind +бы | conditional/subj. particle +по | up to, along +только | only +ее | her +мне | to me +было | it was +вот | here is/are, particle +от | away from +меня | me +еще | still, yet, more +нет | no, there isnt/arent +о | about +из | out of +ему | to him +теперь | now +когда | when +даже | even +ну | so, well +вдруг | suddenly +ли | interrogative particle +если | if +уже | already, but homonym of 'narrower' +или | or +ни | neither +быть | to be +был | he was +него | prepositional form of его +до | up to +вас | you accusative +нибудь | indef. suffix preceded by hyphen +опять | again +уж | already, but homonym of 'adder' +вам | to you +сказал | he said +ведь | particle 'after all' +там | there +потом | then +себя | oneself +ничего | nothing +ей | to her +может | usually with 'быть' as 'maybe' +они | they +тут | here +где | where +есть | there is/are +надо | got to, must +ней | prepositional form of ей +для | for +мы | we +тебя | thee +их | them, their +чем | than +была | she was +сам | self +чтоб | in order to +без | without +будто | as if +человек | man, person, one +чего | genitive form of 'what' +раз | once +тоже | also +себе | to oneself +под | beneath +жизнь | life +будет | will be +ж | short form of intensifer particle 'же' +тогда | then +кто | who +этот | this +говорил | was saying +того | genitive form of 'that' +потому | for that reason +этого | genitive form of 'this' +какой | which +совсем | altogether +ним | prepositional form of 'его', 'они' +здесь | here +этом | prepositional form of 'этот' +один | one +почти | almost +мой | my +тем | instrumental/dative plural of 'тот', 'то' +чтобы | full form of 'in order that' +нее | her (acc.) +кажется | it seems +сейчас | now +были | they were +куда | where to +зачем | why +сказать | to say +всех | all (acc., gen. preposn. plural) +никогда | never +сегодня | today +можно | possible, one can +при | by +наконец | finally +два | two +об | alternative form of 'о', about +другой | another +хоть | even +после | after +над | above +больше | more +тот | that one (masc.) +через | across, in +эти | these +нас | us +про | about +всего | in all, only, of all +них | prepositional form of 'они' (they) +какая | which, feminine +много | lots +разве | interrogative particle +сказала | she said +три | three +эту | this, acc. fem. sing. +моя | my, feminine +впрочем | moreover, besides +хорошо | good +свою | ones own, acc. fem. sing. +этой | oblique form of 'эта', fem. 'this' +перед | in front of +иногда | sometimes +лучше | better +чуть | a little +том | preposn. form of 'that one' +нельзя | one must not +такой | such a one +им | to them +более | more +всегда | always +конечно | of course +всю | acc. fem. sing of 'all' +между | between + + + | b: some paradigms + | + | personal pronouns + | + | я меня мне мной [мною] + | ты тебя тебе тобой [тобою] + | он его ему им [него, нему, ним] + | она ее эи ею [нее, нэи, нею] + | оно его ему им [него, нему, ним] + | + | мы нас нам нами + | вы вас вам вами + | они их им ими [них, ним, ними] + | + | себя себе собой [собою] + | + | demonstrative pronouns: этот (this), тот (that) + | + | этот эта это эти + | этого эты это эти + | этого этой этого этих + | этому этой этому этим + | этим этой этим [этою] этими + | этом этой этом этих + | + | тот та то те + | того ту то те + | того той того тех + | тому той тому тем + | тем той тем [тою] теми + | том той том тех + | + | determinative pronouns + | + | (a) весь (all) + | + | весь вся все все + | всего всю все все + | всего всей всего всех + | всему всей всему всем + | всем всей всем [всею] всеми + | всем всей всем всех + | + | (b) сам (himself etc) + | + | сам сама само сами + | самого саму само самих + | самого самой самого самих + | самому самой самому самим + | самим самой самим [самою] самими + | самом самой самом самих + | + | stems of verbs 'to be', 'to have', 'to do' and modal + | + | быть бы буд быв есть суть + | име + | дел + | мог мож мочь + | уме + | хоч хот + | долж + | можн + | нужн + | нельзя + +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(RussianStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/sv/analyzer_sv.go b/analysis/lang/sv/analyzer_sv.go new file mode 100644 index 0000000..e8c76e1 --- /dev/null +++ b/analysis/lang/sv/analyzer_sv.go @@ -0,0 +1,60 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sv + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +) + +const AnalyzerName = "sv" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + unicodeTokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopSvFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + stemmerSvFilter, err := cache.TokenFilterNamed(SnowballStemmerName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: unicodeTokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + stopSvFilter, + stemmerSvFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/sv/analyzer_sv_test.go b/analysis/lang/sv/analyzer_sv_test.go new file mode 100644 index 0000000..a3bd5f1 --- /dev/null +++ b/analysis/lang/sv/analyzer_sv_test.go @@ -0,0 +1,70 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sv + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestSwedishAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // stemming + { + input: []byte("jaktkarlarne"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("jaktkarl"), + }, + }, + }, + { + input: []byte("jaktkarlens"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("jaktkarl"), + }, + }, + }, + // stop word + { + input: []byte("och"), + output: analysis.TokenStream{}, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if len(actual) != len(test.output) { + t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) + } + for i, tok := range actual { + if !reflect.DeepEqual(tok.Term, test.output[i].Term) { + t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) + } + } + } +} diff --git a/analysis/lang/sv/stemmer_sv.go b/analysis/lang/sv/stemmer_sv.go new file mode 100644 index 0000000..679fc98 --- /dev/null +++ b/analysis/lang/sv/stemmer_sv.go @@ -0,0 +1,52 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sv + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/snowballstem" + "github.com/blevesearch/snowballstem/swedish" +) + +const SnowballStemmerName = "stemmer_sv_snowball" + +type SwedishStemmerFilter struct { +} + +func NewSwedishStemmerFilter() *SwedishStemmerFilter { + return &SwedishStemmerFilter{} +} + +func (s *SwedishStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + env := snowballstem.NewEnv(string(token.Term)) + swedish.Stem(env) + token.Term = []byte(env.Current()) + } + return input +} + +func SwedishStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewSwedishStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(SnowballStemmerName, SwedishStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/sv/stop_filter_sv.go b/analysis/lang/sv/stop_filter_sv.go new file mode 100644 index 0000000..9d3a57d --- /dev/null +++ b/analysis/lang/sv/stop_filter_sv.go @@ -0,0 +1,36 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sv + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/sv/stop_words_sv.go b/analysis/lang/sv/stop_words_sv.go new file mode 100644 index 0000000..a420e7d --- /dev/null +++ b/analysis/lang/sv/stop_words_sv.go @@ -0,0 +1,160 @@ +package sv + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_sv" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ +// ` was changed to ' to allow for literal string + +var SwedishStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/swedish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" + + | A Swedish stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This is a ranked list (commonest to rarest) of stopwords derived from + | a large text sample. + + | Swedish stop words occasionally exhibit homonym clashes. For example + | så = so, but also seed. These are indicated clearly below. + +och | and +det | it, this/that +att | to (with infinitive) +i | in, at +en | a +jag | I +hon | she +som | who, that +han | he +på | on +den | it, this/that +med | with +var | where, each +sig | him(self) etc +för | for +så | so (also: seed) +till | to +är | is +men | but +ett | a +om | if; around, about +hade | had +de | they, these/those +av | of +icke | not, no +mig | me +du | you +henne | her +då | then, when +sin | his +nu | now +har | have +inte | inte någon = no one +hans | his +honom | him +skulle | 'sake' +hennes | her +där | there +min | my +man | one (pronoun) +ej | nor +vid | at, by, on (also: vast) +kunde | could +något | some etc +från | from, off +ut | out +när | when +efter | after, behind +upp | up +vi | we +dem | them +vara | be +vad | what +över | over +än | than +dig | you +kan | can +sina | his +här | here +ha | have +mot | towards +alla | all +under | under (also: wonder) +någon | some etc +eller | or (else) +allt | all +mycket | much +sedan | since +ju | why +denna | this/that +själv | myself, yourself etc +detta | this/that +åt | to +utan | without +varit | was +hur | how +ingen | no +mitt | my +ni | you +bli | to be, become +blev | from bli +oss | us +din | thy +dessa | these/those +några | some etc +deras | their +blir | from bli +mina | my +samma | (the) same +vilken | who, that +er | you, your +sådan | such a +vår | our +blivit | from bli +dess | its +inom | within +mellan | between +sådant | such a +varför | why +varje | each +vilka | who, that +ditt | thy +vem | who +vilket | who, that +sitta | his +sådana | such a +vart | each +dina | thy +vars | whose +vårt | our +våra | our +ert | your +era | your +vilkas | whose + +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(SwedishStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/tr/analyzer_tr.go b/analysis/lang/tr/analyzer_tr.go new file mode 100644 index 0000000..69e7537 --- /dev/null +++ b/analysis/lang/tr/analyzer_tr.go @@ -0,0 +1,66 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tr + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/bleve/v2/analysis/token/apostrophe" + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +) + +const AnalyzerName = "tr" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) { + unicodeTokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + aposFilter, err := cache.TokenFilterNamed(apostrophe.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopTrFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + stemmerTrFilter, err := cache.TokenFilterNamed(SnowballStemmerName) + if err != nil { + return nil, err + } + rv := analysis.DefaultAnalyzer{ + Tokenizer: unicodeTokenizer, + TokenFilters: []analysis.TokenFilter{ + aposFilter, + toLowerFilter, + stopTrFilter, + stemmerTrFilter, + }, + } + return &rv, nil +} + +func init() { + err := registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/tr/analyzer_tr_test.go b/analysis/lang/tr/analyzer_tr_test.go new file mode 100644 index 0000000..3c4592b --- /dev/null +++ b/analysis/lang/tr/analyzer_tr_test.go @@ -0,0 +1,90 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tr + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestTurkishAnalyzer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + // stemming + { + input: []byte("ağacı"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ağaç"), + }, + }, + }, + { + input: []byte("ağaç"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ağaç"), + }, + }, + }, + // stop word + { + input: []byte("dolayı"), + output: analysis.TokenStream{}, + }, + // apostrophes + { + input: []byte("Kıbrıs'ta"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("kıbrıs"), + }, + }, + }, + { + input: []byte("Van Gölü'ne"), + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("van"), + }, + &analysis.Token{ + Term: []byte("göl"), + }, + }, + }, + } + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(AnalyzerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := analyzer.Analyze(test.input) + if len(actual) != len(test.output) { + t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) + } + for i, tok := range actual { + if !reflect.DeepEqual(tok.Term, test.output[i].Term) { + t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) + } + } + } +} diff --git a/analysis/lang/tr/stemmer_tr.go b/analysis/lang/tr/stemmer_tr.go new file mode 100644 index 0000000..ce36655 --- /dev/null +++ b/analysis/lang/tr/stemmer_tr.go @@ -0,0 +1,52 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tr + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/snowballstem" + "github.com/blevesearch/snowballstem/turkish" +) + +const SnowballStemmerName = "stemmer_tr_snowball" + +type TurkishStemmerFilter struct { +} + +func NewTurkishStemmerFilter() *TurkishStemmerFilter { + return &TurkishStemmerFilter{} +} + +func (s *TurkishStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + env := snowballstem.NewEnv(string(token.Term)) + turkish.Stem(env) + token.Term = []byte(env.Current()) + } + return input +} + +func TurkishStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewTurkishStemmerFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(SnowballStemmerName, TurkishStemmerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/tr/stemmer_tr_test.go b/analysis/lang/tr/stemmer_tr_test.go new file mode 100644 index 0000000..8ad5a56 --- /dev/null +++ b/analysis/lang/tr/stemmer_tr_test.go @@ -0,0 +1,115 @@ +// Copyright (c) 2025 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tr + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestSnowballTurkishStemmer(t *testing.T) { + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("kimsesizler"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("kimsesiz"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("kitaplar"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("kitap"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("arabanın"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("araba"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("bardaklar"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("bardak"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("kediye"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("kedi"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("yazdım"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("yaz"), + }, + }, + }, + } + + cache := registry.NewCache() + filter, err := cache.TokenFilterNamed(SnowballStemmerName) + if err != nil { + t.Fatal(err) + } + for _, test := range tests { + actual := filter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/lang/tr/stop_filter_tr.go b/analysis/lang/tr/stop_filter_tr.go new file mode 100644 index 0000000..0f75ffa --- /dev/null +++ b/analysis/lang/tr/stop_filter_tr.go @@ -0,0 +1,36 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tr + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/token/stop" + "github.com/blevesearch/bleve/v2/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/lang/tr/stop_words_tr.go b/analysis/lang/tr/stop_words_tr.go new file mode 100644 index 0000000..49116c2 --- /dev/null +++ b/analysis/lang/tr/stop_words_tr.go @@ -0,0 +1,239 @@ +package tr + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const StopName = "stop_tr" + +// this content was obtained from: +// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ +// ` was changed to ' to allow for literal string + +var TurkishStopWords = []byte(`# Turkish stopwords from LUCENE-559 +# merged with the list from "Information Retrieval on Turkish Texts" +# (http://www.users.muohio.edu/canf/papers/JASIST2008offPrint.pdf) +acaba +altmış +altı +ama +ancak +arada +aslında +ayrıca +bana +bazı +belki +ben +benden +beni +benim +beri +beş +bile +bin +bir +birçok +biri +birkaç +birkez +birşey +birşeyi +biz +bize +bizden +bizi +bizim +böyle +böylece +bu +buna +bunda +bundan +bunlar +bunları +bunların +bunu +bunun +burada +çok +çünkü +da +daha +dahi +de +defa +değil +diğer +diye +doksan +dokuz +dolayı +dolayısıyla +dört +edecek +eden +ederek +edilecek +ediliyor +edilmesi +ediyor +eğer +elli +en +etmesi +etti +ettiği +ettiğini +gibi +göre +halen +hangi +hatta +hem +henüz +hep +hepsi +her +herhangi +herkesin +hiç +hiçbir +için +iki +ile +ilgili +ise +işte +itibaren +itibariyle +kadar +karşın +katrilyon +kendi +kendilerine +kendini +kendisi +kendisine +kendisini +kez +ki +kim +kimden +kime +kimi +kimse +kırk +milyar +milyon +mu +mü +mı +nasıl +ne +neden +nedenle +nerde +nerede +nereye +niye +niçin +o +olan +olarak +oldu +olduğu +olduğunu +olduklarını +olmadı +olmadığı +olmak +olması +olmayan +olmaz +olsa +olsun +olup +olur +olursa +oluyor +on +ona +ondan +onlar +onlardan +onları +onların +onu +onun +otuz +oysa +öyle +pek +rağmen +sadece +sanki +sekiz +seksen +sen +senden +seni +senin +siz +sizden +sizi +sizin +şey +şeyden +şeyi +şeyler +şöyle +şu +şuna +şunda +şundan +şunları +şunu +tarafından +trilyon +tüm +üç +üzere +var +vardı +ve +veya +ya +yani +yapacak +yapılan +yapılması +yapıyor +yapmak +yaptı +yaptığı +yaptığını +yaptıkları +yedi +yerine +yetmiş +yine +yirmi +yoksa +yüz +zaten +`) + +func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + err := rv.LoadBytes(TurkishStopWords) + return rv, err +} + +func init() { + err := registry.RegisterTokenMap(StopName, TokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/test_words.txt b/analysis/test_words.txt new file mode 100644 index 0000000..b86e254 --- /dev/null +++ b/analysis/test_words.txt @@ -0,0 +1,7 @@ +# full line comment +marty +steve # trailing comment +| different format of comment +dustin +siri | different style trailing comment +multiple words with different whitespace \ No newline at end of file diff --git a/analysis/token/apostrophe/apostrophe.go b/analysis/token/apostrophe/apostrophe.go new file mode 100644 index 0000000..20290a3 --- /dev/null +++ b/analysis/token/apostrophe/apostrophe.go @@ -0,0 +1,57 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apostrophe + +import ( + "bytes" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "apostrophe" + +const RightSingleQuotationMark = "’" +const Apostrophe = "'" +const Apostrophes = Apostrophe + RightSingleQuotationMark + +type ApostropheFilter struct{} + +func NewApostropheFilter() *ApostropheFilter { + return &ApostropheFilter{} +} + +func (s *ApostropheFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + firstApostrophe := bytes.IndexAny(token.Term, Apostrophes) + if firstApostrophe >= 0 { + // found an apostrophe + token.Term = token.Term[0:firstApostrophe] + } + } + + return input +} + +func ApostropheFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewApostropheFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(Name, ApostropheFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/token/apostrophe/apostrophe_test.go b/analysis/token/apostrophe/apostrophe_test.go new file mode 100644 index 0000000..b704dc7 --- /dev/null +++ b/analysis/token/apostrophe/apostrophe_test.go @@ -0,0 +1,99 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apostrophe + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestApostropheFilter(t *testing.T) { + + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Türkiye'de"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Türkiye"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("2003'te"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("2003"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Van"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Van"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Gölü'nü"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Gölü"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("gördüm"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("gördüm"), + }, + }, + }, + } + + for _, test := range tests { + apostropheFilter := NewApostropheFilter() + actual := apostropheFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/token/camelcase/camelcase.go b/analysis/token/camelcase/camelcase.go new file mode 100644 index 0000000..7aab53f --- /dev/null +++ b/analysis/token/camelcase/camelcase.go @@ -0,0 +1,81 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package camelcase + +import ( + "bytes" + "unicode/utf8" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "camelCase" + +// CamelCaseFilter splits a given token into a set of tokens where each resulting token +// falls into one the following classes: +// 1. Upper case followed by lower case letters. +// Terminated by a number, an upper case letter, and a non alpha-numeric symbol. +// 2. Upper case followed by upper case letters. +// Terminated by a number, an upper case followed by a lower case letter, and a non alpha-numeric symbol. +// 3. Lower case followed by lower case letters. +// Terminated by a number, an upper case letter, and a non alpha-numeric symbol. +// 4. Number followed by numbers. +// Terminated by a letter, and a non alpha-numeric symbol. +// 5. Non alpha-numeric symbol followed by non alpha-numeric symbols. +// Terminated by a number, and a letter. +// +// It does a one-time sequential pass over an input token, from left to right. +// The scan is greedy and generates the longest substring that fits into one of the classes. +// +// See the test file for examples of classes and their parsings. +type CamelCaseFilter struct{} + +func NewCamelCaseFilter() *CamelCaseFilter { + return &CamelCaseFilter{} +} + +func (f *CamelCaseFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + rv := make(analysis.TokenStream, 0, len(input)) + + nextPosition := 1 + for _, token := range input { + runeCount := utf8.RuneCount(token.Term) + runes := bytes.Runes(token.Term) + + p := NewParser(runeCount, nextPosition, token.Start) + for i := 0; i < runeCount; i++ { + if i+1 >= runeCount { + p.Push(runes[i], nil) + } else { + p.Push(runes[i], &runes[i+1]) + } + } + rv = append(rv, p.FlushTokens()...) + nextPosition = p.NextPosition() + } + return rv +} + +func CamelCaseFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewCamelCaseFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(Name, CamelCaseFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/token/camelcase/camelcase_test.go b/analysis/token/camelcase/camelcase_test.go new file mode 100644 index 0000000..9220bc5 --- /dev/null +++ b/analysis/token/camelcase/camelcase_test.go @@ -0,0 +1,95 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package camelcase + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestCamelCaseFilter(t *testing.T) { + + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: tokenStream(""), + output: tokenStream(""), + }, + { + input: tokenStream("a"), + output: tokenStream("a"), + }, + + { + input: tokenStream("...aMACMac123macILoveGolang"), + output: tokenStream("...", "a", "MAC", "Mac", "123", "mac", "I", "Love", "Golang"), + }, + { + input: tokenStream("Lang"), + output: tokenStream("Lang"), + }, + { + input: tokenStream("GLang"), + output: tokenStream("G", "Lang"), + }, + { + input: tokenStream("GOLang"), + output: tokenStream("GO", "Lang"), + }, + { + input: tokenStream("GOOLang"), + output: tokenStream("GOO", "Lang"), + }, + { + input: tokenStream("1234"), + output: tokenStream("1234"), + }, + { + input: tokenStream("starbucks"), + output: tokenStream("starbucks"), + }, + { + input: tokenStream("Starbucks TVSamsungIsGREAT000"), + output: tokenStream("Starbucks", " ", "TV", "Samsung", "Is", "GREAT", "000"), + }, + } + + for _, test := range tests { + ccFilter := NewCamelCaseFilter() + actual := ccFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s \n\n got %s", test.output, actual) + } + } +} + +func tokenStream(termStrs ...string) analysis.TokenStream { + tokenStream := make([]*analysis.Token, len(termStrs)) + index := 0 + for i, termStr := range termStrs { + tokenStream[i] = &analysis.Token{ + Term: []byte(termStr), + Position: i + 1, + Start: index, + End: index + len(termStr), + } + index += len(termStr) + } + return analysis.TokenStream(tokenStream) +} diff --git a/analysis/token/camelcase/parser.go b/analysis/token/camelcase/parser.go new file mode 100644 index 0000000..2d93fc4 --- /dev/null +++ b/analysis/token/camelcase/parser.go @@ -0,0 +1,109 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package camelcase + +import ( + "github.com/blevesearch/bleve/v2/analysis" +) + +func (p *Parser) buildTokenFromTerm(buffer []rune) *analysis.Token { + term := analysis.BuildTermFromRunes(buffer) + token := &analysis.Token{ + Term: term, + Position: p.position, + Start: p.index, + End: p.index + len(term), + } + p.position++ + p.index += len(term) + return token +} + +// Parser accepts a symbol and passes it to the current state (representing a class). +// The state can accept it (and accumulate it). Otherwise, the parser creates a new state that +// starts with the pushed symbol. +// +// Parser accumulates a new resulting token every time it switches state. +// Use FlushTokens() to get the results after the last symbol was pushed. +type Parser struct { + bufferLen int + buffer []rune + current State + tokens []*analysis.Token + position int + index int +} + +func NewParser(length, position, index int) *Parser { + return &Parser{ + bufferLen: length, + buffer: make([]rune, 0, length), + tokens: make([]*analysis.Token, 0, length), + position: position, + index: index, + } +} + +func (p *Parser) Push(sym rune, peek *rune) { + if p.current == nil { + // the start of parsing + p.current = p.NewState(sym) + p.buffer = append(p.buffer, sym) + + } else if p.current.Member(sym, peek) { + // same state, just accumulate + p.buffer = append(p.buffer, sym) + + } else { + // the old state is no more, thus convert the buffer + p.tokens = append(p.tokens, p.buildTokenFromTerm(p.buffer)) + + // let the new state begin + p.current = p.NewState(sym) + p.buffer = make([]rune, 0, p.bufferLen) + p.buffer = append(p.buffer, sym) + } +} + +// Note. States have to have different starting symbols. +func (p *Parser) NewState(sym rune) State { + var found State + + found = &LowerCaseState{} + if found.StartSym(sym) { + return found + } + + found = &UpperCaseState{} + if found.StartSym(sym) { + return found + } + + found = &NumberCaseState{} + if found.StartSym(sym) { + return found + } + + return &NonAlphaNumericCaseState{} +} + +func (p *Parser) FlushTokens() []*analysis.Token { + p.tokens = append(p.tokens, p.buildTokenFromTerm(p.buffer)) + return p.tokens +} + +func (p *Parser) NextPosition() int { + return p.position +} diff --git a/analysis/token/camelcase/states.go b/analysis/token/camelcase/states.go new file mode 100644 index 0000000..97e08d0 --- /dev/null +++ b/analysis/token/camelcase/states.go @@ -0,0 +1,87 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package camelcase + +import ( + "unicode" +) + +// States codify the classes that the parser recognizes. +type State interface { + // is _sym_ the start character + StartSym(sym rune) bool + + // is _sym_ a member of a class. + // peek, the next sym on the tape, can also be used to determine a class. + Member(sym rune, peek *rune) bool +} + +type LowerCaseState struct{} + +func (s *LowerCaseState) Member(sym rune, peek *rune) bool { + return unicode.IsLower(sym) +} + +func (s *LowerCaseState) StartSym(sym rune) bool { + return s.Member(sym, nil) +} + +type UpperCaseState struct { + startedCollecting bool // denotes that the start character has been read + collectingUpper bool // denotes if this is a class of all upper case letters +} + +func (s *UpperCaseState) Member(sym rune, peek *rune) bool { + if !(unicode.IsLower(sym) || unicode.IsUpper(sym)) { + return false + } + + if peek != nil && unicode.IsUpper(sym) && unicode.IsLower(*peek) { + return false + } + + if !s.startedCollecting { + // now we have to determine if upper-case letters are collected. + s.startedCollecting = true + s.collectingUpper = unicode.IsUpper(sym) + return true + } + + return s.collectingUpper == unicode.IsUpper(sym) +} + +func (s *UpperCaseState) StartSym(sym rune) bool { + return unicode.IsUpper(sym) +} + +type NumberCaseState struct{} + +func (s *NumberCaseState) Member(sym rune, peek *rune) bool { + return unicode.IsNumber(sym) +} + +func (s *NumberCaseState) StartSym(sym rune) bool { + return s.Member(sym, nil) +} + +type NonAlphaNumericCaseState struct{} + +func (s *NonAlphaNumericCaseState) Member(sym rune, peek *rune) bool { + return !unicode.IsLower(sym) && !unicode.IsUpper(sym) && !unicode.IsNumber(sym) +} + +func (s *NonAlphaNumericCaseState) StartSym(sym rune) bool { + return s.Member(sym, nil) +} diff --git a/analysis/token/compound/dict.go b/analysis/token/compound/dict.go new file mode 100644 index 0000000..f4fc8f5 --- /dev/null +++ b/analysis/token/compound/dict.go @@ -0,0 +1,144 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compound + +import ( + "bytes" + "fmt" + "unicode/utf8" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "dict_compound" + +const defaultMinWordSize = 5 +const defaultMinSubWordSize = 2 +const defaultMaxSubWordSize = 15 +const defaultOnlyLongestMatch = false + +type DictionaryCompoundFilter struct { + dict analysis.TokenMap + minWordSize int + minSubWordSize int + maxSubWordSize int + onlyLongestMatch bool +} + +func NewDictionaryCompoundFilter(dict analysis.TokenMap, minWordSize, minSubWordSize, maxSubWordSize int, onlyLongestMatch bool) *DictionaryCompoundFilter { + return &DictionaryCompoundFilter{ + dict: dict, + minWordSize: minWordSize, + minSubWordSize: minSubWordSize, + maxSubWordSize: maxSubWordSize, + onlyLongestMatch: onlyLongestMatch, + } +} + +func (f *DictionaryCompoundFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + rv := make(analysis.TokenStream, 0, len(input)) + + for _, token := range input { + rv = append(rv, token) + tokenLen := utf8.RuneCount(token.Term) + if tokenLen >= f.minWordSize { + newtokens := f.decompose(token) + for _, newtoken := range newtokens { + rv = append(rv, newtoken) + } + } + } + + return rv +} + +func (f *DictionaryCompoundFilter) decompose(token *analysis.Token) []*analysis.Token { + runes := bytes.Runes(token.Term) + rv := make([]*analysis.Token, 0) + rlen := len(runes) + for i := 0; i <= (rlen - f.minSubWordSize); i++ { + var longestMatchToken *analysis.Token + for j := f.minSubWordSize; j <= f.maxSubWordSize; j++ { + if i+j > rlen { + break + } + _, inDict := f.dict[string(runes[i:i+j])] + if inDict { + newtoken := analysis.Token{ + Term: []byte(string(runes[i : i+j])), + Position: token.Position, + Start: token.Start + i, + End: token.Start + i + j, + Type: token.Type, + KeyWord: token.KeyWord, + } + if f.onlyLongestMatch { + if longestMatchToken == nil || utf8.RuneCount(longestMatchToken.Term) < j { + longestMatchToken = &newtoken + } + } else { + rv = append(rv, &newtoken) + } + } + } + if f.onlyLongestMatch && longestMatchToken != nil { + rv = append(rv, longestMatchToken) + } + } + return rv +} + +func DictionaryCompoundFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + + minWordSize := defaultMinWordSize + minSubWordSize := defaultMinSubWordSize + maxSubWordSize := defaultMaxSubWordSize + onlyLongestMatch := defaultOnlyLongestMatch + + minVal, ok := config["min_word_size"].(float64) + if ok { + minWordSize = int(minVal) + } + minSubVal, ok := config["min_subword_size"].(float64) + if ok { + minSubWordSize = int(minSubVal) + } + maxSubVal, ok := config["max_subword_size"].(float64) + if ok { + maxSubWordSize = int(maxSubVal) + } + onlyVal, ok := config["only_longest_match"].(bool) + if ok { + onlyLongestMatch = onlyVal + } + + dictTokenMapName, ok := config["dict_token_map"].(string) + if !ok { + return nil, fmt.Errorf("must specify dict_token_map") + } + dictTokenMap, err := cache.TokenMapNamed(dictTokenMapName) + if err != nil { + return nil, fmt.Errorf("error building dict compound words filter: %v", err) + } + return NewDictionaryCompoundFilter(dictTokenMap, minWordSize, minSubWordSize, maxSubWordSize, onlyLongestMatch), nil +} + +func init() { + err := registry.RegisterTokenFilter(Name, DictionaryCompoundFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/token/compound/dict_test.go b/analysis/token/compound/dict_test.go new file mode 100644 index 0000000..8d9b093 --- /dev/null +++ b/analysis/token/compound/dict_test.go @@ -0,0 +1,187 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compound + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/tokenmap" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestStopWordsFilter(t *testing.T) { + + inputTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("i"), + Start: 0, + End: 1, + Position: 1, + }, + &analysis.Token{ + Term: []byte("like"), + Start: 2, + End: 6, + Position: 2, + }, + &analysis.Token{ + Term: []byte("to"), + Start: 7, + End: 9, + Position: 3, + }, + &analysis.Token{ + Term: []byte("play"), + Start: 10, + End: 14, + Position: 4, + }, + &analysis.Token{ + Term: []byte("softball"), + Start: 15, + End: 23, + Position: 5, + }, + } + + expectedTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("i"), + Start: 0, + End: 1, + Position: 1, + }, + &analysis.Token{ + Term: []byte("like"), + Start: 2, + End: 6, + Position: 2, + }, + &analysis.Token{ + Term: []byte("to"), + Start: 7, + End: 9, + Position: 3, + }, + &analysis.Token{ + Term: []byte("play"), + Start: 10, + End: 14, + Position: 4, + }, + &analysis.Token{ + Term: []byte("softball"), + Start: 15, + End: 23, + Position: 5, + }, + &analysis.Token{ + Term: []byte("soft"), + Start: 15, + End: 19, + Position: 5, + }, + &analysis.Token{ + Term: []byte("ball"), + Start: 19, + End: 23, + Position: 5, + }, + } + + cache := registry.NewCache() + dictListConfig := map[string]interface{}{ + "type": tokenmap.Name, + "tokens": []interface{}{"factor", "soft", "ball", "team"}, + } + _, err := cache.DefineTokenMap("dict_test", dictListConfig) + if err != nil { + t.Fatal(err) + } + + dictConfig := map[string]interface{}{ + "type": "dict_compound", + "dict_token_map": "dict_test", + } + dictFilter, err := cache.DefineTokenFilter("dict_test", dictConfig) + if err != nil { + t.Fatal(err) + } + + ouputTokenStream := dictFilter.Filter(inputTokenStream) + if !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) { + t.Errorf("expected %#v got %#v", expectedTokenStream, ouputTokenStream) + } +} + +func TestStopWordsFilterLongestMatch(t *testing.T) { + + inputTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("softestball"), + Start: 0, + End: 11, + Position: 1, + }, + } + + expectedTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("softestball"), + Start: 0, + End: 11, + Position: 1, + }, + &analysis.Token{ + Term: []byte("softest"), + Start: 0, + End: 7, + Position: 1, + }, + &analysis.Token{ + Term: []byte("ball"), + Start: 7, + End: 11, + Position: 1, + }, + } + + cache := registry.NewCache() + dictListConfig := map[string]interface{}{ + "type": tokenmap.Name, + "tokens": []interface{}{"soft", "softest", "ball"}, + } + _, err := cache.DefineTokenMap("dict_test", dictListConfig) + if err != nil { + t.Fatal(err) + } + + dictConfig := map[string]interface{}{ + "type": "dict_compound", + "dict_token_map": "dict_test", + "only_longest_match": true, + } + dictFilter, err := cache.DefineTokenFilter("dict_test", dictConfig) + if err != nil { + t.Fatal(err) + } + + ouputTokenStream := dictFilter.Filter(inputTokenStream) + if !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) { + t.Errorf("expected %#v got %#v", expectedTokenStream, ouputTokenStream) + } +} diff --git a/analysis/token/edgengram/edgengram.go b/analysis/token/edgengram/edgengram.go new file mode 100644 index 0000000..d9e4c24 --- /dev/null +++ b/analysis/token/edgengram/edgengram.go @@ -0,0 +1,118 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package edgengram + +import ( + "bytes" + "fmt" + "unicode/utf8" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "edge_ngram" + +type Side bool + +const BACK Side = true +const FRONT Side = false + +type EdgeNgramFilter struct { + back Side + minLength int + maxLength int +} + +func NewEdgeNgramFilter(side Side, minLength, maxLength int) *EdgeNgramFilter { + return &EdgeNgramFilter{ + back: side, + minLength: minLength, + maxLength: maxLength, + } +} + +func (s *EdgeNgramFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + rv := make(analysis.TokenStream, 0, len(input)) + + for _, token := range input { + runeCount := utf8.RuneCount(token.Term) + runes := bytes.Runes(token.Term) + if s.back { + i := runeCount + // index of the starting rune for this token + for ngramSize := s.minLength; ngramSize <= s.maxLength; ngramSize++ { + // build an ngram of this size starting at i + if i-ngramSize >= 0 { + ngramTerm := analysis.BuildTermFromRunes(runes[i-ngramSize : i]) + token := analysis.Token{ + Position: token.Position, + Start: token.Start, + End: token.End, + Type: token.Type, + Term: ngramTerm, + } + rv = append(rv, &token) + } + } + } else { + i := 0 + // index of the starting rune for this token + for ngramSize := s.minLength; ngramSize <= s.maxLength; ngramSize++ { + // build an ngram of this size starting at i + if i+ngramSize <= runeCount { + ngramTerm := analysis.BuildTermFromRunes(runes[i : i+ngramSize]) + token := analysis.Token{ + Position: token.Position, + Start: token.Start, + End: token.End, + Type: token.Type, + Term: ngramTerm, + } + rv = append(rv, &token) + } + } + } + } + + return rv +} + +func EdgeNgramFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + side := FRONT + back, ok := config["back"].(bool) + if ok && back { + side = BACK + } + minVal, ok := config["min"].(float64) + if !ok { + return nil, fmt.Errorf("must specify min") + } + min := int(minVal) + maxVal, ok := config["max"].(float64) + if !ok { + return nil, fmt.Errorf("must specify max") + } + max := int(maxVal) + + return NewEdgeNgramFilter(side, min, max), nil +} + +func init() { + err := registry.RegisterTokenFilter(Name, EdgeNgramFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/token/edgengram/edgengram_test.go b/analysis/token/edgengram/edgengram_test.go new file mode 100644 index 0000000..cb206a5 --- /dev/null +++ b/analysis/token/edgengram/edgengram_test.go @@ -0,0 +1,189 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package edgengram + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestEdgeNgramFilter(t *testing.T) { + + tests := []struct { + side Side + min int + max int + input analysis.TokenStream + output analysis.TokenStream + }{ + { + side: FRONT, + min: 1, + max: 1, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abcde"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a"), + }, + }, + }, + { + side: BACK, + min: 1, + max: 1, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abcde"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("e"), + }, + }, + }, + { + side: FRONT, + min: 1, + max: 3, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abcde"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a"), + }, + &analysis.Token{ + Term: []byte("ab"), + }, + &analysis.Token{ + Term: []byte("abc"), + }, + }, + }, + { + side: BACK, + min: 1, + max: 3, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abcde"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("e"), + }, + &analysis.Token{ + Term: []byte("de"), + }, + &analysis.Token{ + Term: []byte("cde"), + }, + }, + }, + { + side: FRONT, + min: 1, + max: 3, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abcde"), + }, + &analysis.Token{ + Term: []byte("vwxyz"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a"), + }, + &analysis.Token{ + Term: []byte("ab"), + }, + &analysis.Token{ + Term: []byte("abc"), + }, + &analysis.Token{ + Term: []byte("v"), + }, + &analysis.Token{ + Term: []byte("vw"), + }, + &analysis.Token{ + Term: []byte("vwx"), + }, + }, + }, + { + side: BACK, + min: 3, + max: 5, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Beryl"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ryl"), + }, + &analysis.Token{ + Term: []byte("eryl"), + }, + &analysis.Token{ + Term: []byte("Beryl"), + }, + }, + }, + { + side: FRONT, + min: 3, + max: 5, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Beryl"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Ber"), + }, + &analysis.Token{ + Term: []byte("Bery"), + }, + &analysis.Token{ + Term: []byte("Beryl"), + }, + }, + }, + } + + for _, test := range tests { + edgeNgramFilter := NewEdgeNgramFilter(test.side, test.min, test.max) + actual := edgeNgramFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output, actual) + } + } +} diff --git a/analysis/token/elision/elision.go b/analysis/token/elision/elision.go new file mode 100644 index 0000000..721a546 --- /dev/null +++ b/analysis/token/elision/elision.go @@ -0,0 +1,77 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package elision + +import ( + "fmt" + "unicode/utf8" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "elision" + +const RightSingleQuotationMark = '’' +const Apostrophe = '\'' + +type ElisionFilter struct { + articles analysis.TokenMap +} + +func NewElisionFilter(articles analysis.TokenMap) *ElisionFilter { + return &ElisionFilter{ + articles: articles, + } +} + +func (s *ElisionFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + term := token.Term + for i := 0; i < len(term); { + r, size := utf8.DecodeRune(term[i:]) + if r == Apostrophe || r == RightSingleQuotationMark { + // see if the prefix matches one of the articles + prefix := term[0:i] + _, articleMatch := s.articles[string(prefix)] + if articleMatch { + token.Term = term[i+size:] + break + } + } + i += size + } + } + return input +} + +func ElisionFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + articlesTokenMapName, ok := config["articles_token_map"].(string) + if !ok { + return nil, fmt.Errorf("must specify articles_token_map") + } + articlesTokenMap, err := cache.TokenMapNamed(articlesTokenMapName) + if err != nil { + return nil, fmt.Errorf("error building elision filter: %v", err) + } + return NewElisionFilter(articlesTokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(Name, ElisionFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/token/elision/elision_test.go b/analysis/token/elision/elision_test.go new file mode 100644 index 0000000..ed580b0 --- /dev/null +++ b/analysis/token/elision/elision_test.go @@ -0,0 +1,85 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package elision + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/tokenmap" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestElisionFilter(t *testing.T) { + + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ar" + string(Apostrophe) + "word"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("word"), + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ar" + string(RightSingleQuotationMark) + "word"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("word"), + }, + }, + }, + } + + cache := registry.NewCache() + + articleListConfig := map[string]interface{}{ + "type": tokenmap.Name, + "tokens": []interface{}{"ar"}, + } + _, err := cache.DefineTokenMap("articles_test", articleListConfig) + if err != nil { + t.Fatal(err) + } + + elisionConfig := map[string]interface{}{ + "type": "elision", + "articles_token_map": "articles_test", + } + elisionFilter, err := cache.DefineTokenFilter("elision_test", elisionConfig) + if err != nil { + t.Fatal(err) + } + + for _, test := range tests { + + actual := elisionFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/token/hierarchy/hierarchy.go b/analysis/token/hierarchy/hierarchy.go new file mode 100644 index 0000000..6e1ea44 --- /dev/null +++ b/analysis/token/hierarchy/hierarchy.go @@ -0,0 +1,95 @@ +package hierarchy + +import ( + "bytes" + "fmt" + "math" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "hierarchy" + +type HierarchyFilter struct { + maxLevels int + delimiter []byte + splitInput bool +} + +func NewHierarchyFilter(delimiter []byte, maxLevels int, splitInput bool) *HierarchyFilter { + return &HierarchyFilter{ + maxLevels: maxLevels, + delimiter: delimiter, + splitInput: splitInput, + } +} + +func (s *HierarchyFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + rv := make(analysis.TokenStream, 0, s.maxLevels) + + var soFar [][]byte + for _, token := range input { + if s.splitInput { + parts := bytes.Split(token.Term, s.delimiter) + for _, part := range parts { + soFar, rv = s.buildToken(rv, soFar, part) + if len(soFar) >= s.maxLevels { + return rv + } + } + } else { + soFar, rv = s.buildToken(rv, soFar, token.Term) + if len(soFar) >= s.maxLevels { + return rv + } + } + } + + return rv +} + +func (s *HierarchyFilter) buildToken(tokenStream analysis.TokenStream, soFar [][]byte, part []byte) ( + [][]byte, analysis.TokenStream) { + + soFar = append(soFar, part) + term := bytes.Join(soFar, s.delimiter) + + tokenStream = append(tokenStream, &analysis.Token{ + Type: analysis.Shingle, + Term: term, + Start: 0, + End: len(term), + Position: 1, + }) + + return soFar, tokenStream +} + +func HierarchyFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + max := math.MaxInt64 + maxVal, ok := config["max"].(float64) + if ok { + max = int(maxVal) + } + + splitInput := true + splitInputVal, ok := config["split_input"].(bool) + if ok { + splitInput = splitInputVal + } + + delimiter, ok := config["delimiter"].(string) + if !ok { + return nil, fmt.Errorf("must specify delimiter") + } + + return NewHierarchyFilter([]byte(delimiter), max, splitInput), nil +} + +func init() { + err := registry.RegisterTokenFilter(Name, HierarchyFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/token/hierarchy/hierarchy_test.go b/analysis/token/hierarchy/hierarchy_test.go new file mode 100644 index 0000000..74d24bf --- /dev/null +++ b/analysis/token/hierarchy/hierarchy_test.go @@ -0,0 +1,229 @@ +package hierarchy + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestHierarchyFilter(t *testing.T) { + + tests := []struct { + name string + delimiter string + max int + splitInput bool + + input analysis.TokenStream + output analysis.TokenStream + }{ + { + name: "single token a/b/c, delimiter /", + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a/b/c"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a"), + Type: analysis.Shingle, + Start: 0, + End: 1, + Position: 1, + }, + &analysis.Token{ + Term: []byte("a/b"), + Type: analysis.Shingle, + Start: 0, + End: 3, + Position: 1, + }, + &analysis.Token{ + Term: []byte("a/b/c"), + Type: analysis.Shingle, + Start: 0, + End: 5, + Position: 1, + }, + }, + delimiter: "/", + max: 10, + splitInput: true, + }, + { + name: "multiple tokens already split a b c, delimiter /", + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a"), + }, + &analysis.Token{ + Term: []byte("b"), + }, + &analysis.Token{ + Term: []byte("c"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a"), + Type: analysis.Shingle, + Start: 0, + End: 1, + Position: 1, + }, + &analysis.Token{ + Term: []byte("a/b"), + Type: analysis.Shingle, + Start: 0, + End: 3, + Position: 1, + }, + &analysis.Token{ + Term: []byte("a/b/c"), + Type: analysis.Shingle, + Start: 0, + End: 5, + Position: 1, + }, + }, + delimiter: "/", + max: 10, + splitInput: true, + }, + { + name: "single token a/b/c, delimiter /, limit 2", + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a/b/c"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a"), + Type: analysis.Shingle, + Start: 0, + End: 1, + Position: 1, + }, + &analysis.Token{ + Term: []byte("a/b"), + Type: analysis.Shingle, + Start: 0, + End: 3, + Position: 1, + }, + }, + delimiter: "/", + max: 2, + splitInput: true, + }, + { + name: "multiple tokens already split a b c, delimiter /, limit 2", + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a"), + }, + &analysis.Token{ + Term: []byte("b"), + }, + &analysis.Token{ + Term: []byte("c"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a"), + Type: analysis.Shingle, + Start: 0, + End: 1, + Position: 1, + }, + &analysis.Token{ + Term: []byte("a/b"), + Type: analysis.Shingle, + Start: 0, + End: 3, + Position: 1, + }, + }, + delimiter: "/", + max: 2, + splitInput: true, + }, + + { + name: "single token a/b/c, delimiter /, no split", + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a/b/c"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a/b/c"), + Type: analysis.Shingle, + Start: 0, + End: 5, + Position: 1, + }, + }, + delimiter: "/", + max: 10, + splitInput: false, + }, + { + name: "multiple tokens already split a b c, delimiter /, no split", + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a"), + }, + &analysis.Token{ + Term: []byte("b"), + }, + &analysis.Token{ + Term: []byte("c"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a"), + Type: analysis.Shingle, + Start: 0, + End: 1, + Position: 1, + }, + &analysis.Token{ + Term: []byte("a/b"), + Type: analysis.Shingle, + Start: 0, + End: 3, + Position: 1, + }, + &analysis.Token{ + Term: []byte("a/b/c"), + Type: analysis.Shingle, + Start: 0, + End: 5, + Position: 1, + }, + }, + delimiter: "/", + max: 10, + splitInput: false, + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + filter := NewHierarchyFilter([]byte(test.delimiter), test.max, test.splitInput) + actual := filter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output, actual) + } + }) + } + +} diff --git a/analysis/token/keyword/keyword.go b/analysis/token/keyword/keyword.go new file mode 100644 index 0000000..f2e56f3 --- /dev/null +++ b/analysis/token/keyword/keyword.go @@ -0,0 +1,63 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyword + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "keyword_marker" + +type KeyWordMarkerFilter struct { + keyWords analysis.TokenMap +} + +func NewKeyWordMarkerFilter(keyWords analysis.TokenMap) *KeyWordMarkerFilter { + return &KeyWordMarkerFilter{ + keyWords: keyWords, + } +} + +func (f *KeyWordMarkerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + _, isKeyWord := f.keyWords[string(token.Term)] + if isKeyWord { + token.KeyWord = true + } + } + return input +} + +func KeyWordMarkerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + keywordsTokenMapName, ok := config["keywords_token_map"].(string) + if !ok { + return nil, fmt.Errorf("must specify keywords_token_map") + } + keywordsTokenMap, err := cache.TokenMapNamed(keywordsTokenMapName) + if err != nil { + return nil, fmt.Errorf("error building keyword marker filter: %v", err) + } + return NewKeyWordMarkerFilter(keywordsTokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(Name, KeyWordMarkerFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/token/keyword/keyword_test.go b/analysis/token/keyword/keyword_test.go new file mode 100644 index 0000000..8b789f1 --- /dev/null +++ b/analysis/token/keyword/keyword_test.go @@ -0,0 +1,73 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyword + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestKeyWordMarkerFilter(t *testing.T) { + + inputTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a"), + }, + &analysis.Token{ + Term: []byte("walk"), + }, + &analysis.Token{ + Term: []byte("in"), + }, + &analysis.Token{ + Term: []byte("the"), + }, + &analysis.Token{ + Term: []byte("park"), + }, + } + + expectedTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a"), + }, + &analysis.Token{ + Term: []byte("walk"), + KeyWord: true, + }, + &analysis.Token{ + Term: []byte("in"), + }, + &analysis.Token{ + Term: []byte("the"), + }, + &analysis.Token{ + Term: []byte("park"), + KeyWord: true, + }, + } + + keyWordsMap := analysis.NewTokenMap() + keyWordsMap.AddToken("walk") + keyWordsMap.AddToken("park") + + filter := NewKeyWordMarkerFilter(keyWordsMap) + ouputTokenStream := filter.Filter(inputTokenStream) + if !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) { + t.Errorf("expected %#v got %#v", expectedTokenStream[0].KeyWord, ouputTokenStream[0].KeyWord) + } +} diff --git a/analysis/token/length/length.go b/analysis/token/length/length.go new file mode 100644 index 0000000..1091270 --- /dev/null +++ b/analysis/token/length/length.go @@ -0,0 +1,80 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package length + +import ( + "fmt" + "unicode/utf8" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "length" + +type LengthFilter struct { + min int + max int +} + +func NewLengthFilter(min, max int) *LengthFilter { + return &LengthFilter{ + min: min, + max: max, + } +} + +func (f *LengthFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + rv := make(analysis.TokenStream, 0, len(input)) + + for _, token := range input { + wordLen := utf8.RuneCount(token.Term) + if f.min > 0 && f.min > wordLen { + continue + } + if f.max > 0 && f.max < wordLen { + continue + } + rv = append(rv, token) + } + + return rv +} + +func LengthFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + min := 0 + max := 0 + + minVal, ok := config["min"].(float64) + if ok { + min = int(minVal) + } + maxVal, ok := config["max"].(float64) + if ok { + max = int(maxVal) + } + if min == max && max == 0 { + return nil, fmt.Errorf("either min or max must be non-zero") + } + + return NewLengthFilter(min, max), nil +} + +func init() { + err := registry.RegisterTokenFilter(Name, LengthFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/token/length/length_test.go b/analysis/token/length/length_test.go new file mode 100644 index 0000000..b3cb064 --- /dev/null +++ b/analysis/token/length/length_test.go @@ -0,0 +1,99 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package length + +import ( + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestLengthFilter(t *testing.T) { + + inputTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("1"), + }, + &analysis.Token{ + Term: []byte("two"), + }, + &analysis.Token{ + Term: []byte("three"), + }, + } + + lengthFilter := NewLengthFilter(3, 4) + ouputTokenStream := lengthFilter.Filter(inputTokenStream) + if len(ouputTokenStream) != 1 { + t.Fatalf("expected 1 output token") + } + if string(ouputTokenStream[0].Term) != "two" { + t.Errorf("expected term `two`, got `%s`", ouputTokenStream[0].Term) + } +} + +func TestLengthFilterNoMax(t *testing.T) { + + inputTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("1"), + }, + &analysis.Token{ + Term: []byte("two"), + }, + &analysis.Token{ + Term: []byte("three"), + }, + } + + lengthFilter := NewLengthFilter(3, -1) + ouputTokenStream := lengthFilter.Filter(inputTokenStream) + if len(ouputTokenStream) != 2 { + t.Fatalf("expected 2 output token") + } + if string(ouputTokenStream[0].Term) != "two" { + t.Errorf("expected term `two`, got `%s`", ouputTokenStream[0].Term) + } + if string(ouputTokenStream[1].Term) != "three" { + t.Errorf("expected term `three`, got `%s`", ouputTokenStream[0].Term) + } +} + +func TestLengthFilterNoMin(t *testing.T) { + + inputTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("1"), + }, + &analysis.Token{ + Term: []byte("two"), + }, + &analysis.Token{ + Term: []byte("three"), + }, + } + + lengthFilter := NewLengthFilter(-1, 4) + ouputTokenStream := lengthFilter.Filter(inputTokenStream) + if len(ouputTokenStream) != 2 { + t.Fatalf("expected 2 output token") + } + if string(ouputTokenStream[0].Term) != "1" { + t.Errorf("expected term `1`, got `%s`", ouputTokenStream[0].Term) + } + if string(ouputTokenStream[1].Term) != "two" { + t.Errorf("expected term `two`, got `%s`", ouputTokenStream[0].Term) + } +} diff --git a/analysis/token/lowercase/lowercase.go b/analysis/token/lowercase/lowercase.go new file mode 100644 index 0000000..92b9628 --- /dev/null +++ b/analysis/token/lowercase/lowercase.go @@ -0,0 +1,108 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package lowercase implements a TokenFilter which converts +// tokens to lower case according to unicode rules. +package lowercase + +import ( + "bytes" + "unicode" + "unicode/utf8" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +// Name is the name used to register LowerCaseFilter in the bleve registry +const Name = "to_lower" + +type LowerCaseFilter struct { +} + +func NewLowerCaseFilter() *LowerCaseFilter { + return &LowerCaseFilter{} +} + +func (f *LowerCaseFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + token.Term = toLowerDeferredCopy(token.Term) + } + return input +} + +func LowerCaseFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewLowerCaseFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(Name, LowerCaseFilterConstructor) + if err != nil { + panic(err) + } +} + +// toLowerDeferredCopy will function exactly like +// bytes.ToLower() only it will reuse (overwrite) +// the original byte array when possible +// NOTE: because its possible that the lower-case +// form of a rune has a different utf-8 encoded +// length, in these cases a new byte array is allocated +func toLowerDeferredCopy(s []byte) []byte { + j := 0 + for i := 0; i < len(s); { + wid := 1 + r := rune(s[i]) + if r >= utf8.RuneSelf { + r, wid = utf8.DecodeRune(s[i:]) + } + + l := unicode.ToLower(r) + + // If the rune is already lowercased, just move to the + // next rune. + if l == r { + i += wid + j += wid + continue + } + + // Handles the Unicode edge-case where the last + // rune in a word on the greek Σ needs to be converted + // differently. + if l == 'σ' && i+2 == len(s) { + l = 'ς' + } + + lwid := utf8.RuneLen(l) + if lwid > wid { + // utf-8 encoded replacement is wider + // for now, punt and defer + // to bytes.ToLower() for the remainder + // only known to happen with chars + // Rune Ⱥ(570) width 2 - Lower ⱥ(11365) width 3 + // Rune Ⱦ(574) width 2 - Lower ⱦ(11366) width 3 + rest := bytes.ToLower(s[i:]) + rv := make([]byte, j+len(rest)) + copy(rv[:j], s[:j]) + copy(rv[j:], rest) + return rv + } else { + utf8.EncodeRune(s[j:], l) + } + i += wid + j += lwid + } + return s[:j] +} diff --git a/analysis/token/lowercase/lowercase_test.go b/analysis/token/lowercase/lowercase_test.go new file mode 100644 index 0000000..95fd2af --- /dev/null +++ b/analysis/token/lowercase/lowercase_test.go @@ -0,0 +1,166 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lowercase + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestLowerCaseFilter(t *testing.T) { + + inputTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ONE"), + }, + &analysis.Token{ + Term: []byte("two"), + }, + &analysis.Token{ + Term: []byte("ThReE"), + }, + &analysis.Token{ + Term: []byte("steven's"), + }, + // these characters are chosen in particular + // because the utf-8 encoding of the lower-case + // version has a different length + // Rune İ(304) width 2 - Lower i(105) width 1 + // Rune Ⱥ(570) width 2 - Lower ⱥ(11365) width 3 + // Rune Ⱦ(574) width 2 - Lower ⱦ(11366) width 3 + &analysis.Token{ + Term: []byte("İȺȾCAT"), + }, + &analysis.Token{ + Term: []byte("ȺȾCAT"), + }, + &analysis.Token{ + Term: []byte("ὈΔΥΣΣ"), + }, + } + + expectedTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("one"), + }, + &analysis.Token{ + Term: []byte("two"), + }, + &analysis.Token{ + Term: []byte("three"), + }, + &analysis.Token{ + Term: []byte("steven's"), + }, + &analysis.Token{ + Term: []byte("iⱥⱦcat"), + }, + &analysis.Token{ + Term: []byte("ⱥⱦcat"), + }, + &analysis.Token{ + Term: []byte("ὀδυσς"), + }, + } + + filter := NewLowerCaseFilter() + ouputTokenStream := filter.Filter(inputTokenStream) + if !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) { + t.Errorf("expected %#v got %#v", expectedTokenStream, ouputTokenStream) + t.Errorf("expected %s got %s", expectedTokenStream[0].Term, ouputTokenStream[0].Term) + } +} + +func BenchmarkLowerCaseFilter(b *testing.B) { + input := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("A"), + }, + &analysis.Token{ + Term: []byte("boiling"), + }, + &analysis.Token{ + Term: []byte("liquid"), + }, + &analysis.Token{ + Term: []byte("expanding"), + }, + &analysis.Token{ + Term: []byte("vapor"), + }, + &analysis.Token{ + Term: []byte("explosion"), + }, + &analysis.Token{ + Term: []byte("caused"), + }, + &analysis.Token{ + Term: []byte("by"), + }, + &analysis.Token{ + Term: []byte("the"), + }, + &analysis.Token{ + Term: []byte("rupture"), + }, + &analysis.Token{ + Term: []byte("of"), + }, + &analysis.Token{ + Term: []byte("a"), + }, + &analysis.Token{ + Term: []byte("vessel"), + }, + &analysis.Token{ + Term: []byte("containing"), + }, + &analysis.Token{ + Term: []byte("a"), + }, + &analysis.Token{ + Term: []byte("pressurized"), + }, + &analysis.Token{ + Term: []byte("liquid"), + }, + &analysis.Token{ + Term: []byte("above"), + }, + &analysis.Token{ + Term: []byte("its"), + }, + &analysis.Token{ + Term: []byte("boiling"), + }, + &analysis.Token{ + Term: []byte("point"), + }, + &analysis.Token{ + Term: []byte("İȺȾCAT"), + }, + &analysis.Token{ + Term: []byte("ȺȾCAT"), + }, + } + filter := NewLowerCaseFilter() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + filter.Filter(input) + } +} diff --git a/analysis/token/ngram/ngram.go b/analysis/token/ngram/ngram.go new file mode 100644 index 0000000..d8967a0 --- /dev/null +++ b/analysis/token/ngram/ngram.go @@ -0,0 +1,116 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ngram + +import ( + "bytes" + "fmt" + "unicode/utf8" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "ngram" + +type NgramFilter struct { + minLength int + maxLength int +} + +func NewNgramFilter(minLength, maxLength int) *NgramFilter { + return &NgramFilter{ + minLength: minLength, + maxLength: maxLength, + } +} + +func (s *NgramFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + rv := make(analysis.TokenStream, 0, len(input)) + + for _, token := range input { + runeCount := utf8.RuneCount(token.Term) + runes := bytes.Runes(token.Term) + for i := 0; i < runeCount; i++ { + // index of the starting rune for this token + for ngramSize := s.minLength; ngramSize <= s.maxLength; ngramSize++ { + // build an ngram of this size starting at i + if i+ngramSize <= runeCount { + ngramTerm := analysis.BuildTermFromRunes(runes[i : i+ngramSize]) + token := analysis.Token{ + Position: token.Position, + Start: token.Start, + End: token.End, + Type: token.Type, + Term: ngramTerm, + } + rv = append(rv, &token) + } + } + } + } + + return rv +} + +func NgramFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + minVal, ok := config["min"] + if !ok { + return nil, fmt.Errorf("must specify min") + } + + min, err := convertToInt(minVal) + if err != nil { + return nil, err + } + + maxVal, ok := config["max"] + if !ok { + return nil, fmt.Errorf("must specify max") + } + + max, err := convertToInt(maxVal) + if err != nil { + return nil, err + } + + return NewNgramFilter(min, max), nil +} + +func init() { + err := registry.RegisterTokenFilter(Name, NgramFilterConstructor) + if err != nil { + panic(err) + } +} + +// Expects either an int or a flaot64 value +func convertToInt(val interface{}) (int, error) { + var intVal int + var floatVal float64 + var ok bool + + intVal, ok = val.(int) + if ok { + return intVal, nil + } + + floatVal, ok = val.(float64) + if ok { + return int(floatVal), nil + } + + return 0, fmt.Errorf("failed to convert to int value") +} diff --git a/analysis/token/ngram/ngram_test.go b/analysis/token/ngram/ngram_test.go new file mode 100644 index 0000000..877c881 --- /dev/null +++ b/analysis/token/ngram/ngram_test.go @@ -0,0 +1,192 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ngram + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestNgramFilter(t *testing.T) { + + tests := []struct { + min int + max int + input analysis.TokenStream + output analysis.TokenStream + }{ + { + min: 1, + max: 1, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abcde"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a"), + }, + &analysis.Token{ + Term: []byte("b"), + }, + &analysis.Token{ + Term: []byte("c"), + }, + &analysis.Token{ + Term: []byte("d"), + }, + &analysis.Token{ + Term: []byte("e"), + }, + }, + }, + { + min: 2, + max: 2, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abcde"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ab"), + }, + &analysis.Token{ + Term: []byte("bc"), + }, + &analysis.Token{ + Term: []byte("cd"), + }, + &analysis.Token{ + Term: []byte("de"), + }, + }, + }, + { + min: 1, + max: 3, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abcde"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a"), + }, + &analysis.Token{ + Term: []byte("ab"), + }, + &analysis.Token{ + Term: []byte("abc"), + }, + &analysis.Token{ + Term: []byte("b"), + }, + &analysis.Token{ + Term: []byte("bc"), + }, + &analysis.Token{ + Term: []byte("bcd"), + }, + &analysis.Token{ + Term: []byte("c"), + }, + &analysis.Token{ + Term: []byte("cd"), + }, + &analysis.Token{ + Term: []byte("cde"), + }, + &analysis.Token{ + Term: []byte("d"), + }, + &analysis.Token{ + Term: []byte("de"), + }, + &analysis.Token{ + Term: []byte("e"), + }, + }, + }, + } + + for _, test := range tests { + ngramFilter := NewNgramFilter(test.min, test.max) + actual := ngramFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output, actual) + } + } +} + +func TestConversionInt(t *testing.T) { + config := map[string]interface{}{ + "type": Name, + "min": 3, + "max": 8, + } + + f, err := NgramFilterConstructor(config, nil) + + if err != nil { + t.Errorf("Failed to construct the ngram filter: %v", err) + } + + ngram := f.(*NgramFilter) + if ngram.minLength != 3 && ngram.maxLength != 8 { + t.Errorf("Failed to construct the bounds. Got %v and %v.", ngram.minLength, ngram.maxLength) + } +} + +func TestConversionFloat(t *testing.T) { + config := map[string]interface{}{ + "type": Name, + "min": float64(3), + "max": float64(8), + } + + f, err := NgramFilterConstructor(config, nil) + + if err != nil { + t.Errorf("Failed to construct the ngram filter: %v", err) + } + + ngram := f.(*NgramFilter) + if ngram.minLength != 3 && ngram.maxLength != 8 { + t.Errorf("Failed to construct the bounds. Got %v and %v.", ngram.minLength, ngram.maxLength) + } +} + +func TestBadConversion(t *testing.T) { + config := map[string]interface{}{ + "type": Name, + "min": "3", + } + + _, err := NgramFilterConstructor(config, nil) + + if err == nil { + t.Errorf("Expected conversion error.") + } + + if err.Error() != "failed to convert to int value" { + t.Errorf("Wrong error recevied. Got %v.", err) + } +} diff --git a/analysis/token/porter/porter.go b/analysis/token/porter/porter.go new file mode 100644 index 0000000..ed1574b --- /dev/null +++ b/analysis/token/porter/porter.go @@ -0,0 +1,56 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package porter + +import ( + "bytes" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/go-porterstemmer" +) + +const Name = "stemmer_porter" + +type PorterStemmer struct { +} + +func NewPorterStemmer() *PorterStemmer { + return &PorterStemmer{} +} + +func (s *PorterStemmer) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + // if it is not a protected keyword, stem it + if !token.KeyWord { + termRunes := bytes.Runes(token.Term) + stemmedRunes := porterstemmer.StemWithoutLowerCasing(termRunes) + token.Term = analysis.BuildTermFromRunes(stemmedRunes) + } + } + return input +} + +func PorterStemmerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewPorterStemmer(), nil +} + +func init() { + err := registry.RegisterTokenFilter(Name, PorterStemmerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/token/porter/porter_test.go b/analysis/token/porter/porter_test.go new file mode 100644 index 0000000..3e0a79f --- /dev/null +++ b/analysis/token/porter/porter_test.go @@ -0,0 +1,115 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package porter + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestPorterStemmer(t *testing.T) { + + inputTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("walking"), + }, + &analysis.Token{ + Term: []byte("talked"), + }, + &analysis.Token{ + Term: []byte("business"), + }, + &analysis.Token{ + Term: []byte("protected"), + KeyWord: true, + }, + &analysis.Token{ + Term: []byte("cat"), + }, + &analysis.Token{ + Term: []byte("done"), + }, + // a term which does stem, but does not change length + &analysis.Token{ + Term: []byte("marty"), + }, + } + + expectedTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("walk"), + }, + &analysis.Token{ + Term: []byte("talk"), + }, + &analysis.Token{ + Term: []byte("busi"), + }, + &analysis.Token{ + Term: []byte("protected"), + KeyWord: true, + }, + &analysis.Token{ + Term: []byte("cat"), + }, + &analysis.Token{ + Term: []byte("done"), + }, + &analysis.Token{ + Term: []byte("marti"), + }, + } + + filter := NewPorterStemmer() + ouputTokenStream := filter.Filter(inputTokenStream) + if !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) { + t.Errorf("expected %#v got %#v", expectedTokenStream[3], ouputTokenStream[3]) + } +} + +func BenchmarkPorterStemmer(b *testing.B) { + + inputTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("walking"), + }, + &analysis.Token{ + Term: []byte("talked"), + }, + &analysis.Token{ + Term: []byte("business"), + }, + &analysis.Token{ + Term: []byte("protected"), + KeyWord: true, + }, + &analysis.Token{ + Term: []byte("cat"), + }, + &analysis.Token{ + Term: []byte("done"), + }, + } + + filter := NewPorterStemmer() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + filter.Filter(inputTokenStream) + } + +} diff --git a/analysis/token/reverse/reverse.go b/analysis/token/reverse/reverse.go new file mode 100644 index 0000000..5eb708f --- /dev/null +++ b/analysis/token/reverse/reverse.go @@ -0,0 +1,78 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package reverse + +import ( + "unicode" + "unicode/utf8" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +// Name is the name used to register ReverseFilter in the bleve registry +const Name = "reverse" + +type ReverseFilter struct { +} + +func NewReverseFilter() *ReverseFilter { + return &ReverseFilter{} +} + +func (f *ReverseFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + token.Term = reverse(token.Term) + } + return input +} + +func ReverseFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewReverseFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(Name, ReverseFilterConstructor) + if err != nil { + panic(err) + } +} + +// reverse(..) will generate a reversed version of the provided +// unicode array and return it back to its caller. +func reverse(s []byte) []byte { + cursorIn := 0 + inputRunes := []rune(string(s)) + cursorOut := len(s) + output := make([]byte, len(s)) + for i := 0; i < len(inputRunes); { + wid := utf8.RuneLen(inputRunes[i]) + i++ + for i < len(inputRunes) { + r := inputRunes[i] + if unicode.Is(unicode.Mn, r) || unicode.Is(unicode.Me, r) || unicode.Is(unicode.Mc, r) { + wid += utf8.RuneLen(r) + i++ + } else { + break + } + } + copy(output[cursorOut-wid:cursorOut], s[cursorIn:cursorIn+wid]) + cursorIn += wid + cursorOut -= wid + } + + return output +} diff --git a/analysis/token/reverse/reverse_test.go b/analysis/token/reverse/reverse_test.go new file mode 100644 index 0000000..4bb7626 --- /dev/null +++ b/analysis/token/reverse/reverse_test.go @@ -0,0 +1,184 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package reverse + +import ( + "bytes" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestReverseFilter(t *testing.T) { + inputTokenStream := analysis.TokenStream{ + &analysis.Token{}, + &analysis.Token{ + Term: []byte("one"), + }, + &analysis.Token{ + Term: []byte("TWo"), + }, + &analysis.Token{ + Term: []byte("thRee"), + }, + &analysis.Token{ + Term: []byte("four's"), + }, + &analysis.Token{ + Term: []byte("what's this in reverse"), + }, + &analysis.Token{ + Term: []byte("œ∑´®†"), + }, + &analysis.Token{ + Term: []byte("İȺȾCAT÷≥≤µ123"), + }, + &analysis.Token{ + Term: []byte("!@#$%^&*()"), + }, + &analysis.Token{ + Term: []byte("cafés"), + }, + &analysis.Token{ + Term: []byte("¿Dónde estás?"), + }, + &analysis.Token{ + Term: []byte("Me gustaría una cerveza."), + }, + } + + expectedTokenStream := analysis.TokenStream{ + &analysis.Token{}, + &analysis.Token{ + Term: []byte("eno"), + }, + &analysis.Token{ + Term: []byte("oWT"), + }, + &analysis.Token{ + Term: []byte("eeRht"), + }, + &analysis.Token{ + Term: []byte("s'ruof"), + }, + &analysis.Token{ + Term: []byte("esrever ni siht s'tahw"), + }, + &analysis.Token{ + Term: []byte("†®´∑œ"), + }, + &analysis.Token{ + Term: []byte("321µ≤≥÷TACȾȺİ"), + }, + &analysis.Token{ + Term: []byte(")(*&^%$#@!"), + }, + &analysis.Token{ + Term: []byte("séfac"), + }, + &analysis.Token{ + Term: []byte("?sátse ednóD¿"), + }, + &analysis.Token{ + Term: []byte(".azevrec anu aíratsug eM"), + }, + } + + filter := NewReverseFilter() + outputTokenStream := filter.Filter(inputTokenStream) + for i := 0; i < len(expectedTokenStream); i++ { + if !bytes.Equal(outputTokenStream[i].Term, expectedTokenStream[i].Term) { + t.Errorf("[%d] expected %s got %s", + i+1, expectedTokenStream[i].Term, outputTokenStream[i].Term) + } + } +} + +func BenchmarkReverseFilter(b *testing.B) { + input := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("A"), + }, + &analysis.Token{ + Term: []byte("boiling"), + }, + &analysis.Token{ + Term: []byte("liquid"), + }, + &analysis.Token{ + Term: []byte("expanding"), + }, + &analysis.Token{ + Term: []byte("vapor"), + }, + &analysis.Token{ + Term: []byte("explosion"), + }, + &analysis.Token{ + Term: []byte("caused"), + }, + &analysis.Token{ + Term: []byte("by"), + }, + &analysis.Token{ + Term: []byte("the"), + }, + &analysis.Token{ + Term: []byte("rupture"), + }, + &analysis.Token{ + Term: []byte("of"), + }, + &analysis.Token{ + Term: []byte("a"), + }, + &analysis.Token{ + Term: []byte("vessel"), + }, + &analysis.Token{ + Term: []byte("containing"), + }, + &analysis.Token{ + Term: []byte("pressurized"), + }, + &analysis.Token{ + Term: []byte("liquid"), + }, + &analysis.Token{ + Term: []byte("above"), + }, + &analysis.Token{ + Term: []byte("its"), + }, + &analysis.Token{ + Term: []byte("boiling"), + }, + &analysis.Token{ + Term: []byte("point"), + }, + &analysis.Token{ + Term: []byte("İȺȾCAT"), + }, + &analysis.Token{ + Term: []byte("Me gustaría una cerveza."), + }, + } + filter := NewReverseFilter() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + filter.Filter(input) + } +} diff --git a/analysis/token/shingle/shingle.go b/analysis/token/shingle/shingle.go new file mode 100644 index 0000000..29280a7 --- /dev/null +++ b/analysis/token/shingle/shingle.go @@ -0,0 +1,172 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shingle + +import ( + "container/ring" + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "shingle" + +type ShingleFilter struct { + min int + max int + outputOriginal bool + tokenSeparator string + fill string +} + +func NewShingleFilter(min, max int, outputOriginal bool, sep, fill string) *ShingleFilter { + return &ShingleFilter{ + min: min, + max: max, + outputOriginal: outputOriginal, + tokenSeparator: sep, + fill: fill, + } +} + +func (s *ShingleFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + rv := make(analysis.TokenStream, 0, len(input)) + + ring := ring.New(s.max) + itemsInRing := 0 + currentPosition := 0 + for _, token := range input { + if s.outputOriginal { + rv = append(rv, token) + } + + // if there are gaps, insert filler tokens + offset := token.Position - currentPosition + for offset > 1 { + fillerToken := analysis.Token{ + Position: 0, + Start: -1, + End: -1, + Type: analysis.AlphaNumeric, + Term: []byte(s.fill), + } + ring.Value = &fillerToken + if itemsInRing < s.max { + itemsInRing++ + } + rv = append(rv, s.shingleCurrentRingState(ring, itemsInRing)...) + ring = ring.Next() + offset-- + } + currentPosition = token.Position + + ring.Value = token + if itemsInRing < s.max { + itemsInRing++ + } + rv = append(rv, s.shingleCurrentRingState(ring, itemsInRing)...) + ring = ring.Next() + } + + return rv +} + +func (s *ShingleFilter) shingleCurrentRingState(ring *ring.Ring, itemsInRing int) analysis.TokenStream { + rv := make(analysis.TokenStream, 0) + for shingleN := s.min; shingleN <= s.max; shingleN++ { + // if there are enough items in the ring + // to produce a shingle of this size + if itemsInRing >= shingleN { + thisShingleRing := ring.Move(-(shingleN - 1)) + shingledBytes := make([]byte, 0) + pos := 0 + start := -1 + end := 0 + for i := 0; i < shingleN; i++ { + if i != 0 { + shingledBytes = append(shingledBytes, []byte(s.tokenSeparator)...) + } + curr := thisShingleRing.Value.(*analysis.Token) + if pos == 0 && curr.Position != 0 { + pos = curr.Position + } + if start == -1 && curr.Start != -1 { + start = curr.Start + } + if curr.End != -1 { + end = curr.End + } + shingledBytes = append(shingledBytes, curr.Term...) + thisShingleRing = thisShingleRing.Next() + } + token := analysis.Token{ + Type: analysis.Shingle, + Term: shingledBytes, + } + if pos != 0 { + token.Position = pos + } + if start != -1 { + token.Start = start + } + if end != -1 { + token.End = end + } + rv = append(rv, &token) + } + } + return rv +} + +func ShingleFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + minVal, ok := config["min"].(float64) + if !ok { + return nil, fmt.Errorf("must specify min") + } + min := int(minVal) + maxVal, ok := config["max"].(float64) + if !ok { + return nil, fmt.Errorf("must specify max") + } + max := int(maxVal) + + outputOriginal := false + outVal, ok := config["output_original"].(bool) + if ok { + outputOriginal = outVal + } + + sep := " " + sepVal, ok := config["separator"].(string) + if ok { + sep = sepVal + } + + fill := "_" + fillVal, ok := config["filler"].(string) + if ok { + fill = fillVal + } + + return NewShingleFilter(min, max, outputOriginal, sep, fill), nil +} + +func init() { + err := registry.RegisterTokenFilter(Name, ShingleFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/token/shingle/shingle_test.go b/analysis/token/shingle/shingle_test.go new file mode 100644 index 0000000..b545056 --- /dev/null +++ b/analysis/token/shingle/shingle_test.go @@ -0,0 +1,416 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shingle + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestShingleFilter(t *testing.T) { + + tests := []struct { + min int + max int + outputOriginal bool + separator string + filler string + input analysis.TokenStream + output analysis.TokenStream + }{ + { + min: 2, + max: 2, + outputOriginal: false, + separator: " ", + filler: "_", + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("the"), + }, + &analysis.Token{ + Term: []byte("quick"), + }, + &analysis.Token{ + Term: []byte("brown"), + }, + &analysis.Token{ + Term: []byte("fox"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("the quick"), + Type: analysis.Shingle, + }, + &analysis.Token{ + Term: []byte("quick brown"), + Type: analysis.Shingle, + }, + &analysis.Token{ + Term: []byte("brown fox"), + Type: analysis.Shingle, + }, + }, + }, + { + min: 3, + max: 3, + outputOriginal: false, + separator: " ", + filler: "_", + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("the"), + }, + &analysis.Token{ + Term: []byte("quick"), + }, + &analysis.Token{ + Term: []byte("brown"), + }, + &analysis.Token{ + Term: []byte("fox"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("the quick brown"), + Type: analysis.Shingle, + }, + &analysis.Token{ + Term: []byte("quick brown fox"), + Type: analysis.Shingle, + }, + }, + }, + { + min: 2, + max: 3, + outputOriginal: false, + separator: " ", + filler: "_", + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("the"), + }, + &analysis.Token{ + Term: []byte("quick"), + }, + &analysis.Token{ + Term: []byte("brown"), + }, + &analysis.Token{ + Term: []byte("fox"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("the quick"), + Type: analysis.Shingle, + }, + &analysis.Token{ + Term: []byte("quick brown"), + Type: analysis.Shingle, + }, + &analysis.Token{ + Term: []byte("the quick brown"), + Type: analysis.Shingle, + }, + &analysis.Token{ + Term: []byte("brown fox"), + Type: analysis.Shingle, + }, + &analysis.Token{ + Term: []byte("quick brown fox"), + Type: analysis.Shingle, + }, + }, + }, + { + min: 3, + max: 3, + outputOriginal: false, + separator: " ", + filler: "_", + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ugly"), + Position: 1, + }, + &analysis.Token{ + Term: []byte("quick"), + Position: 3, + }, + &analysis.Token{ + Term: []byte("brown"), + Position: 4, + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ugly _ quick"), + Type: analysis.Shingle, + Position: 1, + }, + &analysis.Token{ + Term: []byte("_ quick brown"), + Type: analysis.Shingle, + Position: 3, + }, + }, + }, + { + min: 1, + max: 5, + outputOriginal: false, + separator: " ", + filler: "_", + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("test"), + Position: 1, + }, + &analysis.Token{ + Term: []byte("text"), + Position: 2, + }, + // token 3 removed by stop filter + &analysis.Token{ + Term: []byte("see"), + Position: 4, + }, + &analysis.Token{ + Term: []byte("shingles"), + Position: 5, + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("test"), + Type: analysis.Shingle, + Position: 1, + }, + &analysis.Token{ + Term: []byte("text"), + Type: analysis.Shingle, + Position: 2, + }, + &analysis.Token{ + Term: []byte("test text"), + Type: analysis.Shingle, + Position: 1, + }, + &analysis.Token{ + Term: []byte("_"), + Type: analysis.Shingle, + }, + &analysis.Token{ + Term: []byte("text _"), + Type: analysis.Shingle, + Position: 2, + }, + &analysis.Token{ + Term: []byte("test text _"), + Type: analysis.Shingle, + Position: 1, + }, + &analysis.Token{ + Term: []byte("see"), + Type: analysis.Shingle, + Position: 4, + }, + &analysis.Token{ + Term: []byte("_ see"), + Type: analysis.Shingle, + Position: 4, + }, + &analysis.Token{ + Term: []byte("text _ see"), + Type: analysis.Shingle, + Position: 2, + }, + &analysis.Token{ + Term: []byte("test text _ see"), + Type: analysis.Shingle, + Position: 1, + }, + &analysis.Token{ + Term: []byte("shingles"), + Type: analysis.Shingle, + Position: 5, + }, + &analysis.Token{ + Term: []byte("see shingles"), + Type: analysis.Shingle, + Position: 4, + }, + &analysis.Token{ + Term: []byte("_ see shingles"), + Type: analysis.Shingle, + Position: 4, + }, + &analysis.Token{ + Term: []byte("text _ see shingles"), + Type: analysis.Shingle, + Position: 2, + }, + &analysis.Token{ + Term: []byte("test text _ see shingles"), + Type: analysis.Shingle, + Position: 1, + }, + }, + }, + { + min: 2, + max: 2, + outputOriginal: true, + separator: " ", + filler: "_", + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("the"), + }, + &analysis.Token{ + Term: []byte("quick"), + }, + &analysis.Token{ + Term: []byte("brown"), + }, + &analysis.Token{ + Term: []byte("fox"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("the"), + }, + &analysis.Token{ + Term: []byte("quick"), + }, + &analysis.Token{ + Term: []byte("the quick"), + Type: analysis.Shingle, + }, + &analysis.Token{ + Term: []byte("brown"), + }, + &analysis.Token{ + Term: []byte("quick brown"), + Type: analysis.Shingle, + }, + &analysis.Token{ + Term: []byte("fox"), + }, + &analysis.Token{ + Term: []byte("brown fox"), + Type: analysis.Shingle, + }, + }, + }, + } + + for _, test := range tests { + shingleFilter := NewShingleFilter(test.min, test.max, test.outputOriginal, test.separator, test.filler) + actual := shingleFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output, actual) + } + } +} + +// TestShingleFilterBug431 tests that the shingle filter is in fact stateless +// by making using the same filter instance twice and ensuring we do not get +// contaminated output +func TestShingleFilterBug431(t *testing.T) { + + tests := []struct { + input analysis.TokenStream + output analysis.TokenStream + }{ + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("the"), + }, + &analysis.Token{ + Term: []byte("quick"), + }, + &analysis.Token{ + Term: []byte("brown"), + }, + &analysis.Token{ + Term: []byte("fox"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("the quick"), + Type: analysis.Shingle, + }, + &analysis.Token{ + Term: []byte("quick brown"), + Type: analysis.Shingle, + }, + &analysis.Token{ + Term: []byte("brown fox"), + Type: analysis.Shingle, + }, + }, + }, + { + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a"), + }, + &analysis.Token{ + Term: []byte("sad"), + }, + &analysis.Token{ + Term: []byte("dirty"), + }, + &analysis.Token{ + Term: []byte("sock"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a sad"), + Type: analysis.Shingle, + }, + &analysis.Token{ + Term: []byte("sad dirty"), + Type: analysis.Shingle, + }, + &analysis.Token{ + Term: []byte("dirty sock"), + Type: analysis.Shingle, + }, + }, + }, + } + + shingleFilter := NewShingleFilter(2, 2, false, " ", "_") + for _, test := range tests { + actual := shingleFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output, actual) + } + } + +} diff --git a/analysis/token/snowball/snowball.go b/analysis/token/snowball/snowball.go new file mode 100644 index 0000000..02c1a6c --- /dev/null +++ b/analysis/token/snowball/snowball.go @@ -0,0 +1,62 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package snowball + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + + "github.com/blevesearch/snowball" +) + +const Name = "stemmer_snowball" + +type SnowballStemmer struct { + language string +} + +func NewSnowballStemmer(language string) *SnowballStemmer { + return &SnowballStemmer{ + language: language, + } +} + +func (s *SnowballStemmer) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + // if it is not a protected keyword, stem it + if !token.KeyWord { + stemmed, _ := snowball.Stem(string(token.Term), s.language, true) + token.Term = []byte(stemmed) + } + } + return input +} + +func SnowballStemmerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + language, ok := config["language"].(string) + if !ok { + return nil, fmt.Errorf("must specify language") + } + return NewSnowballStemmer(language), nil +} + +func init() { + err := registry.RegisterTokenFilter(Name, SnowballStemmerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/token/snowball/snowball_test.go b/analysis/token/snowball/snowball_test.go new file mode 100644 index 0000000..a805347 --- /dev/null +++ b/analysis/token/snowball/snowball_test.go @@ -0,0 +1,115 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package snowball + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestSnowballStemmer(t *testing.T) { + + inputTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("walking"), + }, + &analysis.Token{ + Term: []byte("talked"), + }, + &analysis.Token{ + Term: []byte("business"), + }, + &analysis.Token{ + Term: []byte("protected"), + KeyWord: true, + }, + &analysis.Token{ + Term: []byte("cat"), + }, + &analysis.Token{ + Term: []byte("done"), + }, + // a term which does stem, but does not change length + &analysis.Token{ + Term: []byte("marty"), + }, + } + + expectedTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("walk"), + }, + &analysis.Token{ + Term: []byte("talk"), + }, + &analysis.Token{ + Term: []byte("busi"), + }, + &analysis.Token{ + Term: []byte("protected"), + KeyWord: true, + }, + &analysis.Token{ + Term: []byte("cat"), + }, + &analysis.Token{ + Term: []byte("done"), + }, + &analysis.Token{ + Term: []byte("marti"), + }, + } + + filter := NewSnowballStemmer("english") + ouputTokenStream := filter.Filter(inputTokenStream) + if !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) { + t.Errorf("expected %#v got %#v", expectedTokenStream[3], ouputTokenStream[3]) + } +} + +func BenchmarkSnowballStemmer(b *testing.B) { + + inputTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("walking"), + }, + &analysis.Token{ + Term: []byte("talked"), + }, + &analysis.Token{ + Term: []byte("business"), + }, + &analysis.Token{ + Term: []byte("protected"), + KeyWord: true, + }, + &analysis.Token{ + Term: []byte("cat"), + }, + &analysis.Token{ + Term: []byte("done"), + }, + } + + filter := NewSnowballStemmer("english") + b.ResetTimer() + + for i := 0; i < b.N; i++ { + filter.Filter(inputTokenStream) + } + +} diff --git a/analysis/token/stop/stop.go b/analysis/token/stop/stop.go new file mode 100644 index 0000000..09f2d1c --- /dev/null +++ b/analysis/token/stop/stop.go @@ -0,0 +1,73 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package stop implements a TokenFilter removing tokens found in +// a TokenMap. +// +// It constructor takes the following arguments: +// +// "stop_token_map" (string): the name of the token map identifying tokens to +// remove. +package stop + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "stop_tokens" + +type StopTokensFilter struct { + stopTokens analysis.TokenMap +} + +func NewStopTokensFilter(stopTokens analysis.TokenMap) *StopTokensFilter { + return &StopTokensFilter{ + stopTokens: stopTokens, + } +} + +func (f *StopTokensFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + j := 0 + for _, token := range input { + _, isStopToken := f.stopTokens[string(token.Term)] + if !isStopToken { + input[j] = token + j++ + } + } + + return input[:j] +} + +func StopTokensFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + stopTokenMapName, ok := config["stop_token_map"].(string) + if !ok { + return nil, fmt.Errorf("must specify stop_token_map") + } + stopTokenMap, err := cache.TokenMapNamed(stopTokenMapName) + if err != nil { + return nil, fmt.Errorf("error building stop words filter: %v", err) + } + return NewStopTokensFilter(stopTokenMap), nil +} + +func init() { + err := registry.RegisterTokenFilter(Name, StopTokensFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/token/stop/stop_test.go b/analysis/token/stop/stop_test.go new file mode 100644 index 0000000..f63c562 --- /dev/null +++ b/analysis/token/stop/stop_test.go @@ -0,0 +1,124 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stop + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/tokenmap" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestStopWordsFilter(t *testing.T) { + + inputTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a"), + }, + &analysis.Token{ + Term: []byte("walk"), + }, + &analysis.Token{ + Term: []byte("in"), + }, + &analysis.Token{ + Term: []byte("the"), + }, + &analysis.Token{ + Term: []byte("park"), + }, + } + + expectedTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("walk"), + }, + &analysis.Token{ + Term: []byte("park"), + }, + } + + cache := registry.NewCache() + stopListConfig := map[string]interface{}{ + "type": tokenmap.Name, + "tokens": []interface{}{"a", "in", "the"}, + } + _, err := cache.DefineTokenMap("stop_test", stopListConfig) + if err != nil { + t.Fatal(err) + } + + stopConfig := map[string]interface{}{ + "type": "stop_tokens", + "stop_token_map": "stop_test", + } + stopFilter, err := cache.DefineTokenFilter("stop_test", stopConfig) + if err != nil { + t.Fatal(err) + } + + ouputTokenStream := stopFilter.Filter(inputTokenStream) + if !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) { + t.Errorf("expected %#v got %#v", expectedTokenStream, ouputTokenStream) + } +} + +func BenchmarkStopWordsFilter(b *testing.B) { + + inputTokenStream := analysis.TokenStream{ + &analysis.Token{ + Term: []byte("a"), + }, + &analysis.Token{ + Term: []byte("walk"), + }, + &analysis.Token{ + Term: []byte("in"), + }, + &analysis.Token{ + Term: []byte("the"), + }, + &analysis.Token{ + Term: []byte("park"), + }, + } + + cache := registry.NewCache() + stopListConfig := map[string]interface{}{ + "type": tokenmap.Name, + "tokens": []interface{}{"a", "in", "the"}, + } + _, err := cache.DefineTokenMap("stop_test", stopListConfig) + if err != nil { + b.Fatal(err) + } + + stopConfig := map[string]interface{}{ + "type": "stop_tokens", + "stop_token_map": "stop_test", + } + stopFilter, err := cache.DefineTokenFilter("stop_test", stopConfig) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + + for i := 0; i < b.N; i++ { + stopFilter.Filter(inputTokenStream) + } + +} diff --git a/analysis/token/truncate/truncate.go b/analysis/token/truncate/truncate.go new file mode 100644 index 0000000..2019358 --- /dev/null +++ b/analysis/token/truncate/truncate.go @@ -0,0 +1,62 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package truncate + +import ( + "fmt" + "unicode/utf8" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "truncate_token" + +type TruncateTokenFilter struct { + length int +} + +func NewTruncateTokenFilter(length int) *TruncateTokenFilter { + return &TruncateTokenFilter{ + length: length, + } +} + +func (s *TruncateTokenFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + wordLen := utf8.RuneCount(token.Term) + if wordLen > s.length { + token.Term = analysis.TruncateRunes(token.Term, wordLen-s.length) + } + } + return input +} + +func TruncateTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + lenVal, ok := config["length"].(float64) + if !ok { + return nil, fmt.Errorf("must specify length") + } + length := int(lenVal) + + return NewTruncateTokenFilter(length), nil +} + +func init() { + err := registry.RegisterTokenFilter(Name, TruncateTokenFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/token/truncate/truncate_test.go b/analysis/token/truncate/truncate_test.go new file mode 100644 index 0000000..083e277 --- /dev/null +++ b/analysis/token/truncate/truncate_test.go @@ -0,0 +1,79 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package truncate + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestTruncateTokenFilter(t *testing.T) { + + tests := []struct { + length int + input analysis.TokenStream + output analysis.TokenStream + }{ + { + length: 5, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abcdefgh"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("abcde"), + }, + }, + }, + { + length: 3, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("こんにちは世界"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("こんに"), + }, + }, + }, + { + length: 10, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("แยกคำภาษาไทยก็ทำได้นะจ้ะ"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("แยกคำภาษาไ"), + }, + }, + }, + } + + for _, test := range tests { + truncateTokenFilter := NewTruncateTokenFilter(test.length) + actual := truncateTokenFilter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/token/unicodenorm/unicodenorm.go b/analysis/token/unicodenorm/unicodenorm.go new file mode 100644 index 0000000..ba0243a --- /dev/null +++ b/analysis/token/unicodenorm/unicodenorm.go @@ -0,0 +1,82 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unicodenorm + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + "golang.org/x/text/unicode/norm" +) + +const Name = "normalize_unicode" + +const NFC = "nfc" +const NFD = "nfd" +const NFKC = "nfkc" +const NFKD = "nfkd" + +var forms = map[string]norm.Form{ + NFC: norm.NFC, + NFD: norm.NFD, + NFKC: norm.NFKC, + NFKD: norm.NFKD, +} + +type UnicodeNormalizeFilter struct { + form norm.Form +} + +func NewUnicodeNormalizeFilter(formName string) (*UnicodeNormalizeFilter, error) { + form, ok := forms[formName] + if !ok { + return nil, fmt.Errorf("no form named %s", formName) + } + return &UnicodeNormalizeFilter{ + form: form, + }, nil +} + +func MustNewUnicodeNormalizeFilter(formName string) *UnicodeNormalizeFilter { + filter, err := NewUnicodeNormalizeFilter(formName) + if err != nil { + panic(err) + } + return filter +} + +func (s *UnicodeNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + token.Term = s.form.Bytes(token.Term) + } + return input +} + +func UnicodeNormalizeFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + formVal, ok := config["form"].(string) + if !ok { + return nil, fmt.Errorf("must specify form") + } + form := formVal + return NewUnicodeNormalizeFilter(form) +} + +func init() { + err := registry.RegisterTokenFilter(Name, UnicodeNormalizeFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/token/unicodenorm/unicodenorm_test.go b/analysis/token/unicodenorm/unicodenorm_test.go new file mode 100644 index 0000000..0ff2bd8 --- /dev/null +++ b/analysis/token/unicodenorm/unicodenorm_test.go @@ -0,0 +1,162 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unicodenorm + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +// the following tests come from the lucene +// test cases for CJK width filter +// which is our basis for using this +// as a substitute for that +func TestUnicodeNormalization(t *testing.T) { + + tests := []struct { + formName string + input analysis.TokenStream + output analysis.TokenStream + }{ + { + formName: NFKD, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Ｔｅｓｔ"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("Test"), + }, + }, + }, + { + formName: NFKD, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("１２３４"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("1234"), + }, + }, + }, + { + formName: NFKD, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ｶﾀｶﾅ"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("カタカナ"), + }, + }, + }, + { + formName: NFKC, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ｳﾞｨｯﾂ"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ヴィッツ"), + }, + }, + }, + { + formName: NFKC, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("ﾊﾟﾅｿﾆｯｸ"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("パナソニック"), + }, + }, + }, + { + formName: NFD, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u212B"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u0041\u030A"), + }, + }, + }, + { + formName: NFC, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u212B"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u00C5"), + }, + }, + }, + { + formName: NFKD, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\uFB01"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u0066\u0069"), + }, + }, + }, + { + formName: NFKC, + input: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\uFB01"), + }, + }, + output: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("\u0066\u0069"), + }, + }, + }, + } + + for _, test := range tests { + filter := MustNewUnicodeNormalizeFilter(test.formName) + actual := filter.Filter(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) + t.Errorf("expected %#v, got %#v", test.output[0].Term, actual[0].Term) + } + } +} diff --git a/analysis/token/unique/unique.go b/analysis/token/unique/unique.go new file mode 100644 index 0000000..e48c4f9 --- /dev/null +++ b/analysis/token/unique/unique.go @@ -0,0 +1,56 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unique + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "unique" + +// UniqueTermFilter retains only the tokens which mark the first occurrence of +// a term. Tokens whose term appears in a preceding token are dropped. +type UniqueTermFilter struct{} + +func NewUniqueTermFilter() *UniqueTermFilter { + return &UniqueTermFilter{} +} + +func (f *UniqueTermFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + encounteredTerms := make(map[string]struct{}, len(input)/4) + j := 0 + for _, token := range input { + term := string(token.Term) + if _, ok := encounteredTerms[term]; ok { + continue + } + encounteredTerms[term] = struct{}{} + input[j] = token + j++ + } + return input[:j] +} + +func UniqueTermFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewUniqueTermFilter(), nil +} + +func init() { + err := registry.RegisterTokenFilter(Name, UniqueTermFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/token/unique/unique_test.go b/analysis/token/unique/unique_test.go new file mode 100644 index 0000000..bc17130 --- /dev/null +++ b/analysis/token/unique/unique_test.go @@ -0,0 +1,84 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unique + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestUniqueTermFilter(t *testing.T) { + var tests = []struct { + input analysis.TokenStream + // expected indices of input which should be included in the output. We + // use indices instead of another TokenStream, since position/start/end + // should be preserved. + expectedIndices []int + }{ + { + input: tokenStream(), + expectedIndices: []int{}, + }, + { + input: tokenStream("a"), + expectedIndices: []int{0}, + }, + { + input: tokenStream("each", "term", "in", "this", "sentence", "is", "unique"), + expectedIndices: []int{0, 1, 2, 3, 4, 5, 6}, + }, + { + input: tokenStream("Lui", "è", "alto", "e", "lei", "è", "bassa"), + expectedIndices: []int{0, 1, 2, 3, 4, 6}, + }, + { + input: tokenStream("a", "a", "A", "a", "a", "A"), + expectedIndices: []int{0, 2}, + }, + } + uniqueTermFilter := NewUniqueTermFilter() + for _, test := range tests { + expected := subStream(test.input, test.expectedIndices) + actual := uniqueTermFilter.Filter(test.input) + if !reflect.DeepEqual(actual, expected) { + t.Errorf("expected %s \n\n got %s", expected, actual) + } + } +} + +func tokenStream(termStrs ...string) analysis.TokenStream { + tokenStream := make([]*analysis.Token, len(termStrs)) + index := 0 + for i, termStr := range termStrs { + tokenStream[i] = &analysis.Token{ + Term: []byte(termStr), + Position: i + 1, + Start: index, + End: index + len(termStr), + } + index += len(termStr) + } + return analysis.TokenStream(tokenStream) +} + +func subStream(stream analysis.TokenStream, indices []int) analysis.TokenStream { + result := make(analysis.TokenStream, len(indices)) + for i, index := range indices { + result[i] = stream[index] + } + return result +} diff --git a/analysis/tokenizer/character/character.go b/analysis/tokenizer/character/character.go new file mode 100644 index 0000000..9c4f7e4 --- /dev/null +++ b/analysis/tokenizer/character/character.go @@ -0,0 +1,76 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package character + +import ( + "unicode/utf8" + + "github.com/blevesearch/bleve/v2/analysis" +) + +type IsTokenRune func(r rune) bool + +type CharacterTokenizer struct { + isTokenRun IsTokenRune +} + +func NewCharacterTokenizer(f IsTokenRune) *CharacterTokenizer { + return &CharacterTokenizer{ + isTokenRun: f, + } +} + +func (c *CharacterTokenizer) Tokenize(input []byte) analysis.TokenStream { + + rv := make(analysis.TokenStream, 0, 1024) + + offset := 0 + start := 0 + end := 0 + count := 0 + for currRune, size := utf8.DecodeRune(input[offset:]); currRune != utf8.RuneError; currRune, size = utf8.DecodeRune(input[offset:]) { + isToken := c.isTokenRun(currRune) + if isToken { + end = offset + size + } else { + if end-start > 0 { + // build token + rv = append(rv, &analysis.Token{ + Term: input[start:end], + Start: start, + End: end, + Position: count + 1, + Type: analysis.AlphaNumeric, + }) + count++ + } + start = offset + size + end = start + } + offset += size + } + // if we ended in the middle of a token, finish it + if end-start > 0 { + // build token + rv = append(rv, &analysis.Token{ + Term: input[start:end], + Start: start, + End: end, + Position: count + 1, + Type: analysis.AlphaNumeric, + }) + } + return rv +} diff --git a/analysis/tokenizer/character/character_test.go b/analysis/tokenizer/character/character_test.go new file mode 100644 index 0000000..30d00e8 --- /dev/null +++ b/analysis/tokenizer/character/character_test.go @@ -0,0 +1,84 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package character + +import ( + "reflect" + "testing" + "unicode" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestCharacterTokenizer(t *testing.T) { + tests := []struct { + input []byte + output analysis.TokenStream + }{ + { + []byte("Hello World."), + analysis.TokenStream{ + { + Start: 0, + End: 5, + Term: []byte("Hello"), + Position: 1, + Type: analysis.AlphaNumeric, + }, + { + Start: 6, + End: 11, + Term: []byte("World"), + Position: 2, + Type: analysis.AlphaNumeric, + }, + }, + }, + { + []byte("dominique@mcdiabetes.com"), + analysis.TokenStream{ + { + Start: 0, + End: 9, + Term: []byte("dominique"), + Position: 1, + Type: analysis.AlphaNumeric, + }, + { + Start: 10, + End: 20, + Term: []byte("mcdiabetes"), + Position: 2, + Type: analysis.AlphaNumeric, + }, + { + Start: 21, + End: 24, + Term: []byte("com"), + Position: 3, + Type: analysis.AlphaNumeric, + }, + }, + }, + } + + tokenizer := NewCharacterTokenizer(unicode.IsLetter) + for _, test := range tests { + actual := tokenizer.Tokenize(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("Expected %v, got %v for %s", test.output, actual, string(test.input)) + } + } +} diff --git a/analysis/tokenizer/exception/exception.go b/analysis/tokenizer/exception/exception.go new file mode 100644 index 0000000..30c8eff --- /dev/null +++ b/analysis/tokenizer/exception/exception.go @@ -0,0 +1,144 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package exception implements a Tokenizer which extracts pieces matched by a +// regular expression from the input data, delegates the rest to another +// tokenizer, then insert back extracted parts in the token stream. Use it to +// preserve sequences which a regular tokenizer would alter or remove. +// +// Its constructor takes the following arguments: +// +// "exceptions" ([]string): one or more Go regular expressions matching the +// sequence to preserve. Multiple expressions are combined with "|". +// +// "tokenizer" (string): the name of the tokenizer processing the data not +// matched by "exceptions". +package exception + +import ( + "fmt" + "regexp" + "strings" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "exception" + +type ExceptionsTokenizer struct { + exception *regexp.Regexp + remaining analysis.Tokenizer +} + +func NewExceptionsTokenizer(exception *regexp.Regexp, remaining analysis.Tokenizer) *ExceptionsTokenizer { + return &ExceptionsTokenizer{ + exception: exception, + remaining: remaining, + } +} + +func (t *ExceptionsTokenizer) Tokenize(input []byte) analysis.TokenStream { + rv := make(analysis.TokenStream, 0) + matches := t.exception.FindAllIndex(input, -1) + currInput := 0 + lastPos := 0 + for _, match := range matches { + start := match[0] + end := match[1] + if start > currInput { + // need to defer to remaining for unprocessed section + intermediate := t.remaining.Tokenize(input[currInput:start]) + // add intermediate tokens to our result stream + for _, token := range intermediate { + // adjust token offsets + token.Position += lastPos + token.Start += currInput + token.End += currInput + rv = append(rv, token) + } + lastPos += len(intermediate) + currInput = start + } + + // create single token with this regexp match + token := &analysis.Token{ + Term: input[start:end], + Start: start, + End: end, + Position: lastPos + 1, + } + rv = append(rv, token) + lastPos++ + currInput = end + + } + + if currInput < len(input) { + // need to defer to remaining for unprocessed section + intermediate := t.remaining.Tokenize(input[currInput:]) + // add intermediate tokens to our result stream + for _, token := range intermediate { + // adjust token offsets + token.Position += lastPos + token.Start += currInput + token.End += currInput + rv = append(rv, token) + } + } + + return rv +} + +func ExceptionsTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { + exceptions := []string{} + iexceptions, ok := config["exceptions"].([]interface{}) + if ok { + for _, exception := range iexceptions { + exception, ok := exception.(string) + if ok { + exceptions = append(exceptions, exception) + } + } + } + aexceptions, ok := config["exceptions"].([]string) + if ok { + exceptions = append(exceptions, aexceptions...) + } + if len(exceptions) == 0 { + return nil, fmt.Errorf("no pattern found in 'exception' property") + } + exceptionPattern := strings.Join(exceptions, "|") + r, err := regexp.Compile(exceptionPattern) + if err != nil { + return nil, fmt.Errorf("unable to build regexp tokenizer: %v", err) + } + + remainingName, ok := config["tokenizer"].(string) + if !ok { + return nil, fmt.Errorf("must specify tokenizer for remaining input") + } + remaining, err := cache.TokenizerNamed(remainingName) + if err != nil { + return nil, err + } + return NewExceptionsTokenizer(r, remaining), nil +} + +func init() { + err := registry.RegisterTokenizer(Name, ExceptionsTokenizerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/tokenizer/exception/exception_test.go b/analysis/tokenizer/exception/exception_test.go new file mode 100644 index 0000000..1c4acb0 --- /dev/null +++ b/analysis/tokenizer/exception/exception_test.go @@ -0,0 +1,171 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package exception + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + _ "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestExceptionsTokenizer(t *testing.T) { + tests := []struct { + config map[string]interface{} + input []byte + patterns []string + result analysis.TokenStream + }{ + { + input: []byte("test http://blevesearch.com/ words"), + config: map[string]interface{}{ + "type": "exception", + "tokenizer": "unicode", + "exceptions": []interface{}{ + `[hH][tT][tT][pP][sS]?://(\S)*`, + `[fF][iI][lL][eE]://(\S)*`, + `[fF][tT][pP]://(\S)*`, + }, + }, + result: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("test"), + Position: 1, + Start: 0, + End: 4, + }, + &analysis.Token{ + Term: []byte("http://blevesearch.com/"), + Position: 2, + Start: 5, + End: 28, + }, + &analysis.Token{ + Term: []byte("words"), + Position: 3, + Start: 29, + End: 34, + }, + }, + }, + { + input: []byte("what ftp://blevesearch.com/ songs"), + config: map[string]interface{}{ + "type": "exception", + "tokenizer": "unicode", + "exceptions": []interface{}{ + `[hH][tT][tT][pP][sS]?://(\S)*`, + `[fF][iI][lL][eE]://(\S)*`, + `[fF][tT][pP]://(\S)*`, + }, + }, + result: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("what"), + Position: 1, + Start: 0, + End: 4, + }, + &analysis.Token{ + Term: []byte("ftp://blevesearch.com/"), + Position: 2, + Start: 5, + End: 27, + }, + &analysis.Token{ + Term: []byte("songs"), + Position: 3, + Start: 28, + End: 33, + }, + }, + }, + { + input: []byte("please email marty@couchbase.com the URL https://blevesearch.com/"), + config: map[string]interface{}{ + "type": "exception", + "tokenizer": "unicode", + "exceptions": []interface{}{ + `[hH][tT][tT][pP][sS]?://(\S)*`, + `[fF][iI][lL][eE]://(\S)*`, + `[fF][tT][pP]://(\S)*`, + `\S+@\S+`, + }, + }, + result: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("please"), + Position: 1, + Start: 0, + End: 6, + }, + &analysis.Token{ + Term: []byte("email"), + Position: 2, + Start: 7, + End: 12, + }, + &analysis.Token{ + Term: []byte("marty@couchbase.com"), + Position: 3, + Start: 13, + End: 32, + }, + &analysis.Token{ + Term: []byte("the"), + Position: 4, + Start: 33, + End: 36, + }, + &analysis.Token{ + Term: []byte("URL"), + Position: 5, + Start: 37, + End: 40, + }, + &analysis.Token{ + Term: []byte("https://blevesearch.com/"), + Position: 6, + Start: 41, + End: 65, + }, + }, + }, + } + + // remaining := unicode.NewUnicodeTokenizer() + for _, test := range tests { + + // build the requested exception tokenizer + cache := registry.NewCache() + tokenizer, err := cache.DefineTokenizer("custom", test.config) + if err != nil { + t.Fatal(err) + } + + // pattern := strings.Join(test.patterns, "|") + // r, err := regexp.Compile(pattern) + // if err != nil { + // t.Fatal(err) + // } + // tokenizer := NewExceptionsTokenizer(r, remaining) + actual := tokenizer.Tokenize(test.input) + if !reflect.DeepEqual(actual, test.result) { + t.Errorf("expected %v, got %v", test.result, actual) + } + } +} diff --git a/analysis/tokenizer/letter/letter.go b/analysis/tokenizer/letter/letter.go new file mode 100644 index 0000000..ed4dba1 --- /dev/null +++ b/analysis/tokenizer/letter/letter.go @@ -0,0 +1,36 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package letter + +import ( + "unicode" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/character" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "letter" + +func TokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { + return character.NewCharacterTokenizer(unicode.IsLetter), nil +} + +func init() { + err := registry.RegisterTokenizer(Name, TokenizerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/tokenizer/regexp/regexp.go b/analysis/tokenizer/regexp/regexp.go new file mode 100644 index 0000000..94c1879 --- /dev/null +++ b/analysis/tokenizer/regexp/regexp.go @@ -0,0 +1,87 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package regexp + +import ( + "fmt" + "regexp" + "strconv" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "regexp" + +var IdeographRegexp = regexp.MustCompile(`\p{Han}|\p{Hangul}|\p{Hiragana}|\p{Katakana}`) + +type RegexpTokenizer struct { + r *regexp.Regexp +} + +func NewRegexpTokenizer(r *regexp.Regexp) *RegexpTokenizer { + return &RegexpTokenizer{ + r: r, + } +} + +func (rt *RegexpTokenizer) Tokenize(input []byte) analysis.TokenStream { + matches := rt.r.FindAllIndex(input, -1) + rv := make(analysis.TokenStream, 0, len(matches)) + for i, match := range matches { + matchBytes := input[match[0]:match[1]] + if match[1]-match[0] > 0 { + token := analysis.Token{ + Term: matchBytes, + Start: match[0], + End: match[1], + Position: i + 1, + Type: detectTokenType(matchBytes), + } + rv = append(rv, &token) + } + } + return rv +} + +func RegexpTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { + rval, ok := config["regexp"].(string) + if !ok { + return nil, fmt.Errorf("must specify regexp") + } + r, err := regexp.Compile(rval) + if err != nil { + return nil, fmt.Errorf("unable to build regexp tokenizer: %v", err) + } + return NewRegexpTokenizer(r), nil +} + +func init() { + err := registry.RegisterTokenizer(Name, RegexpTokenizerConstructor) + if err != nil { + panic(err) + } +} + +func detectTokenType(termBytes []byte) analysis.TokenType { + if IdeographRegexp.Match(termBytes) { + return analysis.Ideographic + } + _, err := strconv.ParseFloat(string(termBytes), 64) + if err == nil { + return analysis.Numeric + } + return analysis.AlphaNumeric +} diff --git a/analysis/tokenizer/regexp/regexp_test.go b/analysis/tokenizer/regexp/regexp_test.go new file mode 100644 index 0000000..cfad915 --- /dev/null +++ b/analysis/tokenizer/regexp/regexp_test.go @@ -0,0 +1,166 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package regexp + +import ( + "reflect" + "regexp" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestBoundary(t *testing.T) { + + wordRegex := regexp.MustCompile(`\p{Han}|\p{Hangul}|\p{Hiragana}|\p{Katakana}|\w+`) + + tests := []struct { + input []byte + output analysis.TokenStream + }{ + { + []byte("Hello World."), + analysis.TokenStream{ + { + Start: 0, + End: 5, + Term: []byte("Hello"), + Position: 1, + Type: analysis.AlphaNumeric, + }, + { + Start: 6, + End: 11, + Term: []byte("World"), + Position: 2, + Type: analysis.AlphaNumeric, + }, + }, + }, + { + []byte("こんにちは世界"), + analysis.TokenStream{ + { + Start: 0, + End: 3, + Term: []byte("こ"), + Position: 1, + Type: analysis.Ideographic, + }, + { + Start: 3, + End: 6, + Term: []byte("ん"), + Position: 2, + Type: analysis.Ideographic, + }, + { + Start: 6, + End: 9, + Term: []byte("に"), + Position: 3, + Type: analysis.Ideographic, + }, + { + Start: 9, + End: 12, + Term: []byte("ち"), + Position: 4, + Type: analysis.Ideographic, + }, + { + Start: 12, + End: 15, + Term: []byte("は"), + Position: 5, + Type: analysis.Ideographic, + }, + { + Start: 15, + End: 18, + Term: []byte("世"), + Position: 6, + Type: analysis.Ideographic, + }, + { + Start: 18, + End: 21, + Term: []byte("界"), + Position: 7, + Type: analysis.Ideographic, + }, + }, + }, + { + []byte(""), + analysis.TokenStream{}, + }, + } + + for _, test := range tests { + tokenizer := NewRegexpTokenizer(wordRegex) + actual := tokenizer.Tokenize(test.input) + + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("Expected %v, got %v for %s", test.output, actual, string(test.input)) + } + } +} + +func TestBugProducingEmptyTokens(t *testing.T) { + + wordRegex := regexp.MustCompile(`[0-9a-zA-Z_]*`) + + tests := []struct { + input []byte + output analysis.TokenStream + }{ + { + []byte("Chatha Edwards Sr."), + analysis.TokenStream{ + { + Start: 0, + End: 6, + Term: []byte("Chatha"), + Position: 1, + Type: analysis.AlphaNumeric, + }, + { + Start: 7, + End: 14, + Term: []byte("Edwards"), + Position: 2, + Type: analysis.AlphaNumeric, + }, + { + Start: 15, + End: 17, + Term: []byte("Sr"), + Position: 3, + Type: analysis.AlphaNumeric, + }, + }, + }, + } + + for _, test := range tests { + tokenizer := NewRegexpTokenizer(wordRegex) + actual := tokenizer.Tokenize(test.input) + + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("Expected %v, got %v for %s", test.output, actual, string(test.input)) + } + } +} diff --git a/analysis/tokenizer/single/single.go b/analysis/tokenizer/single/single.go new file mode 100644 index 0000000..7f3abd2 --- /dev/null +++ b/analysis/tokenizer/single/single.go @@ -0,0 +1,52 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package single + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "single" + +type SingleTokenTokenizer struct { +} + +func NewSingleTokenTokenizer() *SingleTokenTokenizer { + return &SingleTokenTokenizer{} +} + +func (t *SingleTokenTokenizer) Tokenize(input []byte) analysis.TokenStream { + return analysis.TokenStream{ + &analysis.Token{ + Term: input, + Position: 1, + Start: 0, + End: len(input), + Type: analysis.AlphaNumeric, + }, + } +} + +func SingleTokenTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { + return NewSingleTokenTokenizer(), nil +} + +func init() { + err := registry.RegisterTokenizer(Name, SingleTokenTokenizerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/tokenizer/single/single_test.go b/analysis/tokenizer/single/single_test.go new file mode 100644 index 0000000..7cc134c --- /dev/null +++ b/analysis/tokenizer/single/single_test.go @@ -0,0 +1,76 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package single + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestSingleTokenTokenizer(t *testing.T) { + + tests := []struct { + input []byte + output analysis.TokenStream + }{ + { + []byte("Hello World"), + analysis.TokenStream{ + { + Start: 0, + End: 11, + Term: []byte("Hello World"), + Position: 1, + Type: analysis.AlphaNumeric, + }, + }, + }, + { + []byte("こんにちは世界"), + analysis.TokenStream{ + { + Start: 0, + End: 21, + Term: []byte("こんにちは世界"), + Position: 1, + Type: analysis.AlphaNumeric, + }, + }, + }, + { + []byte("แยกคำภาษาไทยก็ทำได้นะจ้ะ"), + analysis.TokenStream{ + { + Start: 0, + End: 72, + Term: []byte("แยกคำภาษาไทยก็ทำได้นะจ้ะ"), + Position: 1, + Type: analysis.AlphaNumeric, + }, + }, + }, + } + + for _, test := range tests { + tokenizer := NewSingleTokenTokenizer() + actual := tokenizer.Tokenize(test.input) + + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("Expected %v, got %v for %s", test.output, actual, string(test.input)) + } + } +} diff --git a/analysis/tokenizer/unicode/unicode.go b/analysis/tokenizer/unicode/unicode.go new file mode 100644 index 0000000..b694a3e --- /dev/null +++ b/analysis/tokenizer/unicode/unicode.go @@ -0,0 +1,134 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unicode + +import ( + "github.com/blevesearch/segment" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "unicode" + +type UnicodeTokenizer struct { +} + +func NewUnicodeTokenizer() *UnicodeTokenizer { + return &UnicodeTokenizer{} +} + +func (rt *UnicodeTokenizer) Tokenize(input []byte) analysis.TokenStream { + rvx := make([]analysis.TokenStream, 0, 10) // When rv gets full, append to rvx. + rv := make(analysis.TokenStream, 0, 1) + + ta := []analysis.Token(nil) + taNext := 0 + + segmenter := segment.NewWordSegmenterDirect(input) + start := 0 + pos := 1 + + guessRemaining := func(end int) int { + avgSegmentLen := end / (len(rv) + 1) + if avgSegmentLen < 1 { + avgSegmentLen = 1 + } + + remainingLen := len(input) - end + + return remainingLen / avgSegmentLen + } + + for segmenter.Segment() { + segmentBytes := segmenter.Bytes() + end := start + len(segmentBytes) + if segmenter.Type() != segment.None { + if taNext >= len(ta) { + remainingSegments := guessRemaining(end) + if remainingSegments > 1000 { + remainingSegments = 1000 + } + if remainingSegments < 1 { + remainingSegments = 1 + } + + ta = make([]analysis.Token, remainingSegments) + taNext = 0 + } + + token := &ta[taNext] + taNext++ + + token.Term = segmentBytes + token.Start = start + token.End = end + token.Position = pos + token.Type = convertType(segmenter.Type()) + + if len(rv) >= cap(rv) { // When rv is full, save it into rvx. + rvx = append(rvx, rv) + + rvCap := cap(rv) * 2 + if rvCap > 256 { + rvCap = 256 + } + + rv = make(analysis.TokenStream, 0, rvCap) // Next rv cap is bigger. + } + + rv = append(rv, token) + pos++ + } + start = end + } + + if len(rvx) > 0 { + n := len(rv) + for _, r := range rvx { + n += len(r) + } + rall := make(analysis.TokenStream, 0, n) + for _, r := range rvx { + rall = append(rall, r...) + } + return append(rall, rv...) + } + + return rv +} + +func UnicodeTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { + return NewUnicodeTokenizer(), nil +} + +func init() { + err := registry.RegisterTokenizer(Name, UnicodeTokenizerConstructor) + if err != nil { + panic(err) + } +} + +func convertType(segmentWordType int) analysis.TokenType { + switch segmentWordType { + case segment.Ideo: + return analysis.Ideographic + case segment.Kana: + return analysis.Ideographic + case segment.Number: + return analysis.Numeric + } + return analysis.AlphaNumeric +} diff --git a/analysis/tokenizer/unicode/unicode_test.go b/analysis/tokenizer/unicode/unicode_test.go new file mode 100644 index 0000000..0e0cfd4 --- /dev/null +++ b/analysis/tokenizer/unicode/unicode_test.go @@ -0,0 +1,202 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unicode + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/segment" +) + +func TestUnicode(t *testing.T) { + + tests := []struct { + input []byte + output analysis.TokenStream + }{ + { + []byte("Hello World"), + analysis.TokenStream{ + { + Start: 0, + End: 5, + Term: []byte("Hello"), + Position: 1, + Type: analysis.AlphaNumeric, + }, + { + Start: 6, + End: 11, + Term: []byte("World"), + Position: 2, + Type: analysis.AlphaNumeric, + }, + }, + }, + { + []byte("steven's"), + analysis.TokenStream{ + { + Start: 0, + End: 8, + Term: []byte("steven's"), + Position: 1, + Type: analysis.AlphaNumeric, + }, + }, + }, + { + []byte("こんにちは世界"), + analysis.TokenStream{ + { + Start: 0, + End: 3, + Term: []byte("こ"), + Position: 1, + Type: analysis.Ideographic, + }, + { + Start: 3, + End: 6, + Term: []byte("ん"), + Position: 2, + Type: analysis.Ideographic, + }, + { + Start: 6, + End: 9, + Term: []byte("に"), + Position: 3, + Type: analysis.Ideographic, + }, + { + Start: 9, + End: 12, + Term: []byte("ち"), + Position: 4, + Type: analysis.Ideographic, + }, + { + Start: 12, + End: 15, + Term: []byte("は"), + Position: 5, + Type: analysis.Ideographic, + }, + { + Start: 15, + End: 18, + Term: []byte("世"), + Position: 6, + Type: analysis.Ideographic, + }, + { + Start: 18, + End: 21, + Term: []byte("界"), + Position: 7, + Type: analysis.Ideographic, + }, + }, + }, + { + []byte("age 25"), + analysis.TokenStream{ + { + Start: 0, + End: 3, + Term: []byte("age"), + Position: 1, + Type: analysis.AlphaNumeric, + }, + { + Start: 4, + End: 6, + Term: []byte("25"), + Position: 2, + Type: analysis.Numeric, + }, + }, + }, + { + []byte("カ"), + analysis.TokenStream{ + { + Start: 0, + End: 3, + Term: []byte("カ"), + Position: 1, + Type: analysis.Ideographic, + }, + }, + }, + } + + for _, test := range tests { + tokenizer := NewUnicodeTokenizer() + actual := tokenizer.Tokenize(test.input) + + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("Expected %v, got %v for %s", test.output, actual, string(test.input)) + } + } +} + +var sampleLargeInput = []byte(`There are three characteristics of liquids which are relevant to the discussion of a BLEVE: +If a liquid in a sealed container is boiled, the pressure inside the container increases. As the liquid changes to a gas it expands - this expansion in a vented container would cause the gas and liquid to take up more space. In a sealed container the gas and liquid are not able to take up more space and so the pressure rises. Pressurized vessels containing liquids can reach an equilibrium where the liquid stops boiling and the pressure stops rising. This occurs when no more heat is being added to the system (either because it has reached ambient temperature or has had a heat source removed). +The boiling temperature of a liquid is dependent on pressure - high pressures will yield high boiling temperatures, and low pressures will yield low boiling temperatures. A common simple experiment is to place a cup of water in a vacuum chamber, and then reduce the pressure in the chamber until the water boils. By reducing the pressure the water will boil even at room temperature. This works both ways - if the pressure is increased beyond normal atmospheric pressures, the boiling of hot water could be suppressed far beyond normal temperatures. The cooling system of a modern internal combustion engine is a real-world example. +When a liquid boils it turns into a gas. The resulting gas takes up far more space than the liquid did. +Typically, a BLEVE starts with a container of liquid which is held above its normal, atmospheric-pressure boiling temperature. Many substances normally stored as liquids, such as CO2, oxygen, and other similar industrial gases have boiling temperatures, at atmospheric pressure, far below room temperature. In the case of water, a BLEVE could occur if a pressurized chamber of water is heated far beyond the standard 100 °C (212 °F). That container, because the boiling water pressurizes it, is capable of holding liquid water at very high temperatures. +If the pressurized vessel, containing liquid at high temperature (which may be room temperature, depending on the substance) ruptures, the pressure which prevents the liquid from boiling is lost. If the rupture is catastrophic, where the vessel is immediately incapable of holding any pressure at all, then there suddenly exists a large mass of liquid which is at very high temperature and very low pressure. This causes the entire volume of liquid to instantaneously boil, which in turn causes an extremely rapid expansion. Depending on temperatures, pressures and the substance involved, that expansion may be so rapid that it can be classified as an explosion, fully capable of inflicting severe damage on its surroundings.`) + +func BenchmarkTokenizeEnglishText(b *testing.B) { + + tokenizer := NewUnicodeTokenizer() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + tokenizer.Tokenize(sampleLargeInput) + } + +} + +func TestConvertType(t *testing.T) { + tests := []struct { + in int + out analysis.TokenType + }{ + { + segment.Ideo, analysis.Ideographic, + }, + { + segment.Kana, analysis.Ideographic, + }, + { + segment.Number, analysis.Numeric, + }, + { + segment.Letter, analysis.AlphaNumeric, + }, + } + + for _, test := range tests { + actual := convertType(test.in) + if actual != test.out { + t.Errorf("expected %d, got %d for %d", test.out, actual, test.in) + } + } +} diff --git a/analysis/tokenizer/web/web.go b/analysis/tokenizer/web/web.go new file mode 100644 index 0000000..a534f20 --- /dev/null +++ b/analysis/tokenizer/web/web.go @@ -0,0 +1,50 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package web + +import ( + "regexp" + "strings" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/exception" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "web" + +var email = `(?:[a-z0-9!#$%&'*+/=?^_` + "`" + `{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_` + "`" + `{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])` +var url = `(?i)\b((?:[a-z][\w-]+:(?:/{1,3}|[a-z0-9%])|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s` + "`" + `!()\[\]{};:'".,<>?«»“”‘’]))` +var twitterHandle = `@([a-zA-Z0-9_]){1,15}` +var twitterHashtag = `#([a-zA-Z0-9_])+` +var exceptions = []string{email, url, twitterHandle, twitterHashtag} + +var exceptionsRegexp = regexp.MustCompile(strings.Join(exceptions, "|")) + +func TokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { + remainingTokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + return exception.NewExceptionsTokenizer(exceptionsRegexp, remainingTokenizer), nil +} + +func init() { + err := registry.RegisterTokenizer(Name, TokenizerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/tokenizer/web/web_test.go b/analysis/tokenizer/web/web_test.go new file mode 100644 index 0000000..d5dbcce --- /dev/null +++ b/analysis/tokenizer/web/web_test.go @@ -0,0 +1,148 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package web + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +func TestWeb(t *testing.T) { + + tests := []struct { + input []byte + output analysis.TokenStream + }{ + { + []byte("Hello info@blevesearch.com"), + analysis.TokenStream{ + { + Start: 0, + End: 5, + Term: []byte("Hello"), + Position: 1, + Type: analysis.AlphaNumeric, + }, + { + Start: 6, + End: 26, + Term: []byte("info@blevesearch.com"), + Position: 2, + Type: analysis.AlphaNumeric, + }, + }, + }, + { + []byte("That http://blevesearch.com"), + analysis.TokenStream{ + { + Start: 0, + End: 4, + Term: []byte("That"), + Position: 1, + Type: analysis.AlphaNumeric, + }, + { + Start: 5, + End: 27, + Term: []byte("http://blevesearch.com"), + Position: 2, + Type: analysis.AlphaNumeric, + }, + }, + }, + { + []byte("Hey @blevesearch"), + analysis.TokenStream{ + { + Start: 0, + End: 3, + Term: []byte("Hey"), + Position: 1, + Type: analysis.AlphaNumeric, + }, + { + Start: 4, + End: 16, + Term: []byte("@blevesearch"), + Position: 2, + Type: analysis.AlphaNumeric, + }, + }, + }, + { + []byte("This #bleve"), + analysis.TokenStream{ + { + Start: 0, + End: 4, + Term: []byte("This"), + Position: 1, + Type: analysis.AlphaNumeric, + }, + { + Start: 5, + End: 11, + Term: []byte("#bleve"), + Position: 2, + Type: analysis.AlphaNumeric, + }, + }, + }, + { + []byte("What about @blevesearch?"), + analysis.TokenStream{ + { + Start: 0, + End: 4, + Term: []byte("What"), + Position: 1, + Type: analysis.AlphaNumeric, + }, + { + Start: 5, + End: 10, + Term: []byte("about"), + Position: 2, + Type: analysis.AlphaNumeric, + }, + { + Start: 11, + End: 23, + Term: []byte("@blevesearch"), + Position: 3, + Type: analysis.AlphaNumeric, + }, + }, + }, + } + + cache := registry.NewCache() + tokenizer, err := cache.TokenizerNamed(Name) + if err != nil { + t.Fatal(err) + } + + for _, test := range tests { + + actual := tokenizer.Tokenize(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("Expected %v, got %v for %s", test.output, actual, string(test.input)) + } + } +} diff --git a/analysis/tokenizer/whitespace/whitespace.go b/analysis/tokenizer/whitespace/whitespace.go new file mode 100644 index 0000000..d8b14bb --- /dev/null +++ b/analysis/tokenizer/whitespace/whitespace.go @@ -0,0 +1,40 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package whitespace + +import ( + "unicode" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/character" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "whitespace" + +func TokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { + return character.NewCharacterTokenizer(notSpace), nil +} + +func notSpace(r rune) bool { + return !unicode.IsSpace(r) +} + +func init() { + err := registry.RegisterTokenizer(Name, TokenizerConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/tokenizer/whitespace/whitespace_test.go b/analysis/tokenizer/whitespace/whitespace_test.go new file mode 100644 index 0000000..6bd6b54 --- /dev/null +++ b/analysis/tokenizer/whitespace/whitespace_test.go @@ -0,0 +1,106 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package whitespace + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/character" +) + +func TestBoundary(t *testing.T) { + + tests := []struct { + input []byte + output analysis.TokenStream + }{ + { + []byte("Hello World."), + analysis.TokenStream{ + { + Start: 0, + End: 5, + Term: []byte("Hello"), + Position: 1, + Type: analysis.AlphaNumeric, + }, + { + Start: 6, + End: 12, + Term: []byte("World."), + Position: 2, + Type: analysis.AlphaNumeric, + }, + }, + }, + { + []byte("こんにちは世界"), + analysis.TokenStream{ + { + Start: 0, + End: 21, + Term: []byte("こんにちは世界"), + Position: 1, + Type: analysis.AlphaNumeric, + }, + }, + }, + { + []byte(""), + analysis.TokenStream{}, + }, + { + []byte("abc界"), + analysis.TokenStream{ + { + Start: 0, + End: 6, + Term: []byte("abc界"), + Position: 1, + Type: analysis.AlphaNumeric, + }, + }, + }, + } + + for _, test := range tests { + tokenizer := character.NewCharacterTokenizer(notSpace) + actual := tokenizer.Tokenize(test.input) + + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("Expected %v, got %v for %s", test.output, actual, string(test.input)) + } + } +} + +var sampleLargeInput = []byte(`There are three characteristics of liquids which are relevant to the discussion of a BLEVE: +If a liquid in a sealed container is boiled, the pressure inside the container increases. As the liquid changes to a gas it expands - this expansion in a vented container would cause the gas and liquid to take up more space. In a sealed container the gas and liquid are not able to take up more space and so the pressure rises. Pressurized vessels containing liquids can reach an equilibrium where the liquid stops boiling and the pressure stops rising. This occurs when no more heat is being added to the system (either because it has reached ambient temperature or has had a heat source removed). +The boiling temperature of a liquid is dependent on pressure - high pressures will yield high boiling temperatures, and low pressures will yield low boiling temperatures. A common simple experiment is to place a cup of water in a vacuum chamber, and then reduce the pressure in the chamber until the water boils. By reducing the pressure the water will boil even at room temperature. This works both ways - if the pressure is increased beyond normal atmospheric pressures, the boiling of hot water could be suppressed far beyond normal temperatures. The cooling system of a modern internal combustion engine is a real-world example. +When a liquid boils it turns into a gas. The resulting gas takes up far more space than the liquid did. +Typically, a BLEVE starts with a container of liquid which is held above its normal, atmospheric-pressure boiling temperature. Many substances normally stored as liquids, such as CO2, oxygen, and other similar industrial gases have boiling temperatures, at atmospheric pressure, far below room temperature. In the case of water, a BLEVE could occur if a pressurized chamber of water is heated far beyond the standard 100 °C (212 °F). That container, because the boiling water pressurizes it, is capable of holding liquid water at very high temperatures. +If the pressurized vessel, containing liquid at high temperature (which may be room temperature, depending on the substance) ruptures, the pressure which prevents the liquid from boiling is lost. If the rupture is catastrophic, where the vessel is immediately incapable of holding any pressure at all, then there suddenly exists a large mass of liquid which is at very high temperature and very low pressure. This causes the entire volume of liquid to instantaneously boil, which in turn causes an extremely rapid expansion. Depending on temperatures, pressures and the substance involved, that expansion may be so rapid that it can be classified as an explosion, fully capable of inflicting severe damage on its surroundings.`) + +func BenchmarkTokenizeEnglishText(b *testing.B) { + + tokenizer := character.NewCharacterTokenizer(notSpace) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + tokenizer.Tokenize(sampleLargeInput) + } + +} diff --git a/analysis/tokenmap.go b/analysis/tokenmap.go new file mode 100644 index 0000000..aa4ea31 --- /dev/null +++ b/analysis/tokenmap.go @@ -0,0 +1,76 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package analysis + +import ( + "bufio" + "bytes" + "io" + "os" + "strings" +) + +type TokenMap map[string]bool + +func NewTokenMap() TokenMap { + return make(TokenMap, 0) +} + +// LoadFile reads in a list of tokens from a text file, +// one per line. +// Comments are supported using `#` or `|` +func (t TokenMap) LoadFile(filename string) error { + data, err := os.ReadFile(filename) + if err != nil { + return err + } + return t.LoadBytes(data) +} + +// LoadBytes reads in a list of tokens from memory, +// one per line. +// Comments are supported using `#` or `|` +func (t TokenMap) LoadBytes(data []byte) error { + bytesReader := bytes.NewReader(data) + bufioReader := bufio.NewReader(bytesReader) + line, err := bufioReader.ReadString('\n') + for err == nil { + t.LoadLine(line) + line, err = bufioReader.ReadString('\n') + } + // if the err was EOF we still need to process the last value + if err == io.EOF { + t.LoadLine(line) + return nil + } + return err +} + +func (t TokenMap) LoadLine(line string) { + // find the start of a comment, if any + startComment := strings.IndexAny(line, "#|") + if startComment >= 0 { + line = line[:startComment] + } + + tokens := strings.Fields(line) + for _, token := range tokens { + t.AddToken(token) + } +} + +func (t TokenMap) AddToken(token string) { + t[token] = true +} diff --git a/analysis/tokenmap/custom.go b/analysis/tokenmap/custom.go new file mode 100644 index 0000000..9567852 --- /dev/null +++ b/analysis/tokenmap/custom.go @@ -0,0 +1,65 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package token_map implements a generic TokenMap, often used in conjunction +// with filters to remove or process specific tokens. +// +// Its constructor takes the following arguments: +// +// "filename" (string): the path of a file listing the tokens. Each line may +// contain one or more whitespace separated tokens, followed by an optional +// comment starting with a "#" or "|" character. +// +// "tokens" ([]interface{}): if "filename" is not specified, tokens can be +// passed directly as a sequence of strings wrapped in a []interface{}. +package tokenmap + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "custom" + +func GenericTokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { + rv := analysis.NewTokenMap() + + // first: try to load by filename + filename, ok := config["filename"].(string) + if ok { + err := rv.LoadFile(filename) + return rv, err + } + // next: look for an inline word list + tokens, ok := config["tokens"].([]interface{}) + if ok { + for _, token := range tokens { + tokenStr, ok := token.(string) + if ok { + rv.AddToken(tokenStr) + } + } + return rv, nil + } + return nil, fmt.Errorf("must specify filename or list of tokens for token map") +} + +func init() { + err := registry.RegisterTokenMap(Name, GenericTokenMapConstructor) + if err != nil { + panic(err) + } +} diff --git a/analysis/tokenmap_test.go b/analysis/tokenmap_test.go new file mode 100644 index 0000000..aba2286 --- /dev/null +++ b/analysis/tokenmap_test.go @@ -0,0 +1,43 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package analysis + +import ( + "reflect" + "testing" +) + +func TestTokenMapLoadFile(t *testing.T) { + tokenMap := NewTokenMap() + err := tokenMap.LoadFile("test_words.txt") + if err != nil { + t.Fatal(err) + } + + expectedTokens := NewTokenMap() + expectedTokens.AddToken("marty") + expectedTokens.AddToken("steve") + expectedTokens.AddToken("dustin") + expectedTokens.AddToken("siri") + expectedTokens.AddToken("multiple") + expectedTokens.AddToken("words") + expectedTokens.AddToken("with") + expectedTokens.AddToken("different") + expectedTokens.AddToken("whitespace") + + if !reflect.DeepEqual(tokenMap, expectedTokens) { + t.Errorf("expected %#v, got %#v", expectedTokens, tokenMap) + } +} diff --git a/analysis/type.go b/analysis/type.go new file mode 100644 index 0000000..f819984 --- /dev/null +++ b/analysis/type.go @@ -0,0 +1,120 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package analysis + +import ( + "fmt" + "time" +) + +type CharFilter interface { + Filter([]byte) []byte +} + +type TokenType int + +const ( + AlphaNumeric TokenType = iota + Ideographic + Numeric + DateTime + Shingle + Single + Double + Boolean + IP +) + +// Token represents one occurrence of a term at a particular location in a +// field. +type Token struct { + // Start specifies the byte offset of the beginning of the term in the + // field. + Start int `json:"start"` + + // End specifies the byte offset of the end of the term in the field. + End int `json:"end"` + Term []byte `json:"term"` + + // Position specifies the 1-based index of the token in the sequence of + // occurrences of its term in the field. + Position int `json:"position"` + Type TokenType `json:"type"` + KeyWord bool `json:"keyword"` +} + +func (t *Token) String() string { + return fmt.Sprintf("Start: %d End: %d Position: %d Token: %s Type: %d", t.Start, t.End, t.Position, string(t.Term), t.Type) +} + +type TokenStream []*Token + +// A Tokenizer splits an input string into tokens, the usual behaviour being to +// map words to tokens. +type Tokenizer interface { + Tokenize([]byte) TokenStream +} + +// A TokenFilter adds, transforms or removes tokens from a token stream. +type TokenFilter interface { + Filter(TokenStream) TokenStream +} + +type Analyzer interface { + Analyze([]byte) TokenStream +} + +type DefaultAnalyzer struct { + CharFilters []CharFilter + Tokenizer Tokenizer + TokenFilters []TokenFilter +} + +func (a *DefaultAnalyzer) Analyze(input []byte) TokenStream { + if a.CharFilters != nil { + for _, cf := range a.CharFilters { + input = cf.Filter(input) + } + } + tokens := a.Tokenizer.Tokenize(input) + if a.TokenFilters != nil { + for _, tf := range a.TokenFilters { + tokens = tf.Filter(tokens) + } + } + return tokens +} + +var ErrInvalidDateTime = fmt.Errorf("unable to parse datetime with any of the layouts") + +var ErrInvalidTimestampString = fmt.Errorf("unable to parse timestamp string") +var ErrInvalidTimestampRange = fmt.Errorf("timestamp out of range") + +type DateTimeParser interface { + ParseDateTime(string) (time.Time, string, error) +} + +const SynonymSourceType = "synonym" + +type SynonymSourceVisitor func(name string, item SynonymSource) error + +type SynonymSource interface { + Analyzer() string + Collection() string +} + +type ByteArrayConverter interface { + Convert([]byte) (interface{}, error) +} diff --git a/analysis/util.go b/analysis/util.go new file mode 100644 index 0000000..8e4348a --- /dev/null +++ b/analysis/util.go @@ -0,0 +1,92 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package analysis + +import ( + "bytes" + "unicode/utf8" +) + +func DeleteRune(in []rune, pos int) []rune { + if pos >= len(in) { + return in + } + copy(in[pos:], in[pos+1:]) + return in[:len(in)-1] +} + +func InsertRune(in []rune, pos int, r rune) []rune { + // create a new slice 1 rune larger + rv := make([]rune, len(in)+1) + // copy the characters before the insert pos + copy(rv[0:pos], in[0:pos]) + // set the inserted rune + rv[pos] = r + // copy the characters after the insert pos + copy(rv[pos+1:], in[pos:]) + return rv +} + +// BuildTermFromRunesOptimistic will build a term from the provided runes +// AND optimistically attempt to encode into the provided buffer +// if at any point it appears the buffer is too small, a new buffer is +// allocated and that is used instead +// this should be used in cases where frequently the new term is the same +// length or shorter than the original term (in number of bytes) +func BuildTermFromRunesOptimistic(buf []byte, runes []rune) []byte { + rv := buf + used := 0 + for _, r := range runes { + nextLen := utf8.RuneLen(r) + if used+nextLen > len(rv) { + // alloc new buf + buf = make([]byte, len(runes)*utf8.UTFMax) + // copy work we've already done + copy(buf, rv[:used]) + rv = buf + } + written := utf8.EncodeRune(rv[used:], r) + used += written + } + return rv[:used] +} + +func BuildTermFromRunes(runes []rune) []byte { + return BuildTermFromRunesOptimistic(make([]byte, len(runes)*utf8.UTFMax), runes) +} + +func TruncateRunes(input []byte, num int) []byte { + runes := bytes.Runes(input) + runes = runes[:len(runes)-num] + out := BuildTermFromRunes(runes) + return out +} + +func RunesEndsWith(input []rune, suffix string) bool { + inputLen := len(input) + suffixRunes := []rune(suffix) + suffixLen := len(suffixRunes) + if suffixLen > inputLen { + return false + } + + for i := suffixLen - 1; i >= 0; i-- { + if input[inputLen-(suffixLen-i)] != suffixRunes[i] { + return false + } + } + + return true +} diff --git a/analysis/util_test.go b/analysis/util_test.go new file mode 100644 index 0000000..ec4f89b --- /dev/null +++ b/analysis/util_test.go @@ -0,0 +1,140 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package analysis + +import ( + "reflect" + "testing" +) + +func TestDeleteRune(t *testing.T) { + tests := []struct { + in []rune + delPos int + out []rune + }{ + { + in: []rune{'a', 'b', 'c'}, + delPos: 1, + out: []rune{'a', 'c'}, + }, + } + + for _, test := range tests { + actual := DeleteRune(test.in, test.delPos) + if !reflect.DeepEqual(actual, test.out) { + t.Errorf("expected %#v, got %#v", test.out, actual) + } + } +} + +func TestInsertRune(t *testing.T) { + tests := []struct { + in []rune + insPos int + insRune rune + out []rune + }{ + { + in: []rune{'a', 'b', 'c'}, + insPos: 1, + insRune: 'x', + out: []rune{'a', 'x', 'b', 'c'}, + }, + { + in: []rune{'a', 'b', 'c'}, + insPos: 0, + insRune: 'x', + out: []rune{'x', 'a', 'b', 'c'}, + }, + { + in: []rune{'a', 'b', 'c'}, + insPos: 3, + insRune: 'x', + out: []rune{'a', 'b', 'c', 'x'}, + }, + } + + for _, test := range tests { + actual := InsertRune(test.in, test.insPos, test.insRune) + if !reflect.DeepEqual(actual, test.out) { + t.Errorf("expected %#v, got %#v", test.out, actual) + } + } +} + +func TestBuildTermFromRunes(t *testing.T) { + tests := []struct { + in []rune + }{ + { + in: []rune{'a', 'b', 'c'}, + }, + { + in: []rune{'こ', 'ん', 'に', 'ち', 'は', '世', '界'}, + }, + } + for _, test := range tests { + out := BuildTermFromRunes(test.in) + back := []rune(string(out)) + if !reflect.DeepEqual(back, test.in) { + t.Errorf("expected %v to convert back to %v", out, test.in) + } + } +} + +func TestBuildTermFromRunesOptimistic(t *testing.T) { + tests := []struct { + buf []byte + in []rune + }{ + { + buf: []byte("abc"), + in: []rune{'a', 'b', 'c'}, + }, + { + buf: []byte("こんにちは世界"), + in: []rune{'こ', 'ん', 'に', 'ち', 'は', '世', '界'}, + }, + // same, but don't give enough buffer + { + buf: []byte("ab"), + in: []rune{'a', 'b', 'c'}, + }, + { + buf: []byte("こ"), + in: []rune{'こ', 'ん', 'に', 'ち', 'は', '世', '界'}, + }, + } + for _, test := range tests { + out := BuildTermFromRunesOptimistic(test.buf, test.in) + back := []rune(string(out)) + if !reflect.DeepEqual(back, test.in) { + t.Errorf("expected %v to convert back to %v", out, test.in) + } + } +} + +func BenchmarkBuildTermFromRunes(b *testing.B) { + input := [][]rune{ + {'a', 'b', 'c'}, + {'こ', 'ん', 'に', 'ち', 'は', '世', '界'}, + } + for i := 0; i < b.N; i++ { + for _, i := range input { + BuildTermFromRunes(i) + } + } +} diff --git a/builder.go b/builder.go new file mode 100644 index 0000000..5398739 --- /dev/null +++ b/builder.go @@ -0,0 +1,97 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +type builderImpl struct { + b index.IndexBuilder + m mapping.IndexMapping +} + +func (b *builderImpl) Index(id string, data interface{}) error { + if id == "" { + return ErrorEmptyID + } + + doc := document.NewDocument(id) + err := b.m.MapDocument(doc, data) + if err != nil { + return err + } + err = b.b.Index(doc) + return err +} + +func (b *builderImpl) Close() error { + return b.b.Close() +} + +func newBuilder(path string, mapping mapping.IndexMapping, config map[string]interface{}) (Builder, error) { + if path == "" { + return nil, fmt.Errorf("builder requires path") + } + + err := mapping.Validate() + if err != nil { + return nil, err + } + + if config == nil { + config = map[string]interface{}{} + } + + // the builder does not have an API to interact with internal storage + // however we can pass k/v pairs through the config + mappingBytes, err := util.MarshalJSON(mapping) + if err != nil { + return nil, err + } + config["internal"] = map[string][]byte{ + string(util.MappingInternalKey): mappingBytes, + } + + // do not use real config, as these are options for the builder, + // not the resulting index + meta, err := newIndexMeta(scorch.Name, scorch.Name, map[string]interface{}{}, path) + if err != nil { + return nil, err + } + err = meta.Save(path) + if err != nil { + return nil, err + } + + config["path"] = indexStorePath(path) + + b, err := scorch.NewBuilder(config) + if err != nil { + return nil, err + } + rv := &builderImpl{ + b: b, + m: mapping, + } + + return rv, nil +} diff --git a/builder_test.go b/builder_test.go new file mode 100644 index 0000000..cd9de3e --- /dev/null +++ b/builder_test.go @@ -0,0 +1,88 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "fmt" + "os" + "testing" +) + +func TestBuilder(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "bleve-scorch-builder-test") + if err != nil { + t.Fatal(err) + } + defer func() { + err = os.RemoveAll(tmpDir) + if err != nil { + t.Fatalf("error cleaning up test index") + } + }() + + conf := map[string]interface{}{ + "batchSize": 2, + "mergeMax": 2, + } + b, err := NewBuilder(tmpDir, NewIndexMapping(), conf) + if err != nil { + t.Fatal(err) + } + + for i := 0; i < 10; i++ { + doc := map[string]interface{}{ + "name": "hello", + } + err = b.Index(fmt.Sprintf("%d", i), doc) + if err != nil { + t.Fatal(err) + } + } + + err = b.Close() + if err != nil { + t.Fatal(err) + } + + idx, err := Open(tmpDir) + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err = idx.Close() + if err != nil { + t.Errorf("error closing index: %v", err) + } + }() + + docCount, err := idx.DocCount() + if err != nil { + t.Errorf("error checking doc count: %v", err) + } + if docCount != 10 { + t.Errorf("expected doc count to be 10, got %d", docCount) + } + + q := NewTermQuery("hello") + q.SetField("name") + req := NewSearchRequest(q) + res, err := idx.Search(req) + if err != nil { + t.Errorf("error searching index: %v", err) + } + if res.Total != 10 { + t.Errorf("expected 10 search hits, got %d", res.Total) + } +} diff --git a/cmd/bleve/cmd/bulk.go b/cmd/bleve/cmd/bulk.go new file mode 100644 index 0000000..458c3c7 --- /dev/null +++ b/cmd/bleve/cmd/bulk.go @@ -0,0 +1,116 @@ +// Copyright © 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "bufio" + "encoding/json" + "fmt" + "math/rand" + "os" + + "github.com/spf13/cobra" +) + +var batchSize int + +// bulkCmd represents the bulk command +var bulkCmd = &cobra.Command{ + Use: "bulk [index path] [data paths ...]", + Short: "bulk loads from newline delimited JSON files", + Long: `The bulk command will perform batch loading of documents in one or more newline delimited JSON files.`, + Annotations: map[string]string{ + canMutateBleveIndex: "true", + }, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) < 2 { + return fmt.Errorf("must specify at least one path") + } + + i := 0 + batch := idx.NewBatch() + + for _, file := range args[1:] { + + file, err := os.Open(file) + if err != nil { + return err + } + + fmt.Printf("Indexing: %s\n", file.Name()) + r := bufio.NewReader(file) + + for { + if i%batchSize == 0 { + fmt.Printf("Indexing batch (%d docs)...\n", i) + err := idx.Batch(batch) + if err != nil { + return err + } + batch = idx.NewBatch() + } + + b, _ := r.ReadBytes('\n') + if len(b) == 0 { + break + } + + var doc interface{} = b + var err error + if parseJSON { + err = json.Unmarshal(b, &doc) + if err != nil { + return fmt.Errorf("error parsing JSON: %v", err) + } + } + + docID := randomString(5) + err = batch.Index(docID, doc) + if err != nil { + return err + } + i++ + } + err = idx.Batch(batch) + if err != nil { + return err + } + + err = file.Close() + if err != nil { + return err + } + + } + return nil + }, +} + +var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + +func randomString(n int) string { + b := make([]rune, n) + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + return string(b) +} + +func init() { + RootCmd.AddCommand(bulkCmd) + + bulkCmd.Flags().IntVarP(&batchSize, "batch", "b", 1000, "Batch size for loading.") + bulkCmd.Flags().BoolVarP(&parseJSON, "json", "j", true, "Parse the contents as JSON.") +} diff --git a/cmd/bleve/cmd/check.go b/cmd/bleve/cmd/check.go new file mode 100644 index 0000000..ca4f286 --- /dev/null +++ b/cmd/bleve/cmd/check.go @@ -0,0 +1,131 @@ +// Copyright © 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "log" + + "github.com/blevesearch/bleve/v2" + "github.com/spf13/cobra" +) + +var checkFieldName string +var checkCount int + +// checkCmd represents the check command +var checkCmd = &cobra.Command{ + Use: "check [index path]", + Short: "checks the contents of the index", + Long: `The check command will perform consistency checks on the index.`, + RunE: func(cmd *cobra.Command, args []string) error { + + var fieldNames []string + var err error + if checkFieldName == "" { + fieldNames, err = idx.Fields() + if err != nil { + return err + } + } else { + fieldNames = []string{checkFieldName} + } + fmt.Printf("checking fields: %v\n", fieldNames) + + totalProblems := 0 + for _, fieldName := range fieldNames { + fmt.Printf("checking field: '%s'\n", fieldName) + problems, err := checkField(idx, fieldName) + if err != nil { + log.Fatal(err) + } + totalProblems += problems + } + + if totalProblems != 0 { + return fmt.Errorf("found %d total problems\n", totalProblems) + } + + return nil + }, +} + +func checkField(index bleve.Index, fieldName string) (int, error) { + termDictionary, err := getDictionary(index, fieldName) + if err != nil { + return 0, err + } + fmt.Printf("field contains %d terms\n", len(termDictionary)) + + numTested := 0 + numProblems := 0 + for term, count := range termDictionary { + fmt.Printf("checked %d terms\r", numTested) + if checkCount > 0 && numTested >= checkCount { + break + } + + tq := bleve.NewTermQuery(term) + tq.SetField(fieldName) + req := bleve.NewSearchRequest(tq) + req.Size = 0 + res, err := index.Search(req) + if err != nil { + return 0, err + } + + if res.Total != count { + fmt.Printf("unexpected mismatch for term '%s', dictionary %d, search hits %d\n", term, count, res.Total) + numProblems++ + } + + numTested++ + } + fmt.Printf("done checking %d terms, found %d problems\n", numTested, numProblems) + + return numProblems, nil +} + +func getDictionary(index bleve.Index, field string) (map[string]uint64, error) { + rv := make(map[string]uint64) + i, err := index.Advanced() + if err != nil { + log.Fatal(err) + } + r, err := i.Reader() + if err != nil { + log.Fatal(err) + } + d, err := r.FieldDict(field) + if err != nil { + log.Fatal(err) + } + + de, err := d.Next() + for err == nil && de != nil { + rv[de.Term] = de.Count + de, err = d.Next() + } + if err != nil { + return nil, err + } + return rv, nil +} + +func init() { + RootCmd.AddCommand(checkCmd) + checkCmd.Flags().StringVarP(&checkFieldName, "field", "f", "", "Restrict check to the specified field name.") + checkCmd.Flags().IntVarP(&checkCount, "count", "c", 100, "Check this many terms.") +} diff --git a/cmd/bleve/cmd/count.go b/cmd/bleve/cmd/count.go new file mode 100644 index 0000000..15ef4d6 --- /dev/null +++ b/cmd/bleve/cmd/count.go @@ -0,0 +1,40 @@ +// Copyright © 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// countCmd represents the count command +var countCmd = &cobra.Command{ + Use: "count [index path]", + Short: "counts the number documents in the index", + Long: `The count command will count the number of documents in the index.`, + RunE: func(cmd *cobra.Command, args []string) error { + count, err := idx.DocCount() + if err != nil { + return fmt.Errorf("error counting docs in index: %v", err) + } + fmt.Printf("%d\n", count) + return nil + }, +} + +func init() { + RootCmd.AddCommand(countCmd) +} diff --git a/cmd/bleve/cmd/create.go b/cmd/bleve/cmd/create.go new file mode 100644 index 0000000..fc6834f --- /dev/null +++ b/cmd/bleve/cmd/create.go @@ -0,0 +1,81 @@ +// Copyright © 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/spf13/cobra" +) + +var mappingPath, indexType, storeType string + +// createCmd represents the create command +var createCmd = &cobra.Command{ + Use: "create [index path]", + Short: "creates a new index", + Long: `The create command will create a new empty index.`, + Annotations: map[string]string{ + canMutateBleveIndex: "true", + }, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + // override RootCmd version which opens existing index + if len(args) < 1 { + return fmt.Errorf("must specify path to index") + } + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + var mapping mapping.IndexMapping + var err error + mapping, err = buildMapping() + if err != nil { + return fmt.Errorf("error building mapping: %v", err) + } + idx, err = bleve.NewUsing(args[0], mapping, indexType, storeType, nil) + if err != nil { + return fmt.Errorf("error creating index: %v", err) + } + // the inherited Post action will close the index + return nil + }, +} + +func buildMapping() (mapping.IndexMapping, error) { + mapping := mapping.NewIndexMapping() + if mappingPath != "" { + mappingBytes, err := os.ReadFile(mappingPath) + if err != nil { + return nil, err + } + err = json.Unmarshal(mappingBytes, &mapping) + if err != nil { + return nil, err + } + } + return mapping, nil +} + +func init() { + RootCmd.AddCommand(createCmd) + + createCmd.Flags().StringVarP(&mappingPath, "mapping", "m", "", "Path to a file containing a JSON representation of an index mapping to use.") + createCmd.Flags().StringVarP(&storeType, "store", "s", bleve.Config.DefaultKVStore, "The bleve storage type to use.") + createCmd.Flags().StringVarP(&indexType, "index", "i", bleve.Config.DefaultIndexType, "The bleve index type to use.") +} diff --git a/cmd/bleve/cmd/dictionary.go b/cmd/bleve/cmd/dictionary.go new file mode 100644 index 0000000..3f42d12 --- /dev/null +++ b/cmd/bleve/cmd/dictionary.go @@ -0,0 +1,60 @@ +// Copyright © 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// dictionaryCmd represents the dictionary command +var dictionaryCmd = &cobra.Command{ + Use: "dictionary [index path] [field name]", + Short: "prints the term dictionary for the specified field in the index", + Long: `The dictionary command will print the term dictionary for the specified field.`, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) < 2 { + return fmt.Errorf("must specify field") + } + i, err := idx.Advanced() + if err != nil { + return fmt.Errorf("error getting index: %v", err) + } + r, err := i.Reader() + if err != nil { + return fmt.Errorf("error getting index reader: %v", err) + } + d, err := r.FieldDict(args[1]) + if err != nil { + return fmt.Errorf("error getting field dictionary: %v", err) + } + + de, err := d.Next() + for err == nil && de != nil { + fmt.Printf("%s - %d\n", de.Term, de.Count) + de, err = d.Next() + } + if err != nil { + return fmt.Errorf("error iterating dictionary: %v", err) + } + + return nil + }, +} + +func init() { + RootCmd.AddCommand(dictionaryCmd) +} diff --git a/cmd/bleve/cmd/dump.go b/cmd/bleve/cmd/dump.go new file mode 100644 index 0000000..db76eeb --- /dev/null +++ b/cmd/bleve/cmd/dump.go @@ -0,0 +1,61 @@ +// Copyright © 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/index/upsidedown" + "github.com/spf13/cobra" +) + +var docID string + +// dumpCmd represents the dump command +var dumpCmd = &cobra.Command{ + Use: "dump [index path]", + Short: "dumps the contents of the index", + Long: `The dump command will dump (possibly a section of) the index.`, + RunE: func(cmd *cobra.Command, args []string) error { + i, err := idx.Advanced() + if err != nil { + return fmt.Errorf("error getting index: %v", err) + } + r, err := i.Reader() + if err != nil { + return fmt.Errorf("error getting index reader: %v", err) + } + upsideDownReader, ok := r.(*upsidedown.IndexReader) + if !ok { + return fmt.Errorf("dump is only supported by index type upsidedown") + } + + dumpChan := upsideDownReader.DumpAll() + for rowOrErr := range dumpChan { + switch rowOrErr := rowOrErr.(type) { + case error: + return fmt.Errorf("error dumping: %v", rowOrErr) + case upsidedown.UpsideDownCouchRow: + fmt.Printf("%v\n", rowOrErr) + fmt.Printf("Key: % -100x\nValue: % -100x\n\n", rowOrErr.Key(), rowOrErr.Value()) + } + } + return nil + }, +} + +func init() { + RootCmd.AddCommand(dumpCmd) +} diff --git a/cmd/bleve/cmd/dumpDoc.go b/cmd/bleve/cmd/dumpDoc.go new file mode 100644 index 0000000..4473ab6 --- /dev/null +++ b/cmd/bleve/cmd/dumpDoc.go @@ -0,0 +1,63 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/index/upsidedown" + "github.com/spf13/cobra" +) + +// dumpDocCmd represents the dumpDoc command +var dumpDocCmd = &cobra.Command{ + Use: "doc [index path] [doc id]", + Short: "dump only the rows relating to this doc ID", + Long: `The doc sub-command of dump will only dump the rows relating to this doc ID.`, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) < 2 { + return fmt.Errorf("must specify docid") + } + + i, err := idx.Advanced() + if err != nil { + return fmt.Errorf("error getting index: %v", err) + } + r, err := i.Reader() + if err != nil { + return fmt.Errorf("error getting index reader: %v", err) + } + upsideDownReader, ok := r.(*upsidedown.IndexReader) + if !ok { + return fmt.Errorf("dump doc is only supported by index type upsidedown") + } + + dumpChan := upsideDownReader.DumpDoc(args[1]) + for rowOrErr := range dumpChan { + switch rowOrErr := rowOrErr.(type) { + case error: + return fmt.Errorf("error dumping: %v", rowOrErr) + case upsidedown.UpsideDownCouchRow: + fmt.Printf("%v\n", rowOrErr) + fmt.Printf("Key: % -100x\nValue: % -100x\n\n", rowOrErr.Key(), rowOrErr.Value()) + } + } + return nil + }, +} + +func init() { + dumpCmd.AddCommand(dumpDocCmd) +} diff --git a/cmd/bleve/cmd/dumpFields.go b/cmd/bleve/cmd/dumpFields.go new file mode 100644 index 0000000..0021ff9 --- /dev/null +++ b/cmd/bleve/cmd/dumpFields.go @@ -0,0 +1,59 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/index/upsidedown" + "github.com/spf13/cobra" +) + +// dumpFieldsCmd represents the dumpFields command +var dumpFieldsCmd = &cobra.Command{ + Use: "fields [index path]", + Short: "dump only the field rows", + Long: `The fields sub-command of dump will only dump the field rows.`, + RunE: func(cmd *cobra.Command, args []string) error { + i, err := idx.Advanced() + if err != nil { + return fmt.Errorf("error getting index: %v", err) + } + r, err := i.Reader() + if err != nil { + return fmt.Errorf("error getting index reader: %v", err) + } + upsideDownReader, ok := r.(*upsidedown.IndexReader) + if !ok { + return fmt.Errorf("dump fields is only supported by index type upsidedown") + } + + dumpChan := upsideDownReader.DumpFields() + for rowOrErr := range dumpChan { + switch rowOrErr := rowOrErr.(type) { + case error: + return fmt.Errorf("error dumping: %v", rowOrErr) + case upsidedown.UpsideDownCouchRow: + fmt.Printf("%v\n", rowOrErr) + fmt.Printf("Key: % -100x\nValue: % -100x\n\n", rowOrErr.Key(), rowOrErr.Value()) + } + } + return nil + }, +} + +func init() { + dumpCmd.AddCommand(dumpFieldsCmd) +} diff --git a/cmd/bleve/cmd/fields.go b/cmd/bleve/cmd/fields.go new file mode 100644 index 0000000..11b369f --- /dev/null +++ b/cmd/bleve/cmd/fields.go @@ -0,0 +1,50 @@ +// Copyright © 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// fieldsCmd represents the fields command +var fieldsCmd = &cobra.Command{ + Use: "fields [index path]", + Short: "lists the fields in this index", + Long: `The fields command will list the fields used in this index.`, + RunE: func(cmd *cobra.Command, args []string) error { + i, err := idx.Advanced() + if err != nil { + return fmt.Errorf("error getting index: %v", err) + } + r, err := i.Reader() + if err != nil { + return fmt.Errorf("error getting index reader: %v", err) + } + fields, err := r.Fields() + if err != nil { + return fmt.Errorf("error getting fields: %v", err) + } + for i, field := range fields { + fmt.Printf("%d - %s\n", i, field) + } + return nil + }, +} + +func init() { + RootCmd.AddCommand(fieldsCmd) +} diff --git a/cmd/bleve/cmd/index.go b/cmd/bleve/cmd/index.go new file mode 100644 index 0000000..bfc27fc --- /dev/null +++ b/cmd/bleve/cmd/index.go @@ -0,0 +1,116 @@ +// Copyright © 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + + "github.com/spf13/cobra" +) + +var keepDir, keepExt, parseJSON bool + +// indexCmd represents the index command +var indexCmd = &cobra.Command{ + Use: "index [index path] [data paths ...]", + Short: "adds the files to the index", + Long: `The index command adds the specified files to the index.`, + Annotations: map[string]string{ + canMutateBleveIndex: "true", + }, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) < 2 { + return fmt.Errorf("must specify at least one path") + } + for file := range handleArgs(args[1:]) { + var doc interface{} + // index the files + docID := file.filename + if !keepDir { + _, docID = filepath.Split(docID) + } + if !keepExt { + ext := filepath.Ext(docID) + docID = docID[0 : len(docID)-len(ext)] + } + doc = file.contents + var err error + if parseJSON { + err = json.Unmarshal(file.contents, &doc) + if err != nil { + return fmt.Errorf("error parsing JSON: %v", err) + } + } + fmt.Printf("Indexing: %s\n", docID) + err = idx.Index(docID, doc) + if err != nil { + return fmt.Errorf("error indexing: %v", err) + } + } + return nil + }, +} + +type file struct { + filename string + contents []byte +} + +func handleArgs(args []string) chan file { + rv := make(chan file) + go getAllFiles(args, rv) + return rv +} + +func getAllFiles(args []string, rv chan file) { + for _, arg := range args { + arg = filepath.Clean(arg) + err := filepath.Walk(arg, func(path string, finfo os.FileInfo, err error) error { + if err != nil { + log.Print(err) + return err + } + if finfo.IsDir() { + return nil + } + + bytes, err := os.ReadFile(path) + if err != nil { + log.Fatal(err) + } + rv <- file{ + filename: filepath.Base(path), + contents: bytes, + } + return nil + }) + if err != nil { + log.Fatal(err) + } + } + close(rv) +} + +func init() { + RootCmd.AddCommand(indexCmd) + + indexCmd.Flags().BoolVarP(&keepDir, "keepDir", "d", false, "Keep the directory in the document id.") + indexCmd.Flags().BoolVarP(&keepExt, "keepExt", "x", false, "Keep the extension in the document id.") + indexCmd.Flags().BoolVarP(&parseJSON, "json", "j", true, "Parse the contents as JSON.") +} diff --git a/cmd/bleve/cmd/mapping.go b/cmd/bleve/cmd/mapping.go new file mode 100644 index 0000000..1df471d --- /dev/null +++ b/cmd/bleve/cmd/mapping.go @@ -0,0 +1,42 @@ +// Copyright © 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "encoding/json" + "fmt" + "log" + + "github.com/spf13/cobra" +) + +// mappingCmd represents the mapping command +var mappingCmd = &cobra.Command{ + Use: "mapping [index path]", + Short: "prints the mapping used for this index", + Long: `The mapping command prints a JSON representation of the mapping used for this index.`, + Run: func(cmd *cobra.Command, args []string) { + mapping := idx.Mapping() + jsonBytes, err := json.MarshalIndent(mapping, "", " ") + if err != nil { + log.Fatal(err) + } + fmt.Printf("%s\n", jsonBytes) + }, +} + +func init() { + RootCmd.AddCommand(mappingCmd) +} diff --git a/cmd/bleve/cmd/query.go b/cmd/bleve/cmd/query.go new file mode 100644 index 0000000..e1b4bec --- /dev/null +++ b/cmd/bleve/cmd/query.go @@ -0,0 +1,101 @@ +// Copyright © 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "strings" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/search/query" + "github.com/spf13/cobra" +) + +var limit, skip, repeat int +var explain, highlight, fields bool +var qtype, qfield, sortby string + +// queryCmd represents the query command +var queryCmd = &cobra.Command{ + Use: "query [index path] [query]", + Short: "queries the index", + Long: `The query command will execute a query against the index.`, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) < 2 { + return fmt.Errorf("must specify query") + } + + query := buildQuery(args) + for i := 0; i < repeat; i++ { + req := bleve.NewSearchRequestOptions(query, limit, skip, explain) + if highlight { + req.Highlight = bleve.NewHighlightWithStyle("ansi") + } + if fields { + req.Fields = []string{"*"} + } + if sortby != "" { + if strings.Contains(sortby, ",") { + req.SortBy(strings.Split(sortby, ",")) + } else { + req.SortBy([]string{sortby}) + } + } + res, err := idx.Search(req) + if err != nil { + return fmt.Errorf("error running query: %v", err) + } + fmt.Println(res) + } + return nil + }, +} + +func buildQuery(args []string) query.Query { + var q query.Query + switch qtype { + case "prefix": + pquery := bleve.NewPrefixQuery(strings.Join(args[1:], " ")) + if qfield != "" { + pquery.SetField(qfield) + } + q = pquery + case "term": + pquery := bleve.NewTermQuery(strings.Join(args[1:], " ")) + if qfield != "" { + pquery.SetField(qfield) + } + q = pquery + default: + // build a search with the provided parameters + queryString := strings.Join(args[1:], " ") + q = bleve.NewQueryStringQuery(queryString) + } + return q +} + +func init() { + RootCmd.AddCommand(queryCmd) + + queryCmd.Flags().IntVarP(&repeat, "repeat", "r", 1, "Repeat the query this many times.") + queryCmd.Flags().IntVarP(&limit, "limit", "l", 10, "Limit number of results returned.") + queryCmd.Flags().IntVarP(&skip, "skip", "s", 0, "Skip the first N results.") + queryCmd.Flags().BoolVarP(&explain, "explain", "x", false, "Explain the result scoring.") + queryCmd.Flags().BoolVar(&highlight, "highlight", true, "Highlight matching text in results.") + queryCmd.Flags().BoolVar(&fields, "fields", false, "Load stored fields.") + queryCmd.Flags().StringVarP(&qtype, "type", "t", "query_string", "Type of query to run.") + queryCmd.Flags().StringVarP(&qfield, "field", "f", "", "Restrict query to field, not applicable to query_string queries.") + queryCmd.Flags().StringVarP(&sortby, "sort-by", "b", "", "Sort by field.") +} diff --git a/cmd/bleve/cmd/registry.go b/cmd/bleve/cmd/registry.go new file mode 100644 index 0000000..5c6566b --- /dev/null +++ b/cmd/bleve/cmd/registry.go @@ -0,0 +1,88 @@ +// Copyright © 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "sort" + + "github.com/blevesearch/bleve/v2/registry" + "github.com/spf13/cobra" +) + +// registryCmd represents the registry command +var registryCmd = &cobra.Command{ + Use: "registry", + Short: "registry lists the bleve components compiled into this executable", + Long: `The registry command will list all of the bleve components compiled into this executable.`, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + // override to do nothing + return nil + }, + PersistentPostRunE: func(cmd *cobra.Command, args []string) error { + // override to do nothing + return nil + }, + Run: func(cmd *cobra.Command, args []string) { + types, instances := registry.CharFilterTypesAndInstances() + printType("Char Filter", types, instances) + + types, instances = registry.TokenizerTypesAndInstances() + printType("Tokenizer", types, instances) + + types, instances = registry.TokenMapTypesAndInstances() + printType("Token Map", types, instances) + + types, instances = registry.TokenFilterTypesAndInstances() + printType("Token Filter", types, instances) + + types, instances = registry.AnalyzerTypesAndInstances() + printType("Analyzer", types, instances) + + types, instances = registry.DateTimeParserTypesAndInstances() + printType("Date Time Parser", types, instances) + + types, instances = registry.KVStoreTypesAndInstances() + printType("KV Store", types, instances) + + types, instances = registry.FragmentFormatterTypesAndInstances() + printType("Fragment Formatter", types, instances) + + types, instances = registry.FragmenterTypesAndInstances() + printType("Fragmenter", types, instances) + + types, instances = registry.HighlighterTypesAndInstances() + printType("Highlighter", types, instances) + }, +} + +func printType(label string, types, instances []string) { + sort.Strings(types) + sort.Strings(instances) + fmt.Printf("%s Types:\n", label) + for _, name := range types { + fmt.Printf("\t%s\n", name) + } + fmt.Println() + fmt.Printf("%s Instances:\n", label) + for _, name := range instances { + fmt.Printf("\t%s\n", name) + } + fmt.Println() +} + +func init() { + RootCmd.AddCommand(registryCmd) +} diff --git a/cmd/bleve/cmd/root.go b/cmd/bleve/cmd/root.go new file mode 100644 index 0000000..7dfa465 --- /dev/null +++ b/cmd/bleve/cmd/root.go @@ -0,0 +1,93 @@ +// Copyright © 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "os" + "strconv" + + "github.com/blevesearch/bleve/v2" + "github.com/spf13/cobra" +) + +var cfgFile string + +var idx bleve.Index + +// DefaultOpenReadOnly allows some distributions of this command to default +// to always opening the index read-only +var DefaultOpenReadOnly = false + +const canMutateBleveIndex = "canMutateBleveIndex" + +// CanMutateBleveIndex returns true if the command is capable +// of mutating the bleve index, or false if its operation is +// read-only +func CanMutateBleveIndex(c *cobra.Command) bool { + for k, v := range c.Annotations { + if k == canMutateBleveIndex { + if b, err := strconv.ParseBool(v); err == nil && b { + return true + } + } + } + return false +} + +// RootCmd represents the base command when called without any subcommands +var RootCmd = &cobra.Command{ + Use: "bleve", + Short: "command-line tool to interact with a bleve index", + Long: `Bleve is a command-line tool to interact with a bleve index.`, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if cmd.Use == "bash" || cmd.Use == "zsh" || cmd.Use == "fish" || cmd.Use == "powershell" { + // Not applicable to cobra's completion subcommands + return nil + } + if len(args) < 1 { + return fmt.Errorf("must specify path to index") + } + runtimeConfig := map[string]interface{}{ + "read_only": DefaultOpenReadOnly, + } + var err error + idx, err = bleve.OpenUsing(args[0], runtimeConfig) + if err != nil { + return fmt.Errorf("error opening bleve index: %v", err) + } + return nil + }, + PersistentPostRunE: func(cmd *cobra.Command, args []string) error { + if cmd.Use == "bash" || cmd.Use == "zsh" || cmd.Use == "fish" || cmd.Use == "powershell" { + // Not applicable to cobra's completion subcommands + return nil + } + err := idx.Close() + if err != nil { + return fmt.Errorf("error closing bleve index: %v", err) + } + return nil + }, +} + +// Execute adds all child commands to the root command sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + if err := RootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(-1) + } +} diff --git a/cmd/bleve/cmd/scorch.go b/cmd/bleve/cmd/scorch.go new file mode 100644 index 0000000..d899151 --- /dev/null +++ b/cmd/bleve/cmd/scorch.go @@ -0,0 +1,25 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "github.com/blevesearch/bleve/v2/cmd/bleve/cmd/scorch" +) + +// make scorch command-line tool a bleve sub-command + +func init() { + RootCmd.AddCommand(scorch.RootCmd) +} diff --git a/cmd/bleve/cmd/scorch/ascii.go b/cmd/bleve/cmd/scorch/ascii.go new file mode 100644 index 0000000..f6bdaec --- /dev/null +++ b/cmd/bleve/cmd/scorch/ascii.go @@ -0,0 +1,59 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + "strconv" + + "github.com/blevesearch/bleve/v2/index/scorch/mergeplan" + "github.com/spf13/cobra" +) + +// asciiCmd represents the ascii command +var asciiCmd = &cobra.Command{ + Use: "ascii", + Short: "ascii prints an ascii representation of the segments in a snapshot", + Long: `The ascii command prints an ascii representation of the segments in a given snapshot.`, + RunE: func(cmd *cobra.Command, args []string) error { + + if len(args) < 2 { + return fmt.Errorf("snapshot epoch required") + } else if len(args) < 3 { + snapshotEpoch, err := strconv.ParseUint(args[1], 10, 64) + if err != nil { + return err + } + snapshot, err := index.LoadSnapshot(snapshotEpoch) + if err != nil { + return err + } + segments := snapshot.Segments() + var mergePlanSegments []mergeplan.Segment + for _, v := range segments { + mergePlanSegments = append(mergePlanSegments, v) + } + + str := mergeplan.ToBarChart(args[1], 25, mergePlanSegments, nil) + fmt.Printf("%s\n", str) + } + + return nil + }, +} + +func init() { + RootCmd.AddCommand(asciiCmd) +} diff --git a/cmd/bleve/cmd/scorch/deleted.go b/cmd/bleve/cmd/scorch/deleted.go new file mode 100644 index 0000000..cb2a924 --- /dev/null +++ b/cmd/bleve/cmd/scorch/deleted.go @@ -0,0 +1,55 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + "strconv" + + "github.com/spf13/cobra" +) + +// deletedCmd represents the deleted command +var deletedCmd = &cobra.Command{ + Use: "deleted", + Short: "deleted prints the deleted bitmap for segments in the index snapshot", + Long: `The delete command prints the deleted bitmap for segments in the index snapshot.`, + RunE: func(cmd *cobra.Command, args []string) error { + + if len(args) < 2 { + return fmt.Errorf("snapshot epoch required") + } else if len(args) < 3 { + snapshotEpoch, err := strconv.ParseUint(args[1], 10, 64) + if err != nil { + return err + } + snapshot, err := index.LoadSnapshot(snapshotEpoch) + if err != nil { + return err + } + segments := snapshot.Segments() + for i, segmentSnap := range segments { + deleted := segmentSnap.Deleted() + fmt.Printf("%d %v\n", i, deleted) + } + } + + return nil + }, +} + +func init() { + RootCmd.AddCommand(deletedCmd) +} diff --git a/cmd/bleve/cmd/scorch/info.go b/cmd/bleve/cmd/scorch/info.go new file mode 100644 index 0000000..31e6481 --- /dev/null +++ b/cmd/bleve/cmd/scorch/info.go @@ -0,0 +1,59 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// dictCmd represents the dict command +var infoCmd = &cobra.Command{ + Use: "info", + Short: "info prints basic info about the index", + Long: `The info command prints basic info about the index.`, + RunE: func(cmd *cobra.Command, args []string) error { + + reader, err := index.Reader() + if err != nil { + return err + } + + count, err := reader.DocCount() + if err != nil { + return err + } + + fmt.Printf("doc count: %d\n", count) + + // var numSnapshots int + // var rootSnapshot uint64 + // index.VisitBoltSnapshots(func(snapshotEpoch uint64) error { + // if rootSnapshot == 0 { + // rootSnapshot = snapshotEpoch + // } + // numSnapshots++ + // return nil + // }) + // fmt.Printf("has %d snapshot(s), root: %d\n", numSnapshots, rootSnapshot) + + return nil + }, +} + +func init() { + RootCmd.AddCommand(infoCmd) +} diff --git a/cmd/bleve/cmd/scorch/internal.go b/cmd/bleve/cmd/scorch/internal.go new file mode 100644 index 0000000..dc94979 --- /dev/null +++ b/cmd/bleve/cmd/scorch/internal.go @@ -0,0 +1,61 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + "strconv" + + "github.com/spf13/cobra" +) + +var ascii bool + +// internalCmd represents the internal command +var internalCmd = &cobra.Command{ + Use: "internal", + Short: "internal prints the internal k/v pairs in a snapshot", + Long: `The internal command prints the internal k/v pairs in a snapshot.`, + RunE: func(cmd *cobra.Command, args []string) error { + + if len(args) < 2 { + return fmt.Errorf("snapshot epoch required") + } else if len(args) < 3 { + snapshotEpoch, err := strconv.ParseUint(args[1], 10, 64) + if err != nil { + return err + } + snapshot, err := index.LoadSnapshot(snapshotEpoch) + if err != nil { + return err + } + internal := snapshot.Internal() + for k, v := range internal { + if ascii { + fmt.Printf("%s %s\n", k, string(v)) + } else { + fmt.Printf("%x %x\n", k, v) + } + } + } + + return nil + }, +} + +func init() { + RootCmd.AddCommand(internalCmd) + internalCmd.Flags().BoolVarP(&ascii, "ascii", "a", false, "print key/value in ascii") +} diff --git a/cmd/bleve/cmd/scorch/root.go b/cmd/bleve/cmd/scorch/root.go new file mode 100644 index 0000000..5e2a441 --- /dev/null +++ b/cmd/bleve/cmd/scorch/root.go @@ -0,0 +1,70 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + "os" + + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/spf13/cobra" +) + +var index *scorch.Scorch + +// RootCmd represents the base command when called without any subcommands +var RootCmd = &cobra.Command{ + Use: "scorch", + Short: "command-line tool to interact with a scorch index", + Long: `Scorch is a command-line tool to interact with a scorch index.`, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + + if len(args) < 1 { + return fmt.Errorf("must specify path to scorch index") + } + + readOnly := true + config := map[string]interface{}{ + "read_only": readOnly, + "path": args[0], + } + + idx, err := scorch.NewScorch(scorch.Name, config, nil) + if err != nil { + return err + } + + err = idx.Open() + if err != nil { + return fmt.Errorf("error opening: %v", err) + } + + index = idx.(*scorch.Scorch) + + return nil + }, + PersistentPostRunE: func(cmd *cobra.Command, args []string) error { + return nil + }, +} + +// Execute adds all child commands to the root command sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + if err := RootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(-1) + } +} diff --git a/cmd/bleve/cmd/scorch/snapshot.go b/cmd/bleve/cmd/scorch/snapshot.go new file mode 100644 index 0000000..cf85398 --- /dev/null +++ b/cmd/bleve/cmd/scorch/snapshot.go @@ -0,0 +1,64 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + "strconv" + + seg "github.com/blevesearch/scorch_segment_api/v2" + "github.com/spf13/cobra" +) + +// snapshotCmd represents the snapshot command +var snapshotCmd = &cobra.Command{ + Use: "snapshot", + Short: "info prints details about the snapshots in the index", + Long: `The snapshot command prints details about the snapshots in the index.`, + RunE: func(cmd *cobra.Command, args []string) error { + + if len(args) < 2 { + snapshotEpochs, err := index.RootBoltSnapshotEpochs() + if err != nil { + return err + } + for _, snapshotEpoch := range snapshotEpochs { + fmt.Printf("snapshot epoch: %d\n", snapshotEpoch) + } + } else if len(args) < 3 { + snapshotEpoch, err := strconv.ParseUint(args[1], 10, 64) + if err != nil { + return err + } + snapshot, err := index.LoadSnapshot(snapshotEpoch) + if err != nil { + return err + } + segments := snapshot.Segments() + for i, segmentSnap := range segments { + segment := segmentSnap.Segment() + if segment, ok := segment.(seg.PersistedSegment); ok { + fmt.Printf("%d %s\n", i, segment.Path()) + } + } + } + + return nil + }, +} + +func init() { + RootCmd.AddCommand(snapshotCmd) +} diff --git a/cmd/bleve/gendocs.go b/cmd/bleve/gendocs.go new file mode 100644 index 0000000..3ff9b8d --- /dev/null +++ b/cmd/bleve/gendocs.go @@ -0,0 +1,45 @@ +// Copyright © 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build ignore +// +build ignore + +package main + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/cmd/bleve/cmd" + + "github.com/spf13/cobra/doc" +) + +// you can generate markdown docs by running +// +// $ go run gendocs.go +// +// this also requires doc sub-package of cobra +// which is not kept in this repo +// you can acquire it by running +// +// $ gvt restore + +func main() { + cmd.RootCmd.DisableAutoGenTag = true + identity := func(s string) string { + return fmt.Sprintf(`{{< relref "docs/%s" >}}`, s) + } + emptyStr := func(s string) string { return "" } + doc.GenMarkdownTreeCustom(cmd.RootCmd, "./", emptyStr, identity) +} diff --git a/cmd/bleve/main.go b/cmd/bleve/main.go new file mode 100644 index 0000000..64e6279 --- /dev/null +++ b/cmd/bleve/main.go @@ -0,0 +1,26 @@ +// Copyright © 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/blevesearch/bleve/v2/cmd/bleve/cmd" + + // to support standard set of build tags + _ "github.com/blevesearch/bleve/v2/config" +) + +func main() { + cmd.Execute() +} diff --git a/config.go b/config.go new file mode 100644 index 0000000..d1490bd --- /dev/null +++ b/config.go @@ -0,0 +1,95 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "expvar" + "io" + "log" + "time" + + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search/highlight/highlighter/html" + index "github.com/blevesearch/bleve_index_api" +) + +var bleveExpVar = expvar.NewMap("bleve") + +type configuration struct { + Cache *registry.Cache + DefaultHighlighter string + DefaultKVStore string + DefaultMemKVStore string + DefaultIndexType string + SlowSearchLogThreshold time.Duration + analysisQueue *index.AnalysisQueue +} + +func (c *configuration) SetAnalysisQueueSize(n int) { + if c.analysisQueue != nil { + c.analysisQueue.Close() + } + c.analysisQueue = index.NewAnalysisQueue(n) +} + +func (c *configuration) Shutdown() { + c.SetAnalysisQueueSize(0) +} + +func newConfiguration() *configuration { + return &configuration{ + Cache: registry.NewCache(), + analysisQueue: index.NewAnalysisQueue(4), + } +} + +// Config contains library level configuration +var Config *configuration + +func init() { + bootStart := time.Now() + + // build the default configuration + Config = newConfiguration() + + // set the default highlighter + Config.DefaultHighlighter = html.Name + + // default kv store + Config.DefaultKVStore = "" + + // default mem only kv store + Config.DefaultMemKVStore = gtreap.Name + + // default index + Config.DefaultIndexType = scorch.Name + + bootDuration := time.Since(bootStart) + bleveExpVar.Add("bootDuration", int64(bootDuration)) + indexStats = NewIndexStats() + bleveExpVar.Set("indexes", indexStats) + + initDisk() +} + +var logger = log.New(io.Discard, "bleve", log.LstdFlags) + +// SetLog sets the logger used for logging +// by default log messages are sent to io.Discard +func SetLog(l *log.Logger) { + logger = l +} diff --git a/config/README.md b/config/README.md new file mode 100644 index 0000000..5a8e497 --- /dev/null +++ b/config/README.md @@ -0,0 +1,11 @@ +# Bleve Config + +**NOTE** you probably do not need this package. It is only intended for general purpose applications that want to include large parts of Bleve regardless of whether or not the code is directly using it. + +## General Purpose Applications + +A general purpose application, that must allow users to express the need for Bleve components at runtime can accomplish this by: + +``` +import _ "github.com/blevesearch/bleve/config" +``` \ No newline at end of file diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..492b86f --- /dev/null +++ b/config/config.go @@ -0,0 +1,124 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + // token maps + _ "github.com/blevesearch/bleve/v2/analysis/tokenmap" + + // fragment formatters + _ "github.com/blevesearch/bleve/v2/search/highlight/format/ansi" + _ "github.com/blevesearch/bleve/v2/search/highlight/format/html" + + // fragmenters + _ "github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple" + + // highlighters + _ "github.com/blevesearch/bleve/v2/search/highlight/highlighter/ansi" + _ "github.com/blevesearch/bleve/v2/search/highlight/highlighter/html" + _ "github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple" + + // char filters + _ "github.com/blevesearch/bleve/v2/analysis/char/asciifolding" + _ "github.com/blevesearch/bleve/v2/analysis/char/html" + _ "github.com/blevesearch/bleve/v2/analysis/char/regexp" + _ "github.com/blevesearch/bleve/v2/analysis/char/zerowidthnonjoiner" + + // analyzers + _ "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" + _ "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + _ "github.com/blevesearch/bleve/v2/analysis/analyzer/simple" + _ "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" + _ "github.com/blevesearch/bleve/v2/analysis/analyzer/web" + + // token filters + _ "github.com/blevesearch/bleve/v2/analysis/token/apostrophe" + _ "github.com/blevesearch/bleve/v2/analysis/token/camelcase" + _ "github.com/blevesearch/bleve/v2/analysis/token/compound" + _ "github.com/blevesearch/bleve/v2/analysis/token/edgengram" + _ "github.com/blevesearch/bleve/v2/analysis/token/elision" + _ "github.com/blevesearch/bleve/v2/analysis/token/keyword" + _ "github.com/blevesearch/bleve/v2/analysis/token/length" + _ "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + _ "github.com/blevesearch/bleve/v2/analysis/token/ngram" + _ "github.com/blevesearch/bleve/v2/analysis/token/reverse" + _ "github.com/blevesearch/bleve/v2/analysis/token/shingle" + _ "github.com/blevesearch/bleve/v2/analysis/token/stop" + _ "github.com/blevesearch/bleve/v2/analysis/token/truncate" + _ "github.com/blevesearch/bleve/v2/analysis/token/unicodenorm" + _ "github.com/blevesearch/bleve/v2/analysis/token/unique" + + // tokenizers + _ "github.com/blevesearch/bleve/v2/analysis/tokenizer/exception" + _ "github.com/blevesearch/bleve/v2/analysis/tokenizer/regexp" + _ "github.com/blevesearch/bleve/v2/analysis/tokenizer/single" + _ "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" + _ "github.com/blevesearch/bleve/v2/analysis/tokenizer/web" + _ "github.com/blevesearch/bleve/v2/analysis/tokenizer/whitespace" + + // date time parsers + _ "github.com/blevesearch/bleve/v2/analysis/datetime/flexible" + _ "github.com/blevesearch/bleve/v2/analysis/datetime/iso" + _ "github.com/blevesearch/bleve/v2/analysis/datetime/optional" + _ "github.com/blevesearch/bleve/v2/analysis/datetime/percent" + _ "github.com/blevesearch/bleve/v2/analysis/datetime/sanitized" + _ "github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/microseconds" + _ "github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/milliseconds" + _ "github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/nanoseconds" + _ "github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/seconds" + + // languages + _ "github.com/blevesearch/bleve/v2/analysis/lang/ar" + _ "github.com/blevesearch/bleve/v2/analysis/lang/bg" + _ "github.com/blevesearch/bleve/v2/analysis/lang/ca" + _ "github.com/blevesearch/bleve/v2/analysis/lang/cjk" + _ "github.com/blevesearch/bleve/v2/analysis/lang/ckb" + _ "github.com/blevesearch/bleve/v2/analysis/lang/cs" + _ "github.com/blevesearch/bleve/v2/analysis/lang/da" + _ "github.com/blevesearch/bleve/v2/analysis/lang/de" + _ "github.com/blevesearch/bleve/v2/analysis/lang/el" + _ "github.com/blevesearch/bleve/v2/analysis/lang/en" + _ "github.com/blevesearch/bleve/v2/analysis/lang/es" + _ "github.com/blevesearch/bleve/v2/analysis/lang/eu" + _ "github.com/blevesearch/bleve/v2/analysis/lang/fa" + _ "github.com/blevesearch/bleve/v2/analysis/lang/fi" + _ "github.com/blevesearch/bleve/v2/analysis/lang/fr" + _ "github.com/blevesearch/bleve/v2/analysis/lang/ga" + _ "github.com/blevesearch/bleve/v2/analysis/lang/gl" + _ "github.com/blevesearch/bleve/v2/analysis/lang/hi" + _ "github.com/blevesearch/bleve/v2/analysis/lang/hr" + _ "github.com/blevesearch/bleve/v2/analysis/lang/hu" + _ "github.com/blevesearch/bleve/v2/analysis/lang/hy" + _ "github.com/blevesearch/bleve/v2/analysis/lang/id" + _ "github.com/blevesearch/bleve/v2/analysis/lang/in" + _ "github.com/blevesearch/bleve/v2/analysis/lang/it" + _ "github.com/blevesearch/bleve/v2/analysis/lang/nl" + _ "github.com/blevesearch/bleve/v2/analysis/lang/no" + _ "github.com/blevesearch/bleve/v2/analysis/lang/pl" + _ "github.com/blevesearch/bleve/v2/analysis/lang/pt" + _ "github.com/blevesearch/bleve/v2/analysis/lang/ro" + _ "github.com/blevesearch/bleve/v2/analysis/lang/ru" + _ "github.com/blevesearch/bleve/v2/analysis/lang/sv" + _ "github.com/blevesearch/bleve/v2/analysis/lang/tr" + + // kv stores + _ "github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb" + _ "github.com/blevesearch/bleve/v2/index/upsidedown/store/goleveldb" + _ "github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap" + _ "github.com/blevesearch/bleve/v2/index/upsidedown/store/moss" + + // index types + _ "github.com/blevesearch/bleve/v2/index/upsidedown" +) diff --git a/config_app.go b/config_app.go new file mode 100644 index 0000000..60b1db3 --- /dev/null +++ b/config_app.go @@ -0,0 +1,24 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build appengine || appenginevm +// +build appengine appenginevm + +package bleve + +// in the appengine environment we cannot support disk based indexes +// so we do no extra configuration in this method +func initDisk() { + +} diff --git a/config_disk.go b/config_disk.go new file mode 100644 index 0000000..a9ab1e4 --- /dev/null +++ b/config_disk.go @@ -0,0 +1,26 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !appengine && !appenginevm +// +build !appengine,!appenginevm + +package bleve + +import "github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb" + +// in normal environments we configure boltdb as the default storage +func initDisk() { + // default kv store + Config.DefaultKVStore = boltdb.Name +} diff --git a/data/test/sample-data.json b/data/test/sample-data.json new file mode 100644 index 0000000..dc0bc0c --- /dev/null +++ b/data/test/sample-data.json @@ -0,0 +1,10 @@ +[{"title":"Edinburgh/Leith","name":"Ocean Apartments","address":"2 Western Harbour Midway","directions":null,"phone":"+44 131 553 7394","tollfree":null,"email":null,"fax":null,"url":"http://www.oceanservicedapts.com","checkin":"15H00","checkout":"10H00","price":"from £70","geo":{"lat":55.9812,"lon":-3.2248,"accuracy":"RANGE_INTERPOLATED"},"type":"hotel","id":8576,"country":"United Kingdom","city":"Edinburgh","state":null,"reviews":[{"content":"In my personal opinion, this hotel is one of the many hidden gems of Istanbul. Located in an area not very easy to get if you are driving yourself. I would advise taking a taxi to get there. The service from the first \"hello\" until the last \"bye\" was impecable. The terrace restaurant wiew is second to none. The food was excellent. The staff was very nice. In short, next time I am going to Isnbul, I dont believe I will stay anywehere else. Highly recomended.","ratings":{"Service":5.0,"Cleanliness":5.0,"Overall":5.0,"Value":5.0,"Sleep Quality":5.0,"Rooms":5.0,"Location":5.0},"author":"Lindsey Wiegand III","date":"2013-01-01 16:30:13 +0300"},{"content":"When you first arrive at TomTom Suites you might wonder where you are coming to as the area looks a little run down! But it is in a great location in a street with no passing traffic. Its so quiet. Access to the main sites is easy. Downhill, the tram is within a 5 minute walk and uphill, Iskatel Cadesi ( a buzzing main street with loads of shops and restaurants) and a 2 minute walk round the corner leads to some narrow streets full of atmosphere and restaurants This boutique hotel itself is a sea of tranquility with beautifully appointed rooms, a great breakfast and very helpful staff. Our only regret was that the Terrace bar wasn't open but as the weather was unseasonably inclement it wasn't a problem. We had a great time and would definitely recommend the TomTom Suites","ratings":{"Service":5.0,"Cleanliness":5.0,"Overall":5.0,"Value":5.0,"Sleep Quality":5.0,"Rooms":5.0,"Location":4.0},"author":"Bulah Weissnat","date":"2013-06-01 17:06:53 +0300"},{"content":"Staying at Tomtom was the best possible choice we could have made, everything was simply perfect: the location, the facilities, the staff, the quality of the food, the elegance and modern style of the furniture, the superb view of the terrace where we had breakfast as well as a wonderful dinner and a lovely evening. Many details made us feel at home, such as the Ipod dock in the rooms and the Ipads we could use free of charge during breakfast. Most of all, the friendliness of the staff, all of them extremely helpful. I'd highlight Chiara, who gave us many tips, especially telling us to go to Bagdah st., on the Asian side, a place great place that no books mentioned; and Ali, a wonderful guy, much more than a concierge, and a perfect host who did everything he could to make our stay as good as it gets. the proximity to Istklal street, and the tram make the location convenient for both day and night. Considering everything, including the fair rates cherged, I doubt there's a better choice in town.","ratings":{"Service":5.0,"Cleanliness":5.0,"Overall":5.0,"Value":5.0,"Sleep Quality":5.0,"Rooms":5.0,"Location":5.0},"author":"Ottis Pacocha","date":"2012-08-07 08:16:49 +0300"},{"content":"A really wonderful hotel in a superb location on a traffic free road. It was so peaceful it was easy to forget that we were in the middle of one of the world's busiest cities. The staff were deligthful and couldn't do enough to help. The hotel is in a converted monastary which has been sympathetically refurbished, the artwork depicting istanbul brings a touch of colour - especially in the lift shaft. Our room was enormous and well appointed with a luxurious large bathroom - and yes, a jacuzzi bath. The rooftop restaurant and bar was a perfect way to relax at the end of the day and must have some of the best views in Istanbul over the Bosphorous and the Golden Horn. The food was excellent, modern turkish using french cooking techniques and local produce. Overall I can't wait to go back and I can't recommend the place highly enough.","ratings":{"Service":5.0,"Cleanliness":5.0,"Overall":5.0,"Value":5.0,"Sleep Quality":5.0,"Rooms":5.0,"Location":5.0},"author":"Dr. Amira Murazik","date":"2013-10-14 03:30:16 +0300"},{"content":"My wife, mother and I really enjoyed our recent stay at TomTom. We always prefer smaller boutique places that are well designed and in actual neighborhoods vs. larger impersonal luxury hotels. TomTom didn't dissapoint! As previously mentioned, beautifully designed and furnished rooms, the best toiletries, etc. The roof deck is very nice with a great view. The location is amazing, in-between everywhere that you want to go but far enough away from the touristy madness. I'd argue it's one of the best situated hotels in Istanbul. As mentioned, it's a very steep but quick walk up to the main street of Isklal. However, this can be challenging if you have any mobility issues. In general, the area is rather hilly but so is most of Istanbul. In the other direction, you can walk up into a neighborhood renowned for antiques and nice cafes or down to the tram that takes you to the main tourist attractions or the road to get to Ortokoy or Bebek. At no point does it feel like you are in the \"tourist district\" but you never feel unwelcome or out of place. As for noise, we found the building itself to be quiet and well insulated. You didn't hear noise from the rooms around you and minimal noise from the room above. Nowhere near as bad as what an earlier reviewer described. The only real noise to speak of does come from the kids in the morning and the afternoon who walk on the dead-end road to and from school. Other than this, and some kids playing soccer on occasion, the street outside of the hotel is pretty quiet Breakfast in the morning is great. Freshly baked pastries (done at the hotel) and fruit, cheeses and meats, served by extremely friendly staff. Fuat in particular is a delight to interact with and was extremely helpful. The staff overall is very friendly and accommodating. Everyone at the front desk regardless of time of day did whatever they could to make sure we were receiving the best service possible. There are however a few things that need improvement to make the hotel even better. -While breakfast is excellent, we were less than impressed with dinner. To the point that we ate there once and did not choose to repeat the experience. The service was good but the quality of the food and the cooking was pretty bad esp. considering the prices of the food vs. what can be had in the neighborhood for much less. We are admitted foodies, but we didn't hold the hotel restaurant up to lofty expectations. This needs to be addressed. -While the staff is very friendly and eager to help, the hotel would benefit from a dedicated concierge or someone who really knows about the various restaurants around the city. With a hotel of this caliber, we expected them to be more informed about different places and to let us know how traffic can affect getting there, etc. If you didn't ask them these kinds of questions, it didn't occur to them to tell you. -Lastly, with the windows closed in the room there is zero airflow without having the heat on and it's extremely dry. I understand them not wanting to run the AC unless the temperature warrants, but there should be a way to circulate air without opening the windows in the spring (see my earlier comment about noise from school kids in the morning). In conclusion, we wouldn't hesitate to recommend TomTom to friends and hope to stay there again the next time we are in Istanbul.","ratings":{"Service":4.0,"Cleanliness":5.0,"Overall":5.0,"Value":4.0,"Sleep Quality":4.0,"Rooms":5.0,"Location":5.0},"author":"Marcelle Haley","date":"2015-07-17 19:17:23 +0300"},{"content":"We had been at Istanbul for business purposes and used to stay at the well-known brands at Taxim. This time the trip was for leisure and we were looking for a charming, cozy, friendly, clean, staff friendly boutique hotel. So, THIS IS TomTom suites and we consider ourselves very lucky to stay there. Due to the Italian embassy its socak is not crowded and safe 24/7. Breakfast variety was very satisfactory, view from terrace wonderful, staff very friendly, very quiet and clean. If you are looking for a hotel with ID, we strongly recommend it. Be aware that Istiklal street is only 5 minute walking, but uphill.","ratings":{"Service":5.0,"Cleanliness":5.0,"Overall":5.0,"Value":5.0,"Location":4.0,"Rooms":5.0},"author":"Peggie Little","date":"2014-04-22 04:05:24 +0300"}],"public_likes":["Ms. Braulio Kuhic"],"vacancy":true,"description":"Modern, stylish contemporary serviced apartments 4 miles for Edinburgh's city Centre.","alias":"Serviced Apartments","pets_ok":false,"free_breakfast":true,"free_internet":false,"free_parking":true} +,{"title":"Edinburgh/Old Town","name":"Euro Hostel Edinburgh","address":null,"directions":null,"phone":"+44 8454 900 461","tollfree":null,"email":null,"fax":null,"url":"http://www.euro-hostels.co.uk/Edinburgh_hostel/","checkin":null,"checkout":null,"price":null,"geo":{"lat":55.94825,"lon":-3.18805,"accuracy":"APPROXIMATE"},"type":"hotel","id":8661,"country":"United Kingdom","city":"Edinburgh","state":null,"reviews":[{"content":"A plain and simple hotel, located on a busy street with many company offices nearby I can't imagine a tourist staying here. It is far away from the city center, and anything that a tourist might be interested in. In general, you will need a car, or a taxi to get to anything unless your business office is within walking distance. About €25 from the airport via taxi, this fairly modern looking hotel is very plain and simple in desgin, layout and comforts. Functional, and at a rate of about €85 per night - it was almost a bargain. Surprisingly quite given the location on the street, my \"no smoking\" room was clearly smoked in. Upon complaining - it was explained to me that the room was in fact no-smoking. No offer to move me, no thought that there might be an issue .... who knows, perhaps they had heard this before. Breakfast included, a simple breakfast that was OK in general, and nothing special. I would only recommend staying here if you are close enough to what you need to go to that you can walk.","ratings":{"Service":3.0,"Cleanliness":2.0,"Overall":2.0,"Value":3.0,"Sleep Quality":2.0,"Rooms":2.0,"Location":1.0},"author":"Rachel O'Hara","date":"2014-03-12 14:29:07 +0300"},{"content":"We (2 couples) recently stayed at the Hotel for 4 nights in their standard rooms -- 1 room off the garden and 1 in the main building.Both rooms were on the small side but the quality of the rooms more than compensated for their size.Breakfast was excellent.The hotels main asset in our eyes were all their staff who were very professional and looked after all our needs exceptionally well.The location of the hotel is excellent close to shops sights and restaurants. If you are travelling to Paris for a short trip i would recommend staying here.","ratings":{"Service":5.0,"Cleanliness":5.0,"Overall":4.0,"Value":4.0,"Sleep Quality":5.0,"Rooms":3.0,"Location":5.0},"author":"Viola Reinger","date":"2012-09-21 02:53:29 +0300"}],"public_likes":["Laila Jacobs","Dr. Andreane Berge","Ophelia Walter","Mac Hackett","Belle Bartell","Spencer Erdman","Elna Monahan","Shanelle Hayes","Ms. Wallace Larkin"],"vacancy":false,"description":"Kincaid's Court, Guthrie Street. In Cowgate, open every summer from June 8th until September 2nd. Budget accommodation in 43 apartments used as student residences during term time.","alias":null,"pets_ok":false,"free_breakfast":false,"free_internet":true,"free_parking":true} +,{"title":"Edinburgh/Old Town","name":"The Sheraton Grand Hotel","address":null,"directions":null,"phone":"+44 131 229 9131","tollfree":null,"email":null,"fax":null,"url":null,"checkin":null,"checkout":null,"price":null,"geo":{"lat":55.947,"lon":-3.2073,"accuracy":"APPROXIMATE"},"type":"hotel","id":8662,"country":"United Kingdom","city":"Edinburgh","state":null,"reviews":[{"content":"Really loved this hotel, it was beautifully decorated (very much interior designed) and was spotlessly clean. We had a booked a superior double and when we arrived we were told we'd been upgraded. The room was fairly large by Paris standards and had a day bed, which I assume could sleep a 3rd person/child. The bathroom was really modern and had a bath and large separate shower with large overhead shower head, plus hand held shower. All the decor was tasteful and we were at the back of the hotel overlooking the small courtyard garden, so very quiet, although the hotel is on a quiet street anyway. Short walk to the Jardin du Luxembourg and less that 5 mins to a metro, which was on a direct line to the Gare du Nord, so perfect for us as we took the Eurostar from London. Hotel booked us a table at a great restaurant, superb food, which they recommended. Breakfast was served to order and you got croissants, pain au chocolat, bread and could choose omelettes etc. The fruit salad was freshly made, the yoghurts were the posh ones in glass jars. Would really recommend the hotel, but not for small children - there are a lot of carefully placed vases and objets d'art that little fingers will want to touch...","ratings":{"Cleanliness":5.0,"Sleep Quality":5.0,"Overall":5.0,"Value":4.0,"Service":5.0},"author":"Brittany Ledner Jr.","date":"2012-06-21 07:48:16 +0300"},{"content":"I spent a wonderful week at the Villa Madame, finding the staff very helpful and gracious. My 5th floor room was very clean and light, quiet, and comfortable. The hotel is extremely well located only a few short blocks to the metro, Jardins du Luxemburg, and shopping. The hotel offered free wifi, and much more surprisingly, free international telephone service from my room via voip. Breakfasts were excellent, and the garden area was quiet and comfortable. Just around the corner is Maison du Jardin, a very excellent small restaurant with prix fixe 31 euro dinners. Staff made other great recommendations as well. I would unhesitatingly recommend and will return.","ratings":{"Service":5.0,"Cleanliness":5.0,"Overall":5.0,"Value":4.0,"Sleep Quality":5.0,"Rooms":5.0,"Location":5.0},"author":"Lauren Ortiz","date":"2014-10-01 14:04:03 +0300"},{"content":"My friend and I were backbacking through London and Paris and decided to splurge a little on this hotel. It turned out amazing. You really don't want to risk a hostel in Paris, they are serious dumps. I did a hostel in Paris in 2007. I picked this hotel due to it's proximity to nightlife in the Latin Quarter and it looked nice. The bathroom had a walk in shower with glass door. Flatscreen TV had an aux speaker in the bathroom to hear the tv while doing your thing. Beds were very comfortable and the room we got was big and facing the street. The street was loud during mid afternoon due to the Catholic school playground down the block. We booked a room with two singles and when we went to check in we were told they had inadvertantly given our room away. I started to freak because I was sure we would get a fast one pulled as usually happens in western europe. We were happy that after my face of death look we were upgraded to a larger room with two double beds. Unfortunately for the poor lady who had booked the room they bumped her and I don't know where she was then moved. Overall the staff was slow on check'in. Breakfast was carbs and coffee. They have a person dedicated only to breakfast and coffee however it seems odd because this is a very small hotel. We were the youngest guests in the hotel by about 50 years. Everyone else looked like they were late fifties to mid seventee's. We are single thirtysomethings. I would recommend this hotel to anyone who wants a peacefull nights sleep in Paris.","ratings":{"Service":4.0,"Cleanliness":5.0,"Overall":4.0,"Value":2.0,"Location":5.0,"Rooms":4.0},"author":"Turner Ferry","date":"2013-11-29 14:38:35 +0300"},{"content":"The Villa Madame is a lovely, comfortable, well-located boutique hotel with excellent service. My wife and I enjoyed our stay there immensely. This hotel is chic and luxuriously comfortable especially for the price. Admittedly, the room we had (Classic Double Room) was the smallest hotel room my wife and I had ever seen at about 3m x 3.5m (10' x 12'). The large outdoor terrace, fantastic bathroom (with Hermès products and great water pressure), deliciously comfortable bed, iPod docking station, super-fast free wi-fi and excellent service more than made up for the tiny room. Alex at the front desk was a delight as she answered our every question and was always happy to chat and share information. The location is excellent being only a few minutes walk from either the Rennes or St-Sulpice Metro stops, 100m from the beautiful Jardin du Luxembourg and within easy walking distance of the myriad of shops and restaurants on both the Rue de Rennes and Boulevard Saint-Germain. The hotel serves a decent continental breakfast which seems expensive at 18€ but we found a package that was below the normal rate and included breakfast. The breakfast room, as other reviewers have noted, is small and has only four tables for two. All of the tables were occupied each time we went for breakfast but the hotel happily served us as we sat in the little lounge area so it wasn't ever a problem. They don't serve lunch or dinner but there are two brasseries within 50m of the front door and many more restaurants and coffee shops just a few blocks away. There were a few other minor concerns that I had regarding this hotel: - The minibar is stocked only with two bottles of water. It would be nice if there was a limited selection of other beverages and also a kettle for tea/coffee service. - The satellite television has almost 900 channels (yes, I went through them all late one night) but half of them are Arabic and almost all of the rest are in French save for one German language station, a couple of Italian stations and only Bloomberg for the English speakers. Although one doesn't go to Paris to watch TV in the room, it's nice to relax and watch the news or a movie after a long day on the town but unless the above suits you, you won't have much cause to even turn on the set. - The street is very small and it is very difficult to find parking. This is likely not a problem for most visitors but is something to keep in mind especially if you rent a car. Overall, my wife and I loved this hotel. The few cons are heavily outweighed by the comforts we enjoyed. We are very seasoned travelers and despite the tiny room, this hotel experience was one of our best ever. We will definitely stay at the Villa Madame again.","ratings":{"Service":5.0,"Cleanliness":5.0,"Overall":5.0,"Value":5.0,"Location":5.0,"Rooms":5.0},"author":"Blaze Williamson","date":"2014-04-26 13:59:54 +0300"}],"public_likes":["Madelynn Littel","Marielle Daugherty","Micah Stiedemann","Sandra Howe","Angela Oga"],"vacancy":true,"description":"21 Festival Square. Against the backdrop of majestic Edinburgh Castle, the Sheraton Grand Hotel and Spa combines city centre convenience with warm Scottish hospitality.","alias":null,"pets_ok":false,"free_breakfast":true,"free_internet":false,"free_parking":false} +,{"title":"Edinburgh/Old Town","name":"Radisson Blu Hotel","address":"80 High St","directions":null,"phone":"+44 131 557 9797","tollfree":null,"email":null,"fax":null,"url":"http://www.radissonblu.co.uk/hotel-edinburgh","checkin":null,"checkout":null,"price":null,"geo":{"lat":55.95014,"lon":-3.18667,"accuracy":"ROOFTOP"},"type":"hotel","id":8663,"country":"United Kingdom","city":"Edinburgh","state":null,"reviews":[],"public_likes":[],"vacancy":false,"description":"The Royal Mile. Less than a five-minute walk from major shopping and business districts, and the Edinburgh International Conference Centre is only a short taxi ride away.","alias":null,"pets_ok":false,"free_breakfast":true,"free_internet":true,"free_parking":false} +,{"title":"Edinburgh/Old Town","name":"Hotel Missoni","address":null,"directions":null,"phone":"+44 131 220 6666","tollfree":null,"email":null,"fax":null,"url":"http://www.hotelmissoni.com/hotelmissoni-edinburgh","checkin":null,"checkout":null,"price":null,"geo":{"lat":55.9491,"lon":-3.19275,"accuracy":"APPROXIMATE"},"type":"hotel","id":8664,"country":"United Kingdom","city":"Edinburgh","state":null,"reviews":[{"content":"We stayed for 3 nights in March, 2 consecutive and then one at the end of the trip. The area near the Termini station is not the prettiest, but it is very convenient if you are using public transportation. My suggestion when exiting Termini station is to go RIGHT and walk down about 4 blocks and then right again and over 2 blocks. We were travelling with quite a bit of luggage and we ended up going the wrong way too many times. We had no problems in the area and loved the convenience of the location. We walked to Termini to either pick up the Metro train or a bus to most all of the sites. The room was spotless and the Breakfast was delicious. Assunta - the owner, could not have been more helpful, although her sense of direction is not like ours in America. Just a short walk maybe a lot longer than we Americans are used to. Hoping on the #70 bus gets you to almost any tourist site (or close to) and most buses all head back to Termini.","ratings":{"Service":5.0,"Cleanliness":5.0,"Overall":4.0,"Value":5.0,"Sleep Quality":5.0,"Rooms":3.0,"Location":3.0},"author":"Hermina Schinner","date":"2015-03-06 22:56:15 +0300"},{"content":"This motel may not be the Ritz Carlton but if your looking for value, a great location and safe neighborhood then you've chosen the right place! It is right across the street from the CBS studios, the line for the price is right is just around the corner. My son and I stayed there for a week and received great service from the maids right on to the front desk. The front desk was more helpful than any concierge I have ever seen and they are just a wonderful hard working family. If I ever go back to LA I will definitely stay at the Beverly Inn even if it's only for the friendly service.","ratings":{"Service":5.0,"Cleanliness":4.0,"Overall":4.0,"Value":5.0,"Sleep Quality":5.0,"Rooms":4.0,"Location":5.0},"author":"Jose Swaniawski Sr.","date":"2014-02-12 20:15:16 +0300"},{"content":"This place is car motel that has seen much better days; the beds are old and offer no support, the televisions, carpets, and furnishings are likewise well-used, and the overall effect can be somewhat depressing. However, it does offer limited parking and convenient location at a very inexpensive rate and the bathrooms were clean; it attracts tourists who are more interested in the surrounding neighborhoods (which are quite nice with wonderful restaurants) and less interested in where they stay for the night. There was a cafe down the street when I was there in 2002, it's along major bus lines, with a terrific market close by across the street. It is also right across from the CBS studios, and about four blocks walking from an old-style diner and bakery. The comments on other sites focus on the ethnic background of the clerks and manager, but I found them pleasant and accommodating. That is why I actually felt safe there as a single female traveler (although I suspect others would not feel this way). This is just slightly above average for what you would expect for the price, but you do get what you pay for.","ratings":{"Cleanliness":1.0,"Overall":2.0,"Value":4.0,"Service":3.0,"Rooms":2.0},"author":"Mr. Ellis Heller","date":"2012-03-02 00:20:56 +0300"},{"content":"Dirty, cock roach infested, unsafe, and very noisy. This motel has not been updated in 25 years or more. It is very noisy (even with ear plugs) because you are a stone throw away from the Dolphin Expressway. The carpet and ceramic floors are filthy along with the furniture and bedding. Someone tried entering my room in the middle of the night, thank goodness for deadbolt and door chain. Photos online are very deceiving. RECOMMEND - DO NOT STAY AT THIS MOTEL. This motel should not be part of the Choice Hotel chain. Should be called \"Last Choice\".","ratings":{"Service":2.0,"Cleanliness":1.0,"Overall":1.0,"Value":1.0,"Sleep Quality":1.0,"Rooms":1.0,"Location":1.0},"author":"Lottie Gerhold IV","date":"2014-06-12 22:56:30 +0300"},{"content":"Creepy, dirty, dark, depressing. It looks like the stereotypical motel in the movies where a drug deal goes bad and people get murdered. Rooms smell of chemical perfumed disinfectant and it burns your nose and lungs... but you will be too afraid for your safety to open a window or door! We stayed at this hotel from 3pm to 10pm (never actually slept overnight, thank god!) because we had a late night flight and it was pouring rain in Miami. The hotel says it's \"newly renovated\" and had free WIFI, so we figured it would be a nice place to spend a few hours and relax. HA! NOOOO! We laid on top of the beds, just to watch tv and instantly became itchy. Nothing about this hotel was \"newly renovated.\" The WIFI was extremely slow. The main picture of this hotel is deceiving. It looks a LOT worse and run-down in real life! Don't bother with this place.","ratings":{"Service":2.0,"Cleanliness":1.0,"Overall":1.0,"Value":1.0,"Sleep Quality":1.0,"Rooms":1.0},"author":"Niko Keebler","date":"2014-12-12 12:32:09 +0300"},{"content":"We didn't stay at the hotel so I can't comment on the rooms. We did leave our car there while we went on a cruise ($5/night parking.) While we were gone someone siphoned ALL of the gas out of our vehicle. I have called the manager twice to alert her to the problem. I've left messages concerning \"a security issue at the hotel\" and no one has returned my call. I guess they don't care about security. Next time I will spend the $20/day to park at the port since that seems to be the only secure parking to be had.","ratings":{"Overall":1.0},"author":"Miss Alysha Goldner","date":"2012-01-17 00:40:45 +0300"},{"content":"I stayed at this motel for one night with my partner in August 2010. We had a flight early in the morning from Miami airport so we wanted a hotel close to the airport. I booked this through BOOKING.COM and payed £45. I have never stayed in Miami before so we did not know what areas were good and what was bad. When we checked in I wasa little concerned as there was a security hut and a guard at the door. We checked in and it was very run down and dirty. They asked for a credit card but I insisted on paying cash as there was no way I was going to hand over my card details. We parked our car and tried to find our room. The halls were all outdoors and very run down. There was a sign saying it had just had a refurb. I could not see where from the outside. It seemed that the area in Miami the motel was in was not a very good one, this concerned us a little. We finally found our room and went in. The room was very basic and smelt of damp. The dead lock on the door was broken and there was no safe in the room. We felt that uncomfortable that at night we pushed our suit cases up to the door. The shower had no presure and was only luke warm and the paint was peeling off from the bathroom. What really appauld us was the floor. I took my shoes and socks off and was walking around in bare foot. After two minutes my partner said \"LOOK AT YOUR FEET\". They were black. The floor was that dirty that in 2 mins my feet were black. I wet a towel and rubbed it along the floor. The towel changed to black and the carpet changed colour. The floor could do with a really good clean. The TV reception was very poor and fuzzy and we gave up in the end. Also the internet/Wifi had little/no signal. It was noisey outside and there were Police sirens sounding all night outside. We found it hard to sleep. We only stayed the night as we had to get up at 4 am to check in for our flight and we checked in at 8pm that evening. Otherwise we would have moved hotel. The only plus side was it was close to the airport. We did not stay for the breakfast but if itn was anything like the room we would have passed anyway. We did not use the pool as it was on the otherside of the motel right next to a main highway. It looked dirty and very uninviting. Please only stay at this motel if you have to or if it is free and you are feeling brave.","ratings":{"Service":1.0,"Cleanliness":1.0,"Overall":1.0,"Value":1.0,"Sleep Quality":1.0,"Rooms":2.0,"Location":3.0},"author":"Miss Weldon Flatley","date":"2015-05-21 14:38:02 +0300"}],"public_likes":["Ms. Jaleel Bartell","Rodger Jerde","Hanna Simonis"],"vacancy":false,"description":"1 George IV Bridge. Situated on the Royal Mile and designed by Rosita Missoni.","alias":null,"pets_ok":true,"free_breakfast":true,"free_internet":false,"free_parking":false} +,{"title":"Edinburgh/South","name":"Argyle Backpackers","address":"14 Argyle Pl","directions":"The number 41 bus (catch it outside Waverley railway station) goes right past the front door.","phone":"+44 131 667 9991","tollfree":null,"email":null,"fax":null,"url":"http://www.argyle-backpackers.co.uk/","checkin":null,"checkout":null,"price":"Dorm from £13","geo":{"lat":55.9385,"lon":-3.1912,"accuracy":"ROOFTOP"},"type":"hotel","id":8685,"country":"United Kingdom","city":"Edinburgh","state":null,"reviews":[{"content":"Made a one night reservation at this \"hotel\" without checking the reviews, big mistake. The hotel looks OK from the highway and looked conveniently located. Should have known when the very unfriendly girl at the front desk gave me the room key wrapped in a post-it note because they had run out of the little envelopes. The room smelled weird, when we opened the chest drawers we saw what looked like small roaches, we never took anything out of the suitcase. It was very cold that night in Miami and the heating unit blew cold air. The next morning we woke up to find out there was no hot water in the whole hotel, after many calls and trips to the front desk I was informed that there was someone coming to fix the boiler. To make a long story short, the hot water came back on at 12:00pm; checkout time is 11:00am. They wanted us to leave without taking a shower, the maid was also annoyed at us because we didn't leave the room and she had to finish to go home. Later that night while at a restaurant in South Beach, I noticed some itchy bumps on the left side of the back of my neck and scalp, as well as the knuckles of my fingers, they turned out to be BED BUG BITES!!!","ratings":{"Service":1.0,"Cleanliness":1.0,"Overall":1.0,"Value":1.0,"Sleep Quality":1.0,"Rooms":1.0,"Location":4.0},"author":"Consuelo Thiel","date":"2013-03-28 07:41:35 +0300"},{"content":"Restaurant service was o.k. Your continental breakfast is a joke. We were there 2 nights. We had 4 rooms which were reserved in August 2008. The last couple to get there Friday niight got a terrible room. Nothing in it but the bed. They wre given a different room the next morning but the damage was done. No we will not be back.","ratings":{"Cleanliness":1.0,"Overall":1.0,"Value":1.0,"Service":1.0,"Rooms":1.0},"author":"Ettie Bartell","date":"2012-09-04 06:14:54 +0300"},{"content":"Disappointing, after all these years... I have been staying at the Capri since 2001, when my family moved from SF to the North Bay. It used to be a great deal for us ex-pats and while not a luxury hotel, it was clean and comfortable (plus in a great location).My son and I stayed at the Capri this summer and we were terribly let down. They no longer offer specials and while there is renovation occurring, our room was dingy and the bed very uncomfortable. My 9 year old son said, Mom, let's not stay at the Capri anymore. Sadly, I had to agree.","ratings":{"Service":3.0,"Business service":-1.0,"Cleanliness":2.0,"Check in / front desk":3.0,"Overall":2.0,"Value":2.0,"Rooms":2.0,"Location":4.0},"author":"Harry O'Kon I","date":"2014-09-10 11:07:33 +0300"},{"content":"Best Deal in the Marina If you don't mind 1960's decor this place will fit the bill. It's very reasonable and very clean. Sometimes it books up with the Euro tours. Ask to stay on the 3rd floor, with the high ceilings and roof windows. I have actually heard the fog horns at night. There is also plenty of free parking right on-site.This hotel is right off Union street and within walking distance of the Marina Green. Tons of great restaurants and clubs. One of my favorites is the Brazen Head, which is right across the street. A small English Pub with great food and drinks. It's hard to spot but there's a real small sign out front. Also, El Canasta (sp?) has a great steak burrito.","ratings":{"Service":-1.0,"Business service":-1.0,"Cleanliness":-1.0,"Check in / front desk":-1.0,"Overall":4.0,"Value":-1.0,"Rooms":-1.0,"Location":-1.0},"author":"Sibyl Lind","date":"2014-09-20 18:19:21 +0300"},{"content":"Bed Bugs and Ants The ants didn't bother me. It was the bed bugs I detested.Before staying at the Buena Vista I didn't know what a bed bug looked. But the spots on my arms that looked like flea bites kept appearing after I got home. So I googled them.Yep, sure enough! Up popped a photo of the same type of bug that I had killed after I found it crawling on my husband's pillow while we were staying at the Buena Vista this January 2008!I ordered some all natural bed bug powder to dust all over my house. Even my kids are showing up with the spots.When I called the hotel after I got home to tell them about it they said they would block off that room. But when I called a day or two later there was someone in that room. It doesn't matter if they block off and treat one room. They've got to do the entire premises.","ratings":{"Service":5.0,"Business service":-1.0,"Cleanliness":1.0,"Check in / front desk":5.0,"Overall":2.0,"Value":5.0,"Rooms":1.0,"Location":5.0},"author":"Deshawn Rippin","date":"2014-10-17 15:46:51 +0300"},{"content":"Not a great place to stay This place is falling apart...was once a nice little place to stay but not now. Lobby was shabby and dirty, room was not better...there was mold in the tub, no movies here to buy on cable, iron in room was broke....etc. For the $160 they charged per night ( with a AAA card), I would not go back...there is a very nice small, totally remodeled motel two blocks down that we should have stayed at for the same price and most definitly will next time...It is called Hotel Del Sol...check it out, thehoteldelsol.com....much, much better for the $$$$.","ratings":{"Service":2.0,"Business service":1.0,"Cleanliness":1.0,"Check in / front desk":3.0,"Overall":2.0,"Value":1.0,"Rooms":2.0,"Location":4.0},"author":"Wayne Tremblay III","date":"2012-10-27 01:48:16 +0300"},{"content":"Great Budget Accommodation If you want a small budget sized no frills hotel that offers a resonable level of service then this is for you. The rooms were large and the housekeeping very good. The front desk service was always helpful and friendly with good advice. Traffic noise was not as bad as expected and nor was the beds. Only minus was the continental breakfast American style which differed somewhat from what we experienced in other countries. Too sweet for our taste.Overall a great experience and well located although being closer to eateries would have been appreciated. Can recommend the Liquor Store accross the road. Their staff were great!","ratings":{"Service":3.0,"Business service":-1.0,"Cleanliness":3.0,"Check in / front desk":-1.0,"Overall":3.0,"Value":3.0,"Rooms":4.0,"Location":-1.0},"author":"Margaretta Miller","date":"2012-04-19 20:46:49 +0300"}],"public_likes":["Narciso Wiegand","Graciela Bailey","Kavon Bruen","Aditya Feest","Caleb Medhurst","Ross Rippin","Germaine Kunde"],"vacancy":true,"description":"Two good self-catering kitchens, garden, conservatory/seating area, choice of different sized dorms, and private rooms.Definitely not a party hostel.","alias":null,"pets_ok":true,"free_breakfast":true,"free_internet":true,"free_parking":false} +,{"title":"Abbeville","name":"Chez Mel","alt":null,"address":"63-65 rue Saint-Vulfran","directions":null,"phone":"+33 3 22 19 48 64","tollfree":null,"email":null,"url":null,"hours":null,"image":null,"price":null,"content":"With an old style setting and musical accompaniment, this is a hearty and family-friendly crêpe restaurant. It is also a tea room in the afternoon.","geo":{"lat":50.104437,"lon":1.829432,"accuracy":"RANGE_INTERPOLATED"},"activity":"eat","type":"landmark","id":33,"country":"France","city":"Abbeville","state":"Picardie"} +,{"title":"Aberdour","name":"Aberdour Castle","alt":null,"address":null,"directions":null,"phone":null,"tollfree":null,"email":null,"url":"http://www.historic-scotland.gov.uk/propertyresults/propertyoverview.htm?PropID=PL_001","hours":null,"image":null,"price":null,"content":"Is a fascinating, 12th-century castle which was granted by Robert the Bruce to his friend and nephew, Thomas Randolph, Earl of Moray. It includes the beautiful and well-maintained castle gardens, as well as a spectacular beehive-shaped dovecot built at the end of the sixteenth century.","geo":{"lat":56.0552,"lon":-3.2985,"accuracy":"APPROXIMATE"},"activity":"see","type":"landmark","id":35,"country":"United Kingdom","city":"Aberdour","state":null} +,{"title":"Aberdour","name":"The Silver Sands Beach","alt":null,"address":null,"directions":null,"phone":null,"tollfree":null,"email":null,"url":null,"hours":null,"image":null,"price":null,"content":"is one of Scotland's seven Blue Flag awarded beaches, and is incredibly popular in summer time. For those after a bit of peace and quiet, the '''Black Sands Beach''' may be more to your tastes.","geo":{"lat":56.0544,"lon":-3.2863,"accuracy":"ROOFTOP"},"activity":"see","type":"landmark","id":36,"country":"United Kingdom","city":"Aberdour","state":null} +,{"title":"Aberdour","name":"Aberdour Railway Station","alt":null,"address":null,"directions":null,"phone":null,"tollfree":null,"email":null,"url":null,"hours":null,"image":null,"price":null,"content":"is a beautifully kept and cared for example of a traditional station, and regularly wins the "Best Station and Gardens in Great Britain" award.","geo":{"lat":56.05471,"lon":-3.30089,"accuracy":"RANGE_INTERPOLATED"},"activity":"see","type":"landmark","id":37,"country":"United Kingdom","city":"Aberdour","state":null}] \ No newline at end of file diff --git a/doc.go b/doc.go new file mode 100644 index 0000000..0c55551 --- /dev/null +++ b/doc.go @@ -0,0 +1,37 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package bleve is a library for indexing and searching text. + +Example Opening New Index, Indexing Data + + message := struct{ + Id: "example" + From: "xyz@couchbase.com", + Body: "bleve indexing is easy", + } + + mapping := bleve.NewIndexMapping() + index, _ := bleve.New("example.bleve", mapping) + index.Index(message.Id, message) + +Example Opening Existing Index, Searching Data + + index, _ := bleve.Open("example.bleve") + query := bleve.NewQueryStringQuery("bleve") + searchRequest := bleve.NewSearchRequest(query) + searchResult, _ := index.Search(searchRequest) +*/ +package bleve diff --git a/docs/bleve.png b/docs/bleve.png new file mode 100644 index 0000000..c2227ae Binary files /dev/null and b/docs/bleve.png differ diff --git a/docs/create_and_search_your_first_index.md b/docs/create_and_search_your_first_index.md new file mode 100644 index 0000000..ae24f5b --- /dev/null +++ b/docs/create_and_search_your_first_index.md @@ -0,0 +1,226 @@ +# Creating a Bleve Index + +A simple how-to example using Bleve in Go to create an index, add documents, and run search queries with results. + +```go +package main + +import ( + "fmt" + "log" + + bleve "github.com/blevesearch/bleve/v2" +) + +type Document struct { + ID string `json:"id"` + Title string `json:"title"` + Content string `json:"content"` +} + +func main() { + indexPath := "example.bleve" + // Create a new index + mapping := bleve.NewIndexMapping() + index, err := bleve.New(indexPath, mapping) + if err != nil { + log.Fatal(err) + } + defer index.Close() + + // Add documents + documents := []Document{ + { + ID: "doc", + Title: "Bleve documentation", + Content: "Bleve provides full-text search capabilities.", + }, + { + ID: "doc1", + Title: "Elasticsearch documentation", + Content: "Elasticsearch provides full-text search capabilities as well.", + }, + } + + // Iterate and index the documents + batch := index.NewBatch() + for _, doc := range documents { + batch.Index(doc.ID, doc) + } + if err := index.Batch(batch); err != nil { + log.Fatal(err) + } + + // Search the created index + query := bleve.NewQueryStringQuery("bleve") + searchRequest := bleve.NewSearchRequest(query) + searchRequest.Explain = true + searchRequest.Fields = []string{"title", "content"} + searchResult, err := index.Search(searchRequest) + if err != nil { + log.Fatal(err) + } + fmt.Println(searchResult) +} + +``` + +## Output + +```bash +$ go run main.go + +1 matches, showing 1 through 1, took 262.333µs + 1. doc (0.471405) + title + Bleve documentation + content + Bleve provides full-text search capabilities. +``` + +## Step-by-Step Breakdown + +### 1. Index Creation + +```go +// Create a new index mapping +mapping := bleve.NewIndexMapping() +// Create a new index (this creates a directory on disk) +index, err := bleve.New(indexPath, mapping) +``` + +**What happens:** + +- Creates an index mapping with default settings +- Creates a new index directory `example.bleve/` +- Sets up the underlying storage (Scorch engine by default) + +### 2. Document Indexing + +```go +// Index a document with a unique ID +err := index.Index("doc", map[string]interface{}{ + "title": "My Document", + "content": "This is the document content", + "author": "John Doe", +}) +``` + +**What happens:** + +- Document gets a unique ID (`doc`) +- Fields are automatically mapped based on their Go types +- Text fields are analyzed (tokenized, lowercased, etc.) based on the chosen mapping (here, the default one) +- Document is stored in the search index + +### 3. Searching + +```go +// Create a query +query := bleve.NewQueryStringQuery("search terms") +request := bleve.NewSearchRequest(query) + +// Execute search +results, err := index.Search(request) +``` + +**What happens:** + +- Query string is parsed and analyzed +- Index is searched for matching documents +- Results are scored and ranked by relevance by the algorithm used +- Document metadata and highlights are returned + +## Working with Existing Indexes + +To open an existing index instead of creating a new one: + +```go +// Open existing index +index, err := bleve.Open("example.bleve") +if err != nil { + log.Fatal(err) +} +defer index.Close() +``` + +## Different Query Types + +### 1. Query String Query (Simple) + +```go +query := bleve.NewQueryStringQuery("golang programming") +``` + +### 2. Match Query (Exact Field) + +```go +query := bleve.NewMatchQuery("bleve") +query.SetField("title") // Search only in title field +``` + +### 3. Boolean Query (Complex) + +```go +mustQuery := bleve.NewMatchQuery("golang") +shouldQuery := bleve.NewMatchQuery("programming") + +boolQuery := bleve.NewBooleanQuery() +boolQuery.AddMust(mustQuery) +boolQuery.AddShould(shouldQuery) +``` + +### 4. Range Query (Numeric/Date) + +```go +minPrice := 20.50 +maxPrice := 40.75 +query := bleve.NewNumericRangeQuery(&minPrice, &maxPrice) +query.SetField("price") +``` + +## Advanced Index Configuration + +### Custom Field Mapping + +```go +// We can create customized mappings as well with configurable analyzers +mapping := bleve.NewIndexMapping() + +// Text field with custom analyzer +titleMapping := bleve.NewTextFieldMapping() +titleMapping.Analyzer = "en" // English analyzer + +// Numeric field +priceMapping := bleve.NewNumericFieldMapping() + +// Date field +dateMapping := bleve.NewDateTimeFieldMapping() + +// Document mapping +docMapping := bleve.NewDocumentMapping() +docMapping.AddFieldMappingsAt("title", titleMapping) +docMapping.AddFieldMappingsAt("price", priceMapping) +docMapping.AddFieldMappingsAt("created_at", dateMapping) + +// Add to index mapping +mapping.AddDocumentMapping("product", docMapping) + +// Create index with custom mapping +index, err := bleve.New("products.bleve", mapping) +``` + +### Batch Operations + +For better performance when indexing many documents, we can do indexing in batches: + +```go +batch := index.NewBatch() + +for _, doc := range documents { + batch.Index(doc.ID, doc) +} + +// Execute batch +err := index.Batch(batch) +``` diff --git a/docs/custom_query.md b/docs/custom_query.md new file mode 100644 index 0000000..244aef7 --- /dev/null +++ b/docs/custom_query.md @@ -0,0 +1,137 @@ +# Custom Queries: `custom_filter` and `custom_score` + +Bleve exposes two query nodes for embedding-defined per-hit logic: + +- `custom_filter`: keep or drop each candidate hit. +- `custom_score`: mutate each candidate hit score. + +Bleve itself only executes callbacks that were already provided/attached by the +embedding application. It does not interpret any +embedder-specific payload such as callback source, "params", or requested fields. + +## Query Objects + +`CustomFilterQuery` contains: + +- a child query +- a `searcher.CustomFilterFunc` callback attached by the embedder + +`CustomScoreQuery` contains: + +- a child query +- a `searcher.CustomScoreFunc` callback attached by the embedder + +## Constructors + +Embedding applications can construct the nodes directly with callbacks: + +```go +filterQuery := query.NewCustomFilterQueryWithFilter(childQuery, + func(d *search.DocumentMatch) bool { + return true + }, + nil, // fields; nil when callback does not need field values + nil, // payload; nil when no source/params round-trip is needed +) + +scoreQuery := query.NewCustomScoreQueryWithScorer(childQuery, + func(d *search.DocumentMatch) float64 { + return d.Score + }, + nil, // fields + nil, // payload +) +``` + +Constructor arguments after the callback are: + +- `fields`: field names to preload into `d.Fields` via doc values before the callback runs +- `payload`: non-query custom attributes that must survive JSON round-trips (for example distributed fan-out) + +Pass `nil` for either when not needed: + +```go +payload := map[string]interface{}{ + "params": map[string]interface{}{"weight": 0.05}, + "source": "function score(doc, params) { ... }", +} + +scoreQuery := query.NewCustomScoreQueryWithScorer(childQuery, + func(d *search.DocumentMatch) float64 { + f := d.Fields + return d.Score + (f["abv"].(float64) * 0.05) + }, + []string{"abv", "ibu"}, + payload, +) +``` + +Payload rules: + +- pass only non-query custom-node attributes like `source` and `params` +- do not include `query` in payload; `query` always comes from the child query argument +- do not include `fields` in payload; `fields` come from the dedicated constructor argument +- pass `nil` as the fourth argument when payload round-trip is not needed + +## Field Access and Doc Values Requirement + +When the `fields` constructor argument is non-empty, bleve loads the listed +field values into `DocumentMatch.Fields` before invoking the callback. Field +values are read from **doc values** (column-oriented, memory-mapped), not +stored fields. + +**Doc values must be enabled for every field listed in `"fields"`.** If a field +does not have doc values enabled in the index mapping, its value will be silently +absent from `DocumentMatch.Fields` at callback time — no error is raised, the +field simply won't appear. This can cause filters to drop all hits or scores to +compute incorrectly if the callback assumes the field is present. + +Doc values are enabled by default for all field types (`DocValues: true`) and +for dynamic mappings (`DocValuesDynamic: true`), so no extra configuration is +needed unless these defaults have been explicitly disabled. If they have been +disabled, re-enable them: + +```go +fm := mapping.NewTextFieldMapping() +fm.DocValues = true // default is true; only needed if previously disabled +imap.DefaultMapping.AddFieldMappingsAt("genre", fm) +``` + +For dynamic mappings, ensure `docvalues_dynamic` has not been set to `false` on +the document mapping. + +**Datetime fields require an explicit datetime mapping.** Dynamic mapping indexes +date-like strings as text, which means the standard analyzer tokenizes the value +(e.g. `"2010-07-22 20:00:20"` becomes `["2010", "07", "22", ...]`). The doc +value then contains tokens instead of the original date, making comparisons +fail. To use datetime fields in custom_filter or custom_score callbacks, add an +explicit field mapping with `"type": "datetime"` in the index definition. + +### Field type decoding + +At the bleve layer, doc values are decoded based on the field's mapping type: + +| Mapping type | Go type in `d.Fields` | Notes | +| --- | --- | --- | +| `text` / `keyword` | `string` | Raw text bytes | +| `number` | `float64` | IEEE 754 decoded | +| `boolean` | `bool` | `true` / `false` | +| `datetime` | `float64` | Epoch nanoseconds stored as a floating-point value | + +## Runtime Behavior + +- Bleve builds the child query searcher first. +- If `fields` is non-empty, a `DocValueReader` is created for those fields. +- `custom_filter` wraps the child with `CustomFilterSearcher`. +- `custom_score` wraps the child with `CustomScoreSearcher`. +- On each hit, doc values are loaded into `DocumentMatch.Fields`, then the callback is invoked. +- The callback receives only `*search.DocumentMatch`; it does not receive `SearchContext`. + +## Error Cases + +- missing child query: + - `"custom filter query must have a query"` + - `"custom score query must have a query"` +- missing bound callback: + - `"custom filter query must have a filter callback"` + - `"custom score query must have a score callback"` diff --git a/docs/fast_merge.md b/docs/fast_merge.md new file mode 100644 index 0000000..d477552 --- /dev/null +++ b/docs/fast_merge.md @@ -0,0 +1,64 @@ +# Fast Merge + +* v2.6.0 comes with the support for a feature called "fast merge" where we first train the index on a vector dataset and then build the vector index using this trained information of the centroid layout. +* This is a separate feature when compared to the existing behavior where we were performing the merge in a very naive fashion of reconstructing the participating vector indexes, re-training and then adding back the vectors into the index. Fast merge essentially merges the corresponding centroid cells' data vectors in a block-wise fashion without doing the expensive operations. For this, a centroid/trained segment template needs to be built using enough random samples from a dataset prior to building segments with the actual index data. +* A trade-off with this feature is if new data were introduced that is not available at the time of index introduction, a data drift can develop resulting in reduced recall. +* This feature underneath the hood utilizes the existing [`merge_from` API](https://github.com/blevesearch/faiss/blob/ffd910a91f1acf49b9898a7e514e462db89ee7b3/faiss/Index.h#L396) in our fork of the FAISS codebase. + +## Support + +* This feature is supported primarily for the IVF family of indexes. This happens when + * the field mapping has optimization type as `ivf,rabitq` or when using `bivf-sq8`/`bivf-flat` for binary quantization. + * when the above optimizations aren't used but the scale of data exceeds 10K vectors. + +## Usage + +The feature can be enabled by first passing a key-value pair in the config part while creating a new index. If the flag is false, then the behavior falls back to the more expensive naive merge. + +```go +kvConfig := map[string]interface{}{ + "vector_index_fast_merge": true, +} + +index, err := bleve.NewUsing("example.bleve", bleve.NewIndexMapping(), bleve.Config.DefaultIndexType, bleve.Config.DefaultMemKVStore, kvConfig) +if err != nil { + log.Fatal(err) +} +``` + +The user should now "train" the index on a random sample of the vector dataset they're planning to index and search. + +* It's completely up to the user as to how much data they want to use for training, controlling the batch size used while training and also marking whether the training is complete. +* NOTE: User must index their data only after marking the training as complete, otherwise the batch won't be indexed. + +The training is done using the new `Train()` API which takes the existing `Batch` construct we have + +```go +batch := index.NewBatch() +for _, doc := range trainingDocuments { + batch.Index(doc.ID, doc) +} + +// train the index on the batch of data +// NOTE: the training can be done in an incremental manner as well, by using the same Train() API but repeatedly calling it on a particular batch of data. +if err := index.Train(batch); err != nil { + log.Fatal(err) +} + +batch.Reset() +batch.SetInternal(util.BoltTrainCompleteKey, []byte("true")) +if err := index.Train(batch); err != nil { + log.Fatal(err) +} + +// at this point the index is trained completely, and any training related constructs in the background will be cleaned up +``` + +## Disclaimer + +* This feature is primarily meant for the use case where the user is aware of how much data they want to index and also for a read-heavy workload and little to no updates on the index itself. +* The intention of the feature is to be able to quickly index a massive scale of data on an index in an efficient manner and perform search on it. +* Without this feature, i.e. when the index build happens without a prior training phase + * The user wouldn't have to worry about use cases where the dataset is continuously updated with new "type" of vector. This is because each merge cycle would do the training afresh. + * The user doesn't have a lag in indexing the data either, they can start ingesting the data immediately. +* Based on what's mentioned above, when it comes to update and delete type of workloads on the dataset it's extremely difficult to detect when the data drift will occur. So we end up falling back to the naive way of reconstructing + re-training. diff --git a/docs/geo.md b/docs/geo.md new file mode 100644 index 0000000..542da23 --- /dev/null +++ b/docs/geo.md @@ -0,0 +1,3 @@ +# Geo spatial search + +Redirect to [geo/README.md](https://github.com/blevesearch/bleve/blob/master/geo/README.md) diff --git a/docs/hierarchy.md b/docs/hierarchy.md new file mode 100644 index 0000000..839e9f4 --- /dev/null +++ b/docs/hierarchy.md @@ -0,0 +1,376 @@ +# Hierarchical nested search + +* *v2.6.0* (and after) will come with support for **array indexing and hierarchical nested search**. +* We've achieved this by embedding nested documents within our bleve (scorch) indexes. +* Usage of zap file format: [v17](https://github.com/blevesearch/zapx/blob/master/zap.md). Here we preserve hierarchical document relationships within segments, continuing to conform to the segmented architecture of *scorch*. + +## Supported + +* Indexing `Arrays` allows specifying fields that contain arrays of objects. Each object in the array can have its own set of fields, enabling the representation of hierarchical data structures within a single document. + +```json +{ + "id": "1", + "name": "John Doe", + "addresses": [ + { + "type": "home", + "street": "123 Main St", + "city": "Hometown", + "zip": "12345" + }, + { + "type": "work", + "street": "456 Corporate Blvd", + "city": "Metropolis", + "zip": "67890" + } + ] +} +``` + +* Multi-level arrays: arrays can contain objects that themselves have array fields, allowing for deeply nested structures, such as a list of projects, each with its own list of tasks. + +```json +{ + "id": "2", + "name": "Jane Smith", + "projects": [ + { + "name": "Project Alpha", + "tasks": [ + {"title": "Task 1", "status": "completed"}, + {"title": "Task 2", "status": "in-progress"} + ] + }, + { + "name": "Project Beta", + "tasks": [ + {"title": "Task A", "status": "not-started"}, + {"title": "Task B", "status": "completed"} + ] + } + ] +} +``` + +* Multiple arrays: A document can have multiple fields that are arrays, each representing different hierarchical data, such as a list of phone numbers and a list of email addresses. + +```json +{ + "id": "3", + "name": "Alice Johnson", + "phones": [ + {"type": "mobile", "number": "555-1234"}, + {"type": "home", "number": "555-5678"} + ], + "emails": [ + {"type": "personal", "address": "alice@example.com"}, + {"type": "work", "address": "alice@work.com"} + ] +} +``` + +* Hybrid arrays: Multi-level and multiple arrays can be combined within the same document to represent complex hierarchical data structures, such as a company with multiple departments, each having its own list of employees and projects. + +```json +{ + "id": "doc1", + "company": { + "id": "c1", + "name": "TechCorp", + "departments": [ + { + "name": "Engineering", + "budget": 2000000, + "employees": [ + {"name": "Alice", "role": "Engineer"}, + {"name": "Bob", "role": "Manager"} + ], + "projects": [ + {"title": "Project X", "status": "ongoing"}, + {"title": "Project Y", "status": "completed"} + ] + }, + { + "name": "Sales", + "budget": 300000, + "employees": [ + {"name": "Eve", "role": "Salesperson"}, + {"name": "Mallory", "role": "Manager"} + ], + "projects": [ + {"title": "Project A", "status": "completed"}, + {"title": "Project B", "status": "ongoing"} + ] + } + ], + "locations": [ + {"city": "Athens","country": "Greece"}, + {"city": "Berlin", "country": "USA"} + ] + } +} +``` + +* Earlier versions of Bleve only supported flat arrays of primitive types (e.g., strings, numbers), and would flatten nested structures, losing the hierarchical relationships, so the above complex documents could not be accurately represented or queried. For example, the "employees" and "projects" fields within each department would be flattened, making it impossible to associate employees with their respective departments. + +* From v2.6.0 onwards, Bleve allows for accurate representation and querying of complex nested structures, preserving the relationships between different levels of the hierarchy, across multi-level, multiple and hybrid arrays. + +* The addition of `nested` document mappings enables the definition of fields that contain arrays of objects, giving the option to preserve the hierarchical relationships within the array during indexing. Having `nested` as false (default) will flatten the objects within the array, losing the hierarchy, which was the earlier behavior. + +```json +{ + "departments": { + "dynamic": false, + "enabled": true, + "nested": true, + "properties": { + "employees": { + "dynamic": false, + "enabled": true, + "nested": true + }, + "projects": { + "dynamic": false, + "enabled": true, + "nested": true + } + } + }, + "locations": { + "dynamic": false, + "enabled": true, + "nested": true + } +} +``` + +* Any Bleve query (e.g., `match`, `phrase`, `term`, `fuzzy`, `numeric/date range`, etc.) can be executed against fields within nested documents, with no special handling required. The query processor will automatically traverse the nested structures to find matches. Additional search constructs +like vector search, synonym search, hybrid and pre-filtered vector search integrate seamlessly with hierarchy search. + +* Conjunction Queries (AND queries) and other queries that depend on term co-occurrence within the same hierarchical context will respect the boundaries of nested documents. This means that terms must appear within the same nested object to be considered a match. For example, a conjunction query searching for an employee named "Alice" with the role "Engineer" within the "Engineering" department will only return results where both name and role terms are found within the same employee object, which is itself within an "Engineering" department object. + +* Some other search constructs will have enhanced precision with hierarchy search. + * Field-Level Highlighting: Only fields within the matched nested object are retrieved and highlighted, ensuring highlights appear in the correct hierarchical context. For example, a match in `departments[name=Engineering].employees` highlights only employees in that department. + + * Nested Faceting / Aggregations: Facets are computed within matched nested objects, producing context-aware buckets. E.g., a facet on `departments.projects.status` returns ongoing or completed only for projects in matched departments. + + * Sorting by Nested Fields: Sorting can use fields from the relevant nested object, e.g., ordering companies by `departments.budget` sorts based on the budget of the specific matched department, not unrelated departments. + +* Vector Search (KNN / Multi-KNN): When a document contains an array of objects with vector/multi-vector fields, the final document score and ranking are identical whether or not the array is marked as `nested`. In both cases, the highest-scoring vector is selected; either directly from the array (non-nested) or from the best-matching nested object with its score bubbled up to the parent document. + +* Pre-Filtered Vector Search: When vector search is combined with filters on fields inside a nested array, the filters are applied first to pick which nested items are eligible. Vector similarity is then computed only on the vector fields of those filtered nested objects. For example, if `departments.employees` is a `nested` array, a pre-filtered KNN query for employees with a `skills_vector` matching `machine learning engineer`, a role of `Manager`, and belonging to the `Sales` department will first narrow the candidate set to only employees who meet the requirement, and then compute vector similarity on the `skills_vector` of that filtered subset. This ensures that vector search results come only from the employees that satisfy the filter, and not from unrelated employees in other departments. + +## Indexing + +Below is an example of using the Bleve API to index documents with hierarchical structures, using hybrid arrays and nested mappings. + +```go +// Define a document to be indexed. +docJSON := + `{ + "company": { + "id": "c3", + "name": "WebSolutions", + "departments": [ + { + "name": "HR", + "budget": 800000, + "employees": [ + {"name": "Eve", "role": "Manager"}, + {"name": "Frank", "role": "HR"} + ], + "projects": [ + {"title": "Project Beta", "status": "completed"}, + {"title": "Project B", "status": "ongoing"} + ] + }, + { + "name": "Engineering", + "budget": 200000, + "employees": [ + {"name": "Heidi", "role": "Support Engineer"}, + {"name": "Ivan", "role": "Manager"} + ], + "projects": [ + {"title": "Project Helpdesk", "status": "ongoing"}, + {"title": "Project FAQ", "status": "completed"} + ] + } + ], + "locations": [ + {"city": "Edinburgh", "country": "UK"}, + {"city": "London", "country": "Canada"} + ] + } + }` + +// Define departments as a nested document mapping (since it contains arrays of objects) +// and index name and budget fields +departmentsMapping := bleve.NewNestedDocumentMapping() +departmentsMapping.AddFieldMappingsAt("name", bleve.NewTextFieldMapping()) +departmentsMapping.AddFieldMappingsAt("budget", bleve.NewNumericFieldMapping()) + +// Define employees as a nested document mapping within departments (since it contains arrays of objects) +// and index name and role fields +employeesMapping := bleve.NewNestedDocumentMapping() +employeesMapping.AddFieldMappingsAt("name", bleve.NewTextFieldMapping()) +employeesMapping.AddFieldMappingsAt("role", bleve.NewTextFieldMapping()) +departmentsMapping.AddSubDocumentMapping("employees", employeesMapping) + +// Define projects as a nested document mapping within departments (since it contains arrays of objects) +// and index title and status fields +projectsMapping := bleve.NewNestedDocumentMapping() +projectsMapping.AddFieldMappingsAt("title", bleve.NewTextFieldMapping()) +projectsMapping.AddFieldMappingsAt("status", bleve.NewTextFieldMapping()) +departmentsMapping.AddSubDocumentMapping("projects", projectsMapping) + +// Define locations as a nested document mapping (since it contains arrays of objects) +// and index city and country fields +locationsMapping := bleve.NewNestedDocumentMapping() +locationsMapping.AddFieldMappingsAt("city", bleve.NewTextFieldMapping()) +locationsMapping.AddFieldMappingsAt("country", bleve.NewTextFieldMapping()) + +// Define company as a document mapping and index its name field and +// add departments and locations as sub-document mappings +companyMapping := bleve.NewDocumentMapping() +companyMapping.AddFieldMappingsAt("name", bleve.NewTextFieldMapping()) +companyMapping.AddSubDocumentMapping("departments", departmentsMapping) +companyMapping.AddSubDocumentMapping("locations", locationsMapping) + +// Define the final index mapping and add company as a sub-document mapping in the default mapping +indexMapping := bleve.NewIndexMapping() +indexMapping.DefaultMapping.AddSubDocumentMapping("company", companyMapping) + +// Create the index with the defined mapping +index, err := bleve.New("hierarchy_example.bleve", indexMapping) +if err != nil { + panic(err) +} + +// Unmarshal the document JSON into a map, for indexing +var doc map[string]interface{} +err = json.Unmarshal([]byte(docJSON), &doc) +if err != nil { + panic(err) +} + +// Index the document +err = index.Index("doc1", doc) +if err != nil { + panic(err) +} +``` + +## Querying + +```go +// Open the index +index, err := bleve.Open("hierarchy_example.bleve") +if err != nil { + panic(err) +} + +var ( + req *bleve.SearchRequest + res *bleve.SearchResult +) + +// Example 1: Simple Match Query on a field within a nested document, should work as if it were a flat field +q1 := bleve.NewMatchQuery("Engineer") +q1.SetField("company.departments.employees.role") +req = bleve.NewSearchRequest(q1) +res, err = index.Search(req) +if err != nil { + panic(err) +} +fmt.Println("Match Query Results:", res) + +// Example 2: Conjunction Query (AND) on fields within the same nested document +// like finding employees with name "Eve" and role "Manager". This will only match +// if both terms are in the same employee object. +q1 = bleve.NewMatchQuery("Eve") +q1.SetField("company.departments.employees.name") +q2 := bleve.NewMatchQuery("Manager") +q2.SetField("company.departments.employees.role") +conjQuery := bleve.NewConjunctionQuery( + q1, + q2, +) +req = bleve.NewSearchRequest(conjQuery) +res, err = index.Search(req) +if err != nil { + panic(err) +} +fmt.Println("Conjunction Query Results:", res) + +// Example 3: Multi-level Nested Query, finding projects with status "ongoing" +// within the "Engineering" department. This ensures both conditions are met +// within the correct hierarchy, i.e., the ongoing project must belong to the +// Engineering department. +q1 = bleve.NewMatchQuery("Engineering") +q1.SetField("company.departments.name") +q2 = bleve.NewMatchQuery("ongoing") +q2.SetField("company.departments.projects.status") +multiLevelQuery := bleve.NewConjunctionQuery( + q1, + q2, +) +req = bleve.NewSearchRequest(multiLevelQuery) +res, err = index.Search(req) +if err != nil { + panic(err) +} +fmt.Println("Multi-level Nested Query Results:", res) + +// Example 4: Multiple Arrays Query, finding documents with a location in "London" +// and an employee with the role "Manager". This checks conditions across different arrays. +q1 = bleve.NewMatchQuery("London") +q1.SetField("company.locations.city") +q2 = bleve.NewMatchQuery("Manager") +q2.SetField("company.departments.employees.role") +multiArrayQuery := bleve.NewConjunctionQuery( + q1, + q2, +) +req = bleve.NewSearchRequest(multiArrayQuery) +res, err = index.Search(req) +if err != nil { + panic(err) +} +fmt.Println("Multiple Arrays Query Results:", res) + +// Hybrid Arrays Query, combining multi-level and multiple arrays, +// finding documents with a Manager named Ivan working in Edinburgh, UK +q1 = bleve.NewMatchQuery("Ivan") +q1.SetField("company.departments.employees.name") +q2 = bleve.NewMatchQuery("Manager") +q2.SetField("company.departments.employees.role") +q3 := bleve.NewMatchQuery("Edinburgh") +q3.SetField("company.locations.city") +q4 := bleve.NewMatchQuery("UK") +q4.SetField("company.locations.country") +hybridArrayQuery := bleve.NewConjunctionQuery( + bleve.NewConjunctionQuery( + q1, + q2, + ), + bleve.NewConjunctionQuery( + q3, + q4, + ), +) +req = bleve.NewSearchRequest(hybridArrayQuery) +res, err = index.Search(req) +if err != nil { + panic(err) +} +fmt.Println("Hybrid Arrays Query Results:", res) + +// Close the index when done +err = index.Close() +if err != nil { + panic(err) +} +``` diff --git a/docs/images/indexSizeVsNumDocs.png b/docs/images/indexSizeVsNumDocs.png new file mode 100644 index 0000000..1121170 Binary files /dev/null and b/docs/images/indexSizeVsNumDocs.png differ diff --git a/docs/images/queryTimevsNumDocs.png b/docs/images/queryTimevsNumDocs.png new file mode 100644 index 0000000..151b422 Binary files /dev/null and b/docs/images/queryTimevsNumDocs.png differ diff --git a/docs/images/search_autocomplete.png b/docs/images/search_autocomplete.png new file mode 100644 index 0000000..285f134 Binary files /dev/null and b/docs/images/search_autocomplete.png differ diff --git a/docs/index_update.md b/docs/index_update.md new file mode 100644 index 0000000..74a2125 --- /dev/null +++ b/docs/index_update.md @@ -0,0 +1,45 @@ +# Ability to reduce downtime during index mapping updates + +* *v2.5.4* (and after) will come with support for deleting or modifying any field mapping in the index mapping without requiring a full rebuild of the index +* We do this by storing which portions of the field that have to be deleted within zap and then lazily executing the deletion during subsequent merging of the segments + +## Usage + +While opening an index, if an updated mapping is provided as a string under the key `updated_mapping` within the `runtimeConfig` parameter of `OpenUsing`, then we open the index and try to update it to use the new mapping provided. + +If the update fails, the index is unchanged and an error is returned explaining why the update was unsuccessful. + +## What can be deleted and what can't be deleted? + +Fields can be partially deleted by changing their Index, Store, and DocValues parameters from true to false, or completely removed by deleting the field itself. + +Additionally, document mappings can be deleted either by fully removing them from the index mapping or by setting the Enabled value to false, which deletes all fields defined within that mapping. + +However, if any of the following conditions are met, the index is considered non-updatable. + +* Any additional fields or enabled document mappings in the new index mapping +* Any changes to IncludeInAll, type, IncludeTermVectors and SkipFreqNorm +* Any document mapping having its enabled value changing from false to true +* Text fields with a different analyzer or date time fields with a different date time format +* Vector and VectorBase64 fields changing dims, similarity or vectorIndexOptimizedFor +* Any changes when field is part of `_all` +* Full field deletions when it is covered by any dynamic setting (Index, Store or DocValues Dynamic) +* Any changes to dynamic settings at the top level or any enabled document mapping +* If multiple fields sharing the same field name either from different type mappings or aliases are present, then any non-compatible changes across all of these fields + +## How to enforce immediate deletion? + +Since the deletion is only done during merging, a [force merge](https://github.com/blevesearch/bleve/blob/b82baf10b205511cf12da5cb24330abd9f5b1b74/index/scorch/merge.go#L164) may be used to completely remove the stale data. + +## Sample code to update an existing index + +```go +newMapping := `` +config := map[string]interface{}{ + "updated_mapping": newMapping, +} +index, err := bleve.OpenUsing("", config) +if err != nil { + return err +} +``` diff --git a/docs/pagination.md b/docs/pagination.md new file mode 100644 index 0000000..b853f78 --- /dev/null +++ b/docs/pagination.md @@ -0,0 +1,109 @@ +# Pagination + +## Why pagination matters + +Search queries can match many documents. Pagination lets you fetch and display results in chunks, keeping responses small and fast. + +By default, Bleve returns the first 10 hits sorted by relevance (score), highest first. + +## Two pagination modes + +- `From`/`Size`: simple and stateless; cost grows with page depth. +- `SearchAfter`/`SearchBefore`: efficient for deep paging; requires passing sort keys from the previous page. + +Both modes can be used with any valid sort. + +## `Size`/`From` + +Offset-based pagination uses `Size` (page length) and `From` (number of hits to skip). Bleve collects at least `Size + From` ordered results, then returns the `Size` slice starting at `From`. + +JSON example: + +```json +{ + "query": { "match": "California" }, + "sort": ["-_score"], + "size": 5, + "from": 10 +} +``` + +The result would be 5 hits starting from the 11th hit. + +When to use: + +- Simple, stateless pagination for shallow pages. +- Avoid for deep pages, as memory grows with `From` for deeper pages. + +## `SearchAfter` and `SearchBefore` + +This returns the next (or previous) page based on a boundary defined by the sort keys of a specific hit. This keeps resource usage proportional to the page size, even for deep pages. + +Rules: + +- Use either `SearchAfter` (forward) or `SearchBefore` (backward), not both at once. +- The length of `SearchAfter`/`SearchBefore` must match the length of `Sort`. +- Values are strings representing the sort keys, in the same order as `Sort`. +- Keep the same `query` and `sort` across pages for consistent navigation. + +Where do sort keys come from? + +- Each hit includes `Sort` (and `DecodedSort` from Bleve v2.5.2). Take the last hit's sort keys for `SearchAfter`, or the first hit's sort keys for `SearchBefore`. +- If the field/fields to be searched over are numeric, datetime or geo, the values in the `Sort` field may have garbled values; this is because of how Bleve represents such data types internally. To use such fields as sort keys, use the `DecodedSort` field, which decodes the internal representations. This feature is available from Bleve v2.5.4. + +> When using `DecodedSort`, the `Sort` array in the search request needs to explicitly declare the type of the field for proper decoding. Hence, the `Sort` array must contain either `SortField` objects (for numeric and datetime) or `SortGeoDistance` objects (for geo) rather than just the field names. More info on `SortField` and `SortGeoDistance` can be found in [sort_facet.md](sort_facet.md). + +Forward pagination over `_id` and `_score`: + +```json +{ + "query": { "match": "California" }, + "sort": ["_id", "_score"], + "search_after": ["hotel_10180", "0.998"], + "size": 3 +} +``` + +Backward pagination over `_id` and `_score`: + +```json +{ + "query": { "match": "California" }, + "sort": ["_id", "_score"], + "search_before": ["hotel_17595", "0.623"], + "size": 4 +} +``` + +Pagination using numeric, datetime, and geo fields. Notice how we specify the sort objects, with the "type" field explicitly declared in case of numeric and datetime: + +```json +{ + "query": { + "match_all": {} + }, + "size": 10, + "sort": [ + {"by": "field", "field": "price", "type": "number"}, + {"by": "field", "field": "created_at", "type": "date"}, + {"by": "geo_distance", "field": "location", "location": {"lat": 40.7128, "lon": -74.0060}} + ], + "search_after": ["99.99", "2023-10-15T10:30:00Z", "5.2"] +} +``` + +## Total Sort Order + +Pagination is deterministic. Ensure your `Sort` defines a total order, so that documents with the same sort keys are not left out: + +- Sort strings can be field names (prefix with `-` for descending), `"_score"`, or `"_id"`. +- Always include a stable tie-breaker as the last key, typically `"_id"`. +- Examples: + - `["country", "-age", "_id"]` + - `["-_score", "_id"]` (default score desc with a tie-breaker) + +## Performance guidance + +- Offset pagination cost grows with `From` (collects at least `Size + From` results before slicing). +- `SearchAfter`/`SearchBefore` keeps memory and network proportional to `Size`. +- For large datasets and deep navigation, prefer using `SearchAfter` and `SearchBefore`. diff --git a/docs/persister.md b/docs/persister.md new file mode 100644 index 0000000..d401357 --- /dev/null +++ b/docs/persister.md @@ -0,0 +1,57 @@ +# Scorch Index Memory and File Management + +## Memory Management + +When data is indexed in Scorch — using either the `index.Index()` or `index.Batch()` API — it is added as part of an in-memory "segment". Memory management in Scorch indexing mainly relates to handling these in-memory segments during workloads that involve inserts or updates. + +In scenarios with a continuous stream of incoming data, a large number of in-memory segments can accumulate over time. This is where the persister component comes into play—its job is to flush these in-memory segments to disk. + +Starting with v2.5.0, Scorch supports parallel flushing of in-memory segments to disk, where the persister checks the total in-memory data and distributes the flush across multiple workers. This feature is disabled by default and can be enabled using two configuration options: + +- `NumPersisterWorkers`: This factor decides how many maximum workers can be spawned to flush out the in-memory segments. Each worker will work on a disjoint subset of segments, merge them, and flush them out to the disk. By default the persister deploys only one worker. +- `MaxSizeInMemoryMergePerWorker`: This config decides what's the maximum amount of input data in bytes a single worker can work upon. By default this value is equal to 0 which means that this config is disabled and the worker tries to merge all the data in one shot. Also note that it's imperative that the user set this config if `NumPersisterWorkers > 1`. + +If the index is tuned to have a higher `NumPersisterWorkers` value, the memory can potentially drain out faster and ensure stronger consistency behavior — but there would be a lot of on-disk files, and the background merger would experience the pressure of managing this large number of files, which can be resource-intensive. + +- Tuning this config is very dependent on the available CPU resources. Something to keep in mind here is that the process's RSS can increase if the number of workers — and each of them working upon a large amount of data — is high. + +Increasing the `MaxSizeInMemoryMergePerWorker` value would mean that each worker acts upon a larger amount of data and spends more time merging and flushing it out to disk — which can be healthy behavior in terms of I/O, although it comes at the cost of time. + +- Changing this config is use-case dependent; for example, in use cases where the payload or per-doc size is generally large (for example vector use cases), it would be beneficial to have a larger value for this. + +So, having the ideal values for these two configs is definitely dependent on the use case and can involve a bunch of experiments, keeping the resource usage in mind. + +## File Management + +The persister introducing some number of file segments into the system would change the state of the system, and the merger would wake up and try to manage these on-disk files. + +Management of these files is crucial when it comes to query latency because a higher number of files would dictate searching through a larger number of files and also higher read amplification to some extent, because the backing data structures can potentially be compacted in size across files. + +The merger sees the files on disk and plans out which segments to merge so that the final layout of segment tiers (each tier having multiple files), which grow in a logarithmic way (the chances of larger tiers growing in number would decrease), is maintained. This also implies that deciding this first-tier size becomes important in deciding the number of segment files across all tiers. + +Starting with v2.5.0, this first-tier size is dependent on the file size using the `FloorSegmentFileSize` config, because that's a better metric to consider (unlike the legacy live doc count metric) in order to ensure that the behavior is in line with the use case and aware of the payload/doc size. + +- This config can also be tuned to dictate how the I/O behaviour should be within an index. While tuning this config, it should be in proportion to the `MaxSizeInMemoryMergePerWorker` since that dictates the amount of data flushed out per flush. +- The observation here is that `FloorSegmentFileSize` is less than `MaxSizeInMemoryMergePerWorker` and for an optimal I/O during indexing, this value can be set close to `MaxSizeInMemoryMergePerWorker/6`. + +## Setting a Persister/Merger Config in Index + +The configs are set via the `kvConfig` parameter in the `NewUsing()` or `OpenUsing()` API: + +```go +// setting the persister and merger configs +kvConfig := map[string]interface{}{ + "scorchPersisterOptions": map[string]interface{}{ + "numPersisterWorkers": 4, + "maxSizeInMemoryMergePerWorker": 20000000, + }, + "scorchMergePlanOptions": map[string]interface{}{ + "floorSegmentFileSize": 10000000, + }, +} +// passing the config to the index +index, err := bleve.NewUsing("example.bleve", bleve.NewIndexMapping(), bleve.Config.DefaultIndexType, bleve.Config.DefaultMemKVStore, kvConfig) +if err != nil { + panic(err) +} +``` diff --git a/docs/query-openapi-spec.yaml b/docs/query-openapi-spec.yaml new file mode 100644 index 0000000..02cd586 --- /dev/null +++ b/docs/query-openapi-spec.yaml @@ -0,0 +1,862 @@ +openapi: 3.0.3 +info: + title: Bleve JSON Query Language + description: | + A comprehensive specification for the Bleve Query JSON Language. + + Bleve is a text indexing library for Go that provides full-text search capabilities. + This specification defines the JSON structure for all supported query types in Bleve. + + ## Query Types Supported: + + ### Basic Queries + - **Term Query**: Exact term matching + - **Match Query**: Full-text search with analysis + - **Match Phrase Query**: Phrase matching with proximity + - **Prefix Query**: Prefix matching + - **Wildcard Query**: Pattern matching with * and ? + - **Regexp Query**: Regular expression matching + - **Fuzzy Query**: Fuzzy matching with edit distance + - **Query String Query**: Human-readable query syntax + + ### Range Queries + - **Numeric Range Query**: Numeric value ranges + - **Date Range Query**: Date/time ranges + - **Term Range Query**: Lexicographic term ranges + + ### Boolean Queries + - **Boolean Query**: Must/should/must_not combinations + - **Conjunction Query**: All queries must match + - **Disjunction Query**: Any query can match + + ### Special Queries + - **Match All Query**: Matches all documents + - **Match None Query**: Matches no documents + - **Bool Field Query**: Boolean field matching + - **Doc ID Query**: Document ID matching + + ### Geographic Queries + - **Geo Distance Query**: Distance-based geographic search + - **Geo Bounding Box Query**: Bounding box geographic search + - **Geo Bounding Polygon Query**: Polygon geographic search + - **Geo Shape Query**: Complex geographic shape matching + + ### Vector Queries + - **KNN Query**: K-nearest neighbors vector search + + ### Network Queries + - **IP Range Query**: IP address range matching + + ## Common Properties + + All queries support: + - **boost**: Query boost value (float, default 1.0) + - **field**: Field to search (string, optional) + + ## Query Parsing + + Bleve automatically detects query types based on the presence of specific JSON fields. + The query parser uses a heuristic approach to determine the query type from the JSON structure. + + version: 2.0.0 + contact: + name: Bleve Search + url: https://github.com/blevesearch/bleve + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0 + +tags: + - name: Basic Queries + description: Basic text search queries + - name: Range Queries + description: Range-based queries for numeric, date, and term values + - name: Boolean Queries + description: Compound queries combining multiple sub-queries + - name: Special Queries + description: Special purpose queries like match all/none + - name: Geographic Queries + description: Geographic and spatial search queries + - name: Vector Queries + description: Vector similarity and KNN queries + - name: Network Queries + description: Network and IP address queries + +components: + schemas: + # Base Query Schema + Query: + type: object + description: Base query object - all queries extend this + properties: + boost: + type: number + format: float + description: Query boost value + default: 1.0 + minimum: 0 + field: + type: string + description: Field to search (optional, uses default field if not specified) + additionalProperties: true + + # Basic Queries + TermQuery: + allOf: + - type: object + properties: + term: + type: string + description: Exact term to search for + example: "search" + required: + - term + example: + term: "search" + field: "title" + boost: 2.0 + + MatchQuery: + allOf: + - type: object + properties: + match: + type: string + description: Text to match (will be analyzed) + example: "full text search" + operator: + type: string + enum: [and, or] + description: Boolean operator for multiple terms + default: "or" + fuzziness: + oneOf: + - type: integer + minimum: 0 + maximum: 2 + description: Edit distance for fuzzy matching + - type: string + enum: ["auto"] + description: Automatic fuzziness + default: 0 + prefix_length: + type: integer + description: Prefix length for fuzzy matching + minimum: 0 + default: 0 + analyzer: + type: string + description: Analyzer to use (optional) + required: + - match + example: + match: "full text search" + field: "content" + operator: "and" + fuzziness: 1 + + MatchPhraseQuery: + allOf: + - type: object + properties: + match_phrase: + type: string + description: Phrase to match exactly + example: "exact phrase match" + fuzziness: + oneOf: + - type: integer + minimum: 0 + maximum: 2 + - type: string + enum: ["auto"] + default: 0 + analyzer: + type: string + description: Analyzer to use (optional) + required: + - match_phrase + example: + match_phrase: "exact phrase match" + field: "content" + fuzziness: "auto" + + PrefixQuery: + allOf: + - type: object + properties: + prefix: + type: string + description: Prefix to match + example: "pref" + required: + - prefix + example: + prefix: "pref" + field: "title" + + WildcardQuery: + allOf: + - type: object + properties: + wildcard: + type: string + description: Wildcard pattern (* matches any sequence, ? matches single character) + example: "te*t" + required: + - wildcard + example: + wildcard: "te*t" + field: "title" + + RegexpQuery: + allOf: + - type: object + properties: + regexp: + type: string + description: Regular expression pattern (should not include ^ or $) + example: "te[st]" + required: + - regexp + example: + regexp: "te[st]" + field: "title" + + FuzzyQuery: + allOf: + - type: object + properties: + term: + type: string + description: Term for fuzzy matching + example: "search" + fuzziness: + oneOf: + - type: integer + minimum: 0 + maximum: 2 + - type: string + enum: ["auto"] + default: 1 + prefix_length: + type: integer + description: Prefix length for fuzzy matching + minimum: 0 + default: 0 + required: + - term + example: + term: "search" + field: "title" + fuzziness: "auto" + + QueryStringQuery: + allOf: + - type: object + properties: + query: + type: string + description: Query string in human-readable syntax + example: "title:search AND content:full text" + required: + - query + example: + query: "title:search AND content:full text" + + # Range Queries + NumericRangeQuery: + allOf: + - type: object + properties: + min: + type: number + format: float + description: Minimum value (inclusive by default) + max: + type: number + format: float + description: Maximum value (exclusive by default) + inclusive_min: + type: boolean + description: Whether minimum is inclusive + default: true + inclusive_max: + type: boolean + description: Whether maximum is inclusive + default: false + example: + min: 10.5 + max: 100.0 + inclusive_min: true + inclusive_max: false + field: "price" + + DateRangeQuery: + allOf: + - type: object + properties: + start: + type: string + format: date-time + description: Start date/time + end: + type: string + format: date-time + description: End date/time + inclusive_start: + type: boolean + description: Whether start is inclusive + default: true + inclusive_end: + type: boolean + description: Whether end is inclusive + default: false + example: + start: "2023-01-01T00:00:00Z" + end: "2023-12-31T23:59:59Z" + inclusive_start: true + inclusive_end: true + field: "created_at" + + TermRangeQuery: + allOf: + - type: object + properties: + min: + type: string + description: Minimum term (lexicographic) + max: + type: string + description: Maximum term (lexicographic) + inclusive_min: + type: boolean + description: Whether minimum is inclusive + default: true + inclusive_max: + type: boolean + description: Whether maximum is inclusive + default: false + example: + min: "a" + max: "m" + inclusive_min: true + inclusive_max: false + field: "name" + + # Boolean Queries + BooleanQuery: + allOf: + - type: object + properties: + must: + oneOf: + - type: array + items: + $ref: '#/components/schemas/Query' + - $ref: '#/components/schemas/Query' + description: Queries that must match (can be single query or array) + should: + oneOf: + - type: array + items: + $ref: '#/components/schemas/Query' + - $ref: '#/components/schemas/Query' + description: Queries that should match (boost score, can be single query or array) + must_not: + oneOf: + - type: array + items: + $ref: '#/components/schemas/Query' + - $ref: '#/components/schemas/Query' + description: Queries that must not match (can be single query or array) + filter: + oneOf: + - type: array + items: + $ref: '#/components/schemas/Query' + - $ref: '#/components/schemas/Query' + description: Queries that filter results (no score impact, can be single query or array) + min_should: + type: integer + description: Minimum number of should queries that must match + minimum: 0 + example: + must: + - term: "important" + field: "status" + - match: "urgent" + field: "title" + should: + - match: "critical" + field: "content" + - match: "priority" + field: "tags" + must_not: + - term: "archived" + field: "status" + filter: + - term: "published" + field: "status" + boost: 1.5 + + ConjunctionQuery: + allOf: + - type: object + properties: + conjuncts: + type: array + items: + $ref: '#/components/schemas/Query' + description: Array of queries that must all match + minItems: 1 + required: + - conjuncts + example: + conjuncts: + - term: "search" + field: "title" + - match: "full text" + field: "content" + + DisjunctionQuery: + allOf: + - type: object + properties: + disjuncts: + type: array + items: + $ref: '#/components/schemas/Query' + description: Array of queries where at least one must match + minItems: 1 + min: + type: number + format: float + description: Minimum number of disjuncts that must match + default: 1 + minimum: 0 + required: + - disjuncts + example: + disjuncts: + - term: "urgent" + field: "title" + - term: "critical" + field: "content" + min: 1 + + # Special Queries + MatchAllQuery: + allOf: + - type: object + properties: + match_all: + type: object + description: Empty object indicating match all + example: + match_all: {} + boost: 1.0 + + MatchNoneQuery: + allOf: + - type: object + properties: + match_none: + type: object + description: Empty object indicating match none + example: + match_none: {} + + BoolFieldQuery: + allOf: + - type: object + properties: + bool: + type: boolean + description: Boolean value to match + required: + - bool + example: + bool: true + field: "published" + + DocIDQuery: + allOf: + - type: object + properties: + ids: + type: array + items: + type: string + description: Array of document IDs to match + minItems: 1 + required: + - ids + example: + ids: ["doc1", "doc2", "doc3"] + + # Geographic Queries + GeoDistanceQuery: + allOf: + - type: object + properties: + location: + oneOf: + - type: array + items: + type: number + format: float + minItems: 2 + maxItems: 2 + description: [longitude, latitude] + - type: object + properties: + lon: + type: number + format: float + lat: + type: number + format: float + distance: + type: string + description: Distance string (e.g., "10km", "5mi") + required: + - location + - distance + example: + location: [-122.4194, 37.7749] + distance: "10km" + field: "location" + + GeoBoundingBoxQuery: + allOf: + - type: object + properties: + top_left: + oneOf: + - type: array + items: + type: number + format: float + minItems: 2 + maxItems: 2 + - type: object + properties: + lon: + type: number + format: float + lat: + type: number + format: float + bottom_right: + oneOf: + - type: array + items: + type: number + format: float + minItems: 2 + maxItems: 2 + - type: object + properties: + lon: + type: number + format: float + lat: + type: number + format: float + required: + - top_left + - bottom_right + example: + top_left: [-122.5, 37.8] + bottom_right: [-122.3, 37.7] + field: "location" + + GeoBoundingPolygonQuery: + allOf: + - type: object + properties: + polygon_points: + type: array + items: + oneOf: + - type: array + items: + type: number + format: float + minItems: 2 + maxItems: 2 + - type: object + properties: + lon: + type: number + format: float + lat: + type: number + format: float + description: Array of polygon points [lon, lat] + minItems: 3 + required: + - polygon_points + example: + polygon_points: + - [-122.5, 37.8] + - [-122.3, 37.8] + - [-122.3, 37.7] + - [-122.5, 37.7] + field: "location" + + GeoShapeQuery: + allOf: + - type: object + properties: + geometry: + type: object + description: GeoJSON geometry object + properties: + type: + type: string + enum: [Point, LineString, Polygon, MultiPoint, MultiLineString, MultiPolygon, GeometryCollection] + coordinates: + description: Geometry coordinates (format depends on geometry type) + oneOf: + - type: array + description: Coordinates for Point, LineString, MultiPoint, etc. + - type: array + items: + type: array + description: Nested coordinates for Polygon, MultiLineString, etc. + - type: array + items: + type: array + items: + type: array + description: Deeply nested coordinates for MultiPolygon, etc. + required: + - type + - coordinates + required: + - geometry + example: + geometry: + type: "Polygon" + coordinates: + - [[-122.5, 37.8], [-122.3, 37.8], [-122.3, 37.7], [-122.5, 37.7], [-122.5, 37.8]] + field: "location" + + # Vector Queries + KNNQuery: + allOf: + - type: object + properties: + field: + type: string + description: Vector field name + vector: + type: array + items: + type: number + format: float + description: Query vector + minItems: 1 + k: + type: integer + format: int64 + description: Number of nearest neighbors to return + minimum: 1 + maximum: 10000 + params: + type: object + description: Additional parameters for vector search + required: + - field + - vector + - k + example: + field: "embedding" + vector: [0.1, 0.2, 0.3, 0.4, 0.5] + k: 10 + boost: 1.0 + + # Network Queries + IPRangeQuery: + allOf: + - type: object + properties: + cidr: + type: string + description: CIDR notation for IP range (e.g., "192.168.1.0/24") + required: + - cidr + example: + cidr: "192.168.1.0/24" + field: "ip_address" + + # Phrase Query (internal use) + PhraseQuery: + allOf: + - type: object + properties: + terms: + type: array + items: + type: string + description: Array of terms in phrase order + required: + - terms + example: + terms: ["exact", "phrase", "match"] + field: "content" + + MultiPhraseQuery: + allOf: + - type: object + properties: + terms: + type: array + items: + type: array + items: + type: string + description: Array of term arrays for multi-phrase matching + required: + - terms + example: + terms: + - ["exact", "precise"] + - ["phrase"] + - ["match", "search"] + + examples: + # Basic Query Examples + simple_term: + summary: Simple Term Query + value: + term: "search" + field: "title" + boost: 2.0 + + simple_match: + summary: Simple Match Query + value: + match: "full text search" + field: "content" + operator: "and" + + fuzzy_search: + summary: Fuzzy Search + value: + term: "search" + field: "title" + fuzziness: "auto" + prefix_length: 2 + + # Boolean Query Examples + complex_boolean: + summary: Complex Boolean Query + value: + must: + - term: "important" + field: "status" + - match: "urgent" + field: "title" + should: + - match: "critical" + field: "content" + - match: "priority" + field: "tags" + must_not: + - term: "archived" + field: "status" + filter: + - term: "published" + field: "status" + boost: 1.5 + + # Range Query Examples + price_range: + summary: Price Range Query + value: + min: 10.0 + max: 100.0 + inclusive_min: true + inclusive_max: false + field: "price" + + date_range: + summary: Date Range Query + value: + start: "2023-01-01T00:00:00Z" + end: "2023-12-31T23:59:59Z" + inclusive_start: true + inclusive_end: true + field: "created_at" + + # Geographic Query Examples + geo_distance: + summary: Geographic Distance Query + value: + location: [-122.4194, 37.7749] + distance: "10km" + field: "location" + + geo_bounding_box: + summary: Geographic Bounding Box Query + value: + top_left: [-122.5, 37.8] + bottom_right: [-122.3, 37.7] + field: "location" + + # Vector Query Examples + knn_search: + summary: K-Nearest Neighbors Query + value: + field: "embedding" + vector: [0.1, 0.2, 0.3, 0.4, 0.5] + k: 10 + boost: 1.0 + + # Query String Examples + query_string: + summary: Query String Query + value: + query: "title:search AND (content:full text OR content:search) AND -status:archived" + + responses: + ValidationSuccess: + description: Query validation successful + content: + application/json: + schema: + type: object + properties: + valid: + type: boolean + example: true + query_type: + type: string + example: "term" + message: + type: string + example: "Query is valid" + + ValidationError: + description: Query validation failed + content: + application/json: + schema: + type: object + properties: + valid: + type: boolean + example: false + error: + type: string + example: "unknown query type" + details: + type: string + example: "No recognized query fields found" + +externalDocs: + description: Bleve Documentation + url: https://github.com/blevesearch/bleve diff --git a/docs/score_fusion.md b/docs/score_fusion.md new file mode 100644 index 0000000..61efc1a --- /dev/null +++ b/docs/score_fusion.md @@ -0,0 +1,250 @@ +# Score Fusion for Hybrid Search + +Bleve supports **hybrid search** that combines full-text search (FTS) with vector (kNN) search to leverage the strengths of both approaches: + +* **Full-text search** excels at exact keyword matching, filtering, and structured queries +* **Vector search** captures semantic similarity and handles synonyms and paraphrasing naturally + +With *v2.5.4* onwards - when using hybrid search, you can choose different **score fusion strategies** to combine results from both search methods. This document describes the available fusion strategies and how to use them. + +## Fusion Strategies + +### Additive Score Fusion (Default) + +By default, Bleve combines FTS and kNN scores using a simple weighted addition. See the [Vector Search documentation](vectors.md#querying) for details on the default hybrid search behavior and examples. + +While this approach works well with proper boost tuning, it can be sensitive to different score scales and distributions. The fusion strategies below (RRF and RSF) provide more robust alternatives that handle score normalization automatically. + +### Reciprocal Rank Fusion (RRF) + +Reciprocal Rank Fusion is a **rank-based** algorithm that combines results based on their position in each result list, rather than their raw scores. This makes it robust to different score scales and distributions. + +**Algorithm:** + +For each document appearing in FTS or kNN results, the RRF score is calculated as: + +```math +RRF\_score = w_{\text{fts}} \cdot \frac{1}{k + \text{rank}_{\text{fts}}} + \sum_{i=1}^{n} w_{\text{knn}_i} \cdot \frac{1}{k + \text{rank}_{\text{knn}_i}} +``` + +Where: + +* $\text{rank}_{\text{fts}}$: 1-indexed rank of the document in the FTS result list (or 0 if not present) +* $\text{rank}_{\text{knn}_i}$: 1-indexed rank of the document in the i-th kNN result list (or 0 if not present) +* $k$: rank constant (default: 60) that dampens the impact of rank differences +* $w_{\text{fts}}$: weight from the FTS query boost value +* $w_{\text{knn}_i}$: weight from the i-th kNN query boost value +* $\sum_{i=1}^{n}$: summation over all kNN queries (you can add multiple kNN queries) + +**Advantages:** + +* Distribution-agnostic - no need for score normalization +* Works out of the box with minimal tuning +* Prioritizes documents appearing in both result lists +* Robust to outliers since only ranks matter + +**Disadvantages:** + +* Ignores score magnitude (loses some information) +* May be sensitive to imbalanced result list sizes + +**Usage:** + +```go +// Create a hybrid search with RRF fusion +searchRequest := bleve.NewSearchRequest(bleve.NewMatchQuery("dark chocolate")) +searchRequest.Score = bleve.ScoreRRF // Alternatively, set to "rrf" + +// Add first kNN component +searchRequest.AddKNN( + "embedding", // Vector field + []float32{0.1, 0.2, 0.3, 0.4}, // Query vector + 30, // k neighbors + 1.0, // kNN weight (boost) +) + +// Add second kNN component (optional - you can add multiple) +searchRequest.AddKNN( + "image_embedding", // Different vector field + []float32{0.5, 0.3, 0.1, 0.8}, // Query vector + 20, // k neighbors + 0.5, // kNN weight (boost) +) + +// Optional: Configure RRF parameters +params := bleve.RequestParams{ + ScoreRankConstant: 60, // Rank constant (default: 60) + ScoreWindowSize: 150 // Window size (default: size) +} +searchRequest.AddParams(params) + +searchResult, err := index.Search(searchRequest) +``` + +### Relative Score Fusion (RSF) + +Relative Score Fusion is a **score-based** strategy that normalizes scores from both modalities into a common [0, 1] range using min-max normalization before combining them. + +**Algorithm:** + +1. **Min-max normalize** each result set independently: + + ```math + \text{normalized\_score} = \frac{\text{score} - \text{min\_score}}{\text{max\_score} - \text{min\_score}} + ``` + +2. **Combine** normalized scores using weighted addition: + + ```math + RSF\_score = w_{\text{fts}} \cdot \text{normalized\_score\_fts} + \sum_{i=1}^{n} w_{\text{knn}_i} \cdot \text{normalized\_score\_knn}_i + ``` + +Where: + +* $w_{\text{fts}}$: weight from the FTS query boost value +* $w_{\text{knn}_i}$: weight from the i-th kNN query boost value +* $\sum_{i=1}^{n}$: summation over all kNN queries (you can add multiple kNN queries) + +**Advantages:** + +* Score-aware - retains relevance magnitude information +* Resolves incompatible score ranges +* Easy to understand + +**Disadvantages:** + +* Sensitive to outliers - a single extreme score can skew normalization +* Doesn't account for the shape or distribution of scores + +**Usage:** + +```go +// Create a hybrid search with RSF fusion +searchRequest := bleve.NewSearchRequest(bleve.NewMatchQuery("machine learning")) +searchRequest.Score = bleve.ScoreRSF // Or set to "rsf" + +// Add first kNN component +searchRequest.AddKNN( + "content_vector", // Vector field + []float32{0.5, 0.3, 0.1, 0.8}, // Query vector + 20, // k neighbors + 1.0, // kNN weight (boost) +) + +// Add second kNN component (optional - you can add multiple) +searchRequest.AddKNN( + "title_vector", // Different vector field + []float32{0.2, 0.7, 0.4, 0.1}, // Query vector + 15, // k neighbors + 0.8, // kNN weight (boost) +) + +// Optional: Configure RSF parameters +params := bleve.RequestParams{ + ScoreWindowSize: 150 // Window size (default: size) +} +searchRequest.AddParams(params) + +searchResult, err := index.Search(searchRequest) +``` + +## Parameters + +### Score + +The `Score` field in your search request specifies which fusion strategy to use: + +* **`ScoreRRF ("rrf")`**: Reciprocal Rank Fusion +* **`ScoreRSF ("rsf")`**: Relative Score Fusion +* **Omitted or empty**: Default additive fusion with scores returned + +### Params + +The `Params` object contains additional parameters for score fusion: + +#### Score Window Size + +`ScoreWindowSize` is the maximum number of results to consider from each result list for fusion. + +* **Default**: Same as `Size` parameter +* **Minimum**: Must be ≥ `Size` and ≥ 1 +* **Purpose**: Controls the tradeoff between relevance and performance + +A larger window size increases the chance of finding relevant results but requires more computation. For pagination to work consistently, ensure: + +```text +From + Size <= ScoreWindowSize +``` + +**Example:** + +```json +{ + "score": "rrf", + "params": { + "score_window_size": 150 + }, + "size": 10, + "from": 0 +} +``` + +With window size set to 150, you can paginate through up to 150 results. If you try to access results beyond this (e.g., `from=160`), you'll get an empty result set. + +#### Score Rank Constant + +> *Only applicable for RRF* + +`ScoreRankConstant` controls how much the rank position affects the reciprocal rank score. + +* **Default**: 60 +* **Range**: Any positive integer +* **Effect**: Higher values dampen the impact of rank differences + +**Example:** + +```json +{ + "score": "rrf", + "params": { + "score_rank_constant": 60 + } +} +``` + +## Weighting Queries + +The boost value in your query components controls their relative importance in hybrid search: + +```go +// FTS query with boost 2.0 +query := bleve.NewMatchQuery("search term") +query.SetBoost(2.0) + +searchRequest := bleve.NewSearchRequest(query) + +// kNN query with boost 1.0 +searchRequest.AddKNN("vec", queryVector, 10, 1.0) +``` + +For RRF and RSF, weights determine the **relative importance** of each component's contribution, rather than scaling raw scores. + +**Example:** If `fts_boost = 2.0` and `knn_boost = 1.0`, the FTS contribution is twice as important as the kNN contribution in the final ranking in RRF or RSF. + +## Restrictions + +When using score fusion (`Score` set to `"rrf"` or `"rsf"`), certain features are not supported: + +* **SearchAfter/SearchBefore**: Not compatible with score fusion. For pagination, use `From` and `Size` only. +* **Sort**: Only descending score sort (`-_score`) or default sorting is allowed +* **Faceting**: Only documents included in the FTS result list are considered. Documents that appear exclusively in the kNN result list are ignored during faceting. + +## Choosing a Fusion Strategy + +| Use Case | Recommended Strategy | +| -------- | -------------------- | +| Different score scales (e.g., TF-IDF + L2 distance) | **RRF/RSF** | +| Minimal tuning, out-of-the-box performance | **RRF** | +| Want to preserve score magnitude importance | **RSF** | +| Have well-tuned boost values already | **Additive (default)** | +| Score distributions have extreme outliers | **RRF** | diff --git a/docs/scoring.md b/docs/scoring.md new file mode 100644 index 0000000..480f510 --- /dev/null +++ b/docs/scoring.md @@ -0,0 +1,90 @@ +# Scoring models for document hits + +* Search is performed on a collection of fields using compound queries such as conjunction/disjunction/boolean, etc. However, the scoring itself is done independently for each field and then aggregated to get the final score for a document hit. +* Default scoring scheme for document hits involving text hits: `tf-idf`. +* Nearest-neighbor/vector hits scoring depends on chosen `knn distance` metric, highlighted [here](https://github.com/blevesearch/bleve/blob/master/docs/vectors.md#supported). +* Hybrid search scoring will combine `tf-idf` scores with `knn distance` numbers. +* *v2.5.0* (and after) will come with support for `bm25` scoring for exact searches. + +## BM25 + +When it comes to scoring a document hit for a specific field, BM25 scoring mechanism requires the following stats: + +* fieldLen - The number of analyzed terms in the current document's field. +* avgFieldLen - The average number of analyzed terms in the field across all the documents. +* docTotal - The total number of documents in the index. +* docTerm - The total number of documents containing the query term within the index. + +The scoring formula followed in BM25 is + +```math +\sum_{i}^n IDF(q_i) {{f(q_i,D) * (k1 + 1)}\over{f(q_i,D) + k1 * (1-b+b*{{fieldLen}\over{avgFieldLen}})}} +``` + +$IDF(q_i)$ here refers to Inverse Document Frequency, which talks about how rare (and hence rich in information) is a particular query term $`q_i`$ across all the documents in the index, which is calculated as + +```math + \ln(1 + {{docTotal - docTerm + 0.5}\over{docTerm + 0.5}}) +``` + +Coming back to the BM25 scoring, $f(q_i,D)$ refers to the frequency of the query term in document $D$. The entire equation has certain multipliers + +* $k1$ - helps in controlling the saturation of the score with respect to query term in a document. Basically if the query term's frequency is too high, the score value gets saturated and doesn't increase beyond a certain point. +* $b$ - controls the extent to which the $fieldLen$ normalizes the term's frequency. + +### How to enable and use BM25 + +Bleve v2.5.0 updated the `indexMapping` construct with the concept of `scoringModel`. This is a global (meaning applicable to all fields) setting which drives which scoring algorithm to apply while scoring the document hits. Supported scoring models are defined [here](https://github.com/blevesearch/bleve_index_api/blob/f54d76f0a71a838837159aa44ced0404bb6ec25f/indexing_options.go#L27) + +For instance, while defining the index mapping for the data model that's been decided by the user, following snippet can be referred to enable BM25 + +```go +indexMapping := bleve.NewIndexMapping() +indexMapping.TypeField = "type" +indexMapping.DefaultAnalyzer = "en" +indexMapping.ScoringModel = "bm25" +``` + +During search time there's no explicit change involved, unless the user wants to perform a global scoring. + +### Global Scoring + +Let's say that the user has a dataset which is quite large (let's say 3 million) and to have good throughput, they create 3 shards (with the same index mapping) for the "index". Each of these shards can be `bleve.Index` type and while performing a search over the entire dataset, a `bleve.IndexAlias` can be created over which a search can be performed. This parallelizes things pretty well, both on the indexing path and the search path. + +The concept of global scoring is applicable when the index is "sharded" (similar to above situation). This is because each index has data which is disjoint, and thereby while performing the scoring on document hits on each of them, the value of stats is not complete at a global level, since we're doing a search over the entire dataset using the `bleve.IndexAlias`. For eg: `docTotal` value while scoring the document hits would be 1 million which is incorrect at a global level. + +So in order to keep the scoring roughly same across varying count of the number of shards involved, we provide a mechanism to enable "global scoring". In this type of search, an initial roundtrip is performed to gather and aggregate the stats necessary for the scoring mechanism and in the second phase, the actual search is performed. So naturally this comes at a cost of latency. As a reference here's how the user can go about with it + +```go +multiPartIndex := bleve.NewIndexAlias(shard1, shard2) +// set the alias with the same index mapping which both the shards use. +err = multiPartIndex.SetIndexMapping(indexMapping) +if err != nil { + return err +} + +ctx := context.Background() +ctx = context.WithValue(ctx, search.SearchTypeKey, search.GlobalScoring) + +res, err := multiPartIndex.SearchInContext(ctx, searchRequest) +``` + +A note here is that, this would only matter if the relative order of the document hits vary quite a bit (vs single shard case). This would be possible when the shard count increases quite a bit, in low doc count situations or if there is a heavy skew in the data distribution amongst the shards for some reason. Ideally the shards are created when the data is quite large and each of them index the same amount of data - in which case the scores won't fluctuate much to affect the relative hit order and the user can choose to avoid the global scoring mechanism altogether. + +## TF-IDF + +TF-IDF is the default scoring mechanism involved (for backward compatibility reasons) and requires no change from the user at index or search time to avail it. + +The scoring formula involved is + +```math + \sum_{i}^n f(q_i, D) * {{1}\over{\sqrt{fieldLen}}} * IDF(q_i) +``` + +where $IDF(q_i)$ is + +```math + 1 + {{docTotal}\over{1 + docTerm}} +``` + +Note: TF-IDF formula doesn't accommodate logic for score saturation due to term frequency or fieldLen. So, it's recommended to use BM25 scoring by explicitly setting it in the index mapping. diff --git a/docs/search_autocomplete.md b/docs/search_autocomplete.md new file mode 100644 index 0000000..225323a --- /dev/null +++ b/docs/search_autocomplete.md @@ -0,0 +1,358 @@ +# Edge N-gram Autocomplete in Bleve + +Search autocomplete is a feature which we see in search boxes when suggestions appear while we type. +So when we type `jav`, we see suggestions like: `java` `javascript` `javascript programming` etc. +This is helpful because it saves users time in finding what they are looking for. + +![Alt Text](/docs/images/search_autocomplete.png "search autocomplete suggestion") + +## 2. How Does It Work? + +Autocomplete generally works in three steps: + +1. **Index Time**: Breaking text into searchable pieces (tokens) +2. **Query Time**: Matching user input against indexed tokens +3. **Results**: Returning relevant suggestions quickly + +But before we jump into the flow, let's understand different methods to achieve this and why edge n-grams are the most efficient approach. + +## 3. Different Tokenization Methods + +There are several tokenization approaches, each with its own strengths and weaknesses: + +### 3.1 Single Token Tokenizer + +```go +// analysis/tokenizer/single/single.go +func (t *SingleTokenTokenizer) Tokenize(input []byte) analysis.TokenStream { + return analysis.TokenStream{ + &analysis.Token{ + Term: input, // Here entire input as one token + Position: 1, + Start: 0, + End: len(input), + Type: analysis.AlphaNumeric, + }, + } +} +``` + +**How it works**: Treats the entire input as a single token. + +**Example**: "JavaScript Programming" → [`"JavaScript Programming"`] + +**Pros**: + +- Simple and fast +- Perfect for exact phrase matching +- Minimal index size + +**Cons**: + +- No autocomplete support (can't match partial text) +- Not flexible for search + +**Use case**: Keyword fields, IDs, exact phrase matching + +### 3.2 Whitespace Token Tokenizer + +```go +// analysis/tokenizer/whitespace/whitespace.go +func TokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { + return character.NewCharacterTokenizer(notSpace), nil +} + +func notSpace(r rune) bool { + return !unicode.IsSpace(r) // Split on whitespace +} +``` + +**How it works**: Splits text on whitespace characters. + +**Example**: "JavaScript Programming" → [`"JavaScript"`, `"Programming"`] + +**Pros**: + +- Simple word-based tokenization +- Works well for basic prefix search + +**Cons**: + +- Only matches from word beginnings +- No support for partial word matching +- Limited autocomplete capabilities + +**Use case**: Basic search, word-level indexing + +### 3.3 N-gram Method + +```go +// analysis/token/ngram/ngram.go +func (s *NgramFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + rv := make(analysis.TokenStream, 0, len(input)) + + for _, token := range input { + runeCount := utf8.RuneCount(token.Term) + runes := bytes.Runes(token.Term) + + // ..generate all possible n-grams + } + return rv +} +``` + +**How it works**: Creates ALL possible substrings of specified lengths. + +**Example**: "java" with 2-3 grams → [`"ja"`, `"av"`, `"va"`, `"jav"`, `"ava"`] + +**Pros**: + +- Supports partial matching anywhere in the text +- Coverage of all possible substrings + +**Cons**: + +- **MASSIVE index size** (exponential growth) +- Brings more noise as irrelevant matches (e.g., "av" matching "java") +- Poor performance for autocomplete +- High memory usage + +**Use case**: Full-text substring search (not ideal for autocomplete) + +### 3.4 Edge N-gram Method (preferred for autocomplete) + +```go +// analysis/token/edgengram/edgengram.go +func (s *EdgeNgramFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + rv := make(analysis.TokenStream, 0, len(input)) + + for _, token := range input { + runeCount := utf8.RuneCount(token.Term) + runes := bytes.Runes(token.Term) + // ..builds tokens based on either end, specified in the input + } + return rv +} +``` + +**How it works**: Creates substrings only from the beginning (or end) of each word. + +**Example**: "javascript" with front edge n-grams (1-5) → [`"j"`, `"ja"`, `"jav"`, `"java"`, `"javas"`] + +**Pros**: + +- Perfect for autocomplete (matches prefixes naturally) +- Efficient index size (linear growth vs exponential) +- Fast queries (direct term matching, no complex processing) +- Intuitive results (matches what users expect) +- Highly scalable for large datasets + +**Cons**: + +- Only supports prefix matching (but that's preferred for autocomplete!) +- Slightly larger index than basic tokenization + +**Use case**: Search autocomplete, prefix-based search + +## 4. Why Edge N-grams Are Most Efficient for Autocomplete + +Let's see what happens when a user types "java" with edge_ngram tokenizer: + +```text +Index contains: ["j", "ja", "jav", "java", "javas", "javasc", "javascr", ...] +User types: "java" +Query: ExactTermQuery("java") +Process: Direct hash table lookup for term "java" +Result: Instant match, then retrieve documents containing this term +``` + +**Key advantages of Edge N-gram approach:** + +1. **O(1) lookup**: Direct term matching +2. **No query-time processing**: Terms are pre-computed at index time +3. **Better caching**: Exact term queries cache better than prefix queries +4. **Consistent performance**: Query time doesn't increase with index size + +## 5. Step-by-Step Implementation: Building Search Autocomplete + +Now let's see how to implement this in practice using our exact configuration. We'll build a complete autocomplete system step by step. + +### Step 1: Create Custom Edge N-gram Token Filter + +First, we need to create a custom token filter for edge n-grams. Here's how it looks in our configuration: + +```go +// Create a new index mapping +indexMapping := mapping.NewIndexMapping() + +// 1. Define the edgeGram token filter +edgeGramFilter := map[string]interface{}{ + "type": edgengram.Name, + "min": 2.0, + "max": 4.0, + "back": false, +} + +// Register the token filter +if err := indexMapping.AddCustomTokenFilter("Engram", edgeGramFilter); err != nil { + log.Fatal(err) +} +``` + +**What each setting does:** + +- `"type": "edge_ngram"` - Tells Bleve to use the edge n-gram filter +- `"min": 2` - Start creating tokens from 2 characters ("ja", "sc", etc.) +- `"max": 4` - Stop at 4 characters ("java", "scri", etc.) +- `"back": false` - Create tokens from the front (beginning) of words + +### Step 2: Create Custom Analyzer + +Next, we create an analyzer that uses our custom token filter along with other helpful filters: + +```go +// 2. Define a custom analyzer that uses it +customAnalyzer := map[string]interface{}{ + "type": custom.Name, + "tokenizer": "unicode", + "char_filters": []string{ + zerowidthnonjoiner.Name, + }, + "token_filters": []string{ + "Engram", // our custom edge_ngram filter + "to_lower", + "stop_en", + }, +} + +if err := indexMapping.AddCustomAnalyzer("edgeGramAnalyzer", customAnalyzer); err != nil { + log.Fatal(err) +} +``` + +**The pipeline works like this:** + +1. **Input Text**: "Schaumbergfest Event" + +2. **Tokenizer** (`unicode`): Splits into words + + ```text + ["Schaumbergfest", "Event"] + ``` + +3. **Character Filter** (`zero_width_spaces`): Removes invisible characters + + ```text + ["Schaumbergfest", "Event"] (cleaned) + ``` + +4. **Token Filter 1** (`Engram`): Creates edge n-grams (2-4 chars) + + ```text + ["Sc", "Sch", "Scha", "Ev", "Eve", "Even"] + ``` + +5. **Token Filter 2** (`to_lower`): Makes everything lowercase + + ```text + ["sc", "sch", "scha", "ev", "eve", "even"] + ``` + +6. **Token Filter 3** (`stop_en`): Removes common words (none in this case) + + ```text + ["sc", "sch", "scha", "ev", "eve", "even"] (final tokens) + ``` + +### Step 3: Configure Field Mapping + +Now we tell Bleve which fields to apply our autocomplete analyzer to: + +```go + // 3. Assign analyzer to a field mapping + fieldMapping := mapping.NewTextFieldMapping() + fieldMapping.Analyzer = "edgeGramAnalyzer" + + indexMapping.DefaultMapping.AddFieldMappingsAt("title", fieldMapping) + + indexPath := "example.bleve" + index, err := bleve.New(indexPath, indexMapping) + if err != nil { + log.Fatal(err) + } +``` + +### Step 4: How It Works in Real Search + +When someone searches for "sc", here's what happens: + +**Index contains these tokens:** + +```text +"sc" → [document1: "Schaumbergfest", document2: "Script", ...] +"sch" → [document1: "Schaumbergfest", ...] +"scha" → [document1: "Schaumbergfest", ...] +``` + +**User types "sc":** + +1. Query: `name:sc` +2. Bleve looks up exact term "sc" in the index +3. Finds document with "Schaumbergfest" and "Script" +4. Returns suggestion instantly + +```go + +type Document struct { + ID string `json:"id"` + Title string `json:"title"` +} +// 4. Index Documents +documents := []Document{ + { + ID: "doc1", + Title: "Schaumbergfest", + }, + { + ID: "doc2", + Title: "Script", + }, +} + +batch := index.NewBatch() +for _, doc := range documents { + batch.Index(doc.ID, doc) +} +if err := index.Batch(batch); err != nil { + log.Fatal(err) +} + +// 5. Search the created index +query := bleve.NewMatchQuery("sc") +query.SetField("title") +searchRequest := bleve.NewSearchRequest(query) +searchRequest.Explain = true +searchRequest.Fields = []string{"title"} +searchResult, err := index.Search(searchRequest) +if err != nil { + log.Fatal(err) +} +fmt.Println(searchResult) +``` + +Output: + +```bash + +$ go run main.go + +2 matches, showing 1 through 2, took 189.041µs + 1. doc2 (0.343255) + title + Script + 2. doc1 (0.343255) + title + Schaumbergfest +``` + +Note: To run code, enclose code starting from Step 1 in func main. diff --git a/docs/sort_facet.md b/docs/sort_facet.md new file mode 100644 index 0000000..ee2ba29 --- /dev/null +++ b/docs/sort_facet.md @@ -0,0 +1,785 @@ +

Purpose of Docvalues

+ +

Background

+ +

What are docValues? In the index mapping, there is an option to enable or disable docValues for a specific field mapping. However, what does it actually mean to activate or deactivate docValues, and how does it impact the end user? This document aims to address these questions.

+
+	"default_mapping": {
+		"dynamic": true,
+		"enabled": true,
+		"properties": {
+			"loremIpsum": {
+			"enabled": true,
+			"dynamic": false,
+			"fields": [
+				{
+					"name": "loremIpsum",
+					"type": "text",
+					"store": false,
+					"index": true,
+					"include_term_vectors": false,
+					"include_in_all": false,
+					"docvalues": true
+				}
+			]
+		}
+	}
+
+

Enabling docValues will always result in an increase in the size of your Bleve index, leading to a corresponding increase in disk usage. But what advantages can you expect in return? This document also quantitatively assesses this trade-off with a test case.

+ +

In a more general sense, we recommend enabling docValues on a field mapping if you anticipate queries that involve sorting and/or facet operations on that field. It's important to note, though, that sorting and faceting will work irrespective of whether docValues are enabled or not. This may lead you to wonder if there's any real benefit to enabling docValues since you're allocating extra disk space without an apparent return. The real advantage, however, becomes evident in enhanced query response times and reduced memory consumption during active usage. By accepting a minor increase in the disk space used by your Full-Text Search (FTS) index, you can anticipate better performance in handling search requests that involve sorting and faceting.

+ +

Usage

+ +

The initial use of docValues comes into play when sorting is involved. In the search request JSON, there is a field named "sort." This optional "sort" field can have a slice of JSON objects as its value. Each JSON object must belong to one of the following types: +

    +
  • SortDocID
    • +
  • SortScore (which is the default if none is specified)
    • +
  • SortGeoDistance
    • +
  • SortField
    • +
+

+

DocValues are relevant only when any of the JSON objects in the "sort" field are of type SortGeoDistance or SortField. This means that if you expect queries on a field F, where the queries either do not specify a value for the "sort" field or provide a JSON object of type SortDocID or SortScore, enabling docValues will not improve sorting operations, and as a result, query latency will remain unchanged. It's worth noting that the default sorting object, SortScore, does not require docValues to be enabled for any of the field mappings. Therefore, a search request without a sorting operation will not utilize docValues at all.

+
+ + + + + + + + + + + + + + + + + + + + + + +
No Sort ObjectsSortDocIDSortScoreSortFieldSortGeoDistance
+ 
+{
+  "explain": true,
+  "fields": [
+    "*"
+  ],
+  "highlight": {},
+  "query": {
+    "match": "lorem ipsum",
+    "field":"dolor"
+  },
+  "size": 10,
+  "from": 0
+}
+
+ 		+ 
+{
+  "explain": true,
+  "fields": [
+    "*"
+  ],
+  "highlight": {},
+  "query": {
+    "match": "lorem ipsum",
+    "field":"sit_amet"
+  },
+  "sort":[
+    {
+     "by":"id",
+     "desc":true
+    }
+    ],
+  "size": 10,
+  "from": 0
+}
+
+ 		+ 
+{
+  "explain": true,
+  "fields": [
+    "*"
+  ],
+  "highlight": {},
+  "query": {
+    "match": "lorem ipsum",
+    "field":"sit_amet"
+  },
+  "sort":[
+    {
+     "by":"score",
+    }
+    ],
+  "size": 10,
+  "from": 0
+}
+
+ 		+ 
+{
+  "explain": true,
+  "fields": [
+    "*"
+  ],
+  "highlight": {},
+  "query": {
+    "match": "lorem ipsum",
+    "field":"sit_amet"
+  },
+  "sort":[
+    {
+     "by":"field",
+     "field":"dolor",
+     "type":"auto",
+     "mode":"min",
+     "missing":"last"
+    }
+    ],
+  "size": 10,
+  "from": 0
+}
+
+ 		+ 
+{
+  "explain": true,
+  "fields": [
+    "*"
+  ],
+  "highlight": {},
+  "query": {
+    "match": "lorem ipsum",
+    "field": "dolor"
+  },
+  "sort": [
+    {
+      "by": "geo_distance",
+      "field": "sit_amet",
+      "location": [
+        123.223,
+        34.33
+      ],
+      "unit": "km"
+    }
+  ],
+  "size": 10,
+  "from": 0
+}
+
+
No DocValues usedNo DocValues usedNo DocValues usedDocValues used for field "dolor". Field Mapping for "dolor" may enable docValues.DocValues used for field "dolor". Field Mapping for "dolor" may enable docValues.
+
+

Now, let's consider faceting. The search request object also includes another field called "facets," where you can specify a collection of facet requests, with each request being associated with a unique name. Each of these facet requests can fall into one of three types: +

    +
  • Date range
    • +
  • Numeric range
    • +
  • Term facet
    • +
+Enabling docValues for the fields associated with such facet requests might provide benefits in this context.

+
+ + + + + + + + + + + + + + + + + +
No Facet RequestDate Range FacetNumeric Range FacetTerm Facet
+ 
+{
+  "explain": true,
+  "fields": [
+    "*"
+  ],
+  "highlight": {},
+  "query": {
+    "match": "lorem ipsum",
+    "field": "dolor"
+  },
+  "size": 10,
+  "from": 0
+}
+
+ 		+ 
+{
+  "explain": true,
+  "fields": [
+    "*"
+  ],
+  "highlight": {},
+  "query": {
+    "match": "lorem ipsum",
+    "field": "sit_amet"
+  },
+  "facet": {
+    "facetA": {
+      "size": 1,
+      "field": "dolor",
+      "date_ranges": [
+        {
+          "name": "lorem",
+          "start": "20/August/2001",
+          "end": "22/August/2002",
+          "datetime_parser": "custDT"
+        }
+      ]
+    }
+  },
+  "size": 10,
+  "from": 0
+}
+
+ 		+ 
+{
+  "explain": true,
+  "fields": [
+    "*"
+  ],
+  "highlight": {},
+  "query": {
+    "match": "lorem ipsum",
+    "field": "sit_amet"
+  },
+  "facet": {
+    "facetA": {
+      "size": 1,
+      "field": "dolor",
+      "numeric_ranges":[
+          { 
+            "name":"lorem",
+            "min":22,
+            "max":34
+          }
+        ]
+    }
+  },
+  "size": 10,
+  "from": 0
+}
+
+ 		+ 
+{
+  "explain": true,
+  "fields": [
+    "*"
+  ],
+  "highlight": {},
+  "query": {
+    "match": "lorem ipsum",
+    "field": "sit_amet"
+  },
+  "facet": {
+    "facetA": {
+      "size": 1,
+      "field": "dolor"
+    }
+  },
+  "size": 10,
+  "from": 0
+}
+
+
No DocValues usedDocValues used for field "dolor". Field Mapping for "dolor" may enable docValues.
+
+ +

In summary, when a search request is received by the Bleve index, it extracts all the fields from the sort objects and facet objects. To potentially benefit from docValues, you should consider enabling docValues for the fields mentioned in SortField and SortGeoDistance sort objects, as well as the fields associated with all the facet objects. By doing so, you can optimize sorting and faceting operations in your search queries.

+ +
+ + + + + + + + + + + + + +
Combo ACombo B
+ 
+{
+  "explain": true,
+  "fields": [
+    "*"
+  ],
+  "highlight": {},
+  "query": {
+    "match": "lorem ipsum",
+    "field": "sit_amet"
+  },
+  "facet": {
+    "facetA": {
+      "size": 1,
+      "field": "dolor",
+      "date_ranges": [
+        {
+          "name": "lorem",
+          "start": "20/August/2001",
+          "end": "22/August/2002",
+          "datetime_parser": "custDT"
+        }
+      ]
+    }
+  },
+  "sort":[
+    {
+     "by":"field",
+     "field":"lorem",
+     "type":"auto",
+     "mode":"min",
+     "missing":"last"
+    }
+    ],
+  "size": 10,
+  "from": 0
+}
+
+ 		+ 
+{
+  "explain": true,
+  "fields": [
+    "*"
+  ],
+  "highlight": {},
+  "query": {
+    "match": "lorem ipsum",
+    "field": "sit_amet"
+  },
+  "facet": {
+    "facetA": {
+      "size": 1,
+      "field": "dolor",
+      "numeric_ranges":[
+          { 
+            "name":"lorem",
+            "min":22,
+            "max":34
+          }
+        ]
+    }
+  },
+  "sort": [
+    {
+      "by": "geo_distance",
+      "field": "ipsum",
+      "location": [
+        123.223,
+        34.33
+      ],
+      "unit": "km"
+    }
+  ],
+  "size": 10,
+  "from": 0
+}
+
+
DocValues used for fields "dolor" and "lorem". Field Mapping for "dolor" and "lorem" may enable docValues.DocValues used for fields "dolor" and "ipsum". Field Mapping for "dolor" and "ipsum" may enable docValues.
+
+ +

Empirical Analysis

+ +

To evaluate our hypothesis, I've set up a sample dataset on my personal computer and I've created two Bleve indexes: one with docvalues enabled for three fields (dummyDate, dummyNumber, and dummyTerm), and another where I've disabled docValues for the same three fields. These field mappings were incorporated into the Default Mapping. It's important to mention that for both indexes, DocValues for dynamic fields were enabled, as the default mapping is dynamic.

+ +

The values for dummyDate and dummyNumber were configured to increase monotonically, with dummyDate representing a date value and `dummyNumber` representing a numeric value. This setup was intentional to ensure that facet aggregation would consistently result in cache hits and misses, providing a useful testing scenario.

+ +
+ + + + + + + + + + + + + +
Index AIndex B
+ 
+   "default_mapping": {
+    "dynamic": true,
+    "enabled": true,
+    "properties": {
+     "dummyNumber": {
+      "enabled": true,
+      "dynamic": false,
+      "fields": [
+       {
+        "name": "dummyNumber",
+        "type": "text",
+        "store": false,
+        "index": true,
+        "include_term_vectors": false,
+        "include_in_all": false,
+        "docvalues": true
+       }
+      ]
+     },
+     "dummyTerm": {
+      "enabled": true,
+      "dynamic": false,
+      "fields": [
+       {
+        "name": "dummyTerm",
+        "type": "text",
+        "store": false,
+        "index": true,
+        "include_term_vectors": false,
+        "include_in_all": false,
+        "docvalues": true
+       }
+      ]
+     },
+     "dummyDate": {
+      "enabled": true,
+      "dynamic": false,
+      "fields": [
+       {
+        "name": "dummyDate",
+        "type": "text",
+        "store": false,
+        "index": true,
+        "include_term_vectors": false,
+        "include_in_all": false,
+        "docvalues": true
+       }
+      ]
+     }
+    }
+   }
+
+ 		+ 
+   "default_mapping": {
+    "dynamic": true,
+    "enabled": true,
+    "properties": {
+     "dummyNumber": {
+      "enabled": true,
+      "dynamic": false,
+      "fields": [
+       {
+        "name": "dummyNumber",
+        "type": "text",
+        "store": false,
+        "index": true,
+        "include_term_vectors": false,
+        "include_in_all": false,
+        "docvalues": false
+       }
+      ]
+     },
+     "dummyTerm": {
+      "enabled": true,
+      "dynamic": false,
+      "fields": [
+       {
+        "name": "dummyTerm",
+        "type": "text",
+        "store": false,
+        "index": true,
+        "include_term_vectors": false,
+        "include_in_all": false,
+        "docvalues": false
+       }
+      ]
+     },
+     "dummyDate": {
+      "enabled": true,
+      "dynamic": false,
+      "fields": [
+       {
+        "name": "dummyDate",
+        "type": "text",
+        "store": false,
+        "index": true,
+        "include_term_vectors": false,
+        "include_in_all": false,
+        "docvalues": false
+       }
+      ]
+     }
+    }
+   }
+
+
Docvalues enabled across all three field mappingsDocvalues disabled across all three field mappings
+
+ +Document Format used for the test scenario: + +
+ + + + + + + + + + + + +
Document 1Document 2... Document iDocument 5000
+ 
+{
+	"dummyTerm":"Term",
+	"dummyDate":"2000-01-01T00:00:00",
+	"dummyNumber":1
+}
+
+ 		+ 
+{
+	"dummyTerm":"Term",
+	"dummyDate":"2000-01-01T01:00:00",
+	"dummyNumber":2
+}
+
+ 		+ 
+{
+	"dummyTerm":"Term",
+	"dummyDate":"2000-01-01T01:00:00"+(i hours),
+	"dummyNumber":i
+}
+
+ 		+ 
+{
+	"dummyTerm":"Term",
+	"dummyDate":"2000-01-01T01:00:00 + (5000 hours)",
+	"dummyNumber":5000
+}
+
+
+
+ +

Now I ran the following set of search requests across both the indexes, while increasing the number of documents indexed from 2000 to 4000.

+ +
+ + + + + + + + + + + + +
Request 1Request 2... Request iRequest 1000
+ 
+{
+  "explain": true,
+  "fields": [
+    "*"
+  ],
+  "highlight": {},
+  "query": {
+    "match": "term",
+    "field":"dummyTerm"
+  },
+  "facets":{
+    "myDate":{
+      "field":"dummyDate",
+      "size":100000,
+      "date_ranges":[
+        {
+          "start":"2000-01-01T00:00:00",
+          "end":"2000-01-01T01:00:00"
+        }
+      ]
+    },
+    "myNum":{
+      "field":"dummyNumber",
+      "size":100000,
+      "numeric_ranges":[
+        {
+          "min": 1000,
+          "max": 1001
+        }
+      ]
+    }
+  },
+  "size": 10,
+  "from": 0
+}
+
+ 		+ 
+{
+  "explain": true,
+  "fields": [
+    "*"
+  ],
+  "highlight": {},
+  "query": {
+    "match": "term",
+    "field":"dummyTerm"
+  },
+  "facets":{
+    "myDate":{
+      "field":"dummyDate",
+      "size":100000,
+      "date_ranges":[
+        {
+          "start":"2000-01-01T01:00:00",
+          "end":"2000-01-01T02:00:00"
+        }
+      ]
+    },
+    "myNum":{
+      "field":"dummyNumber",
+      "size":100000,
+      "numeric_ranges":[
+        {
+          "min": 999,
+          "max": 1000
+        }
+      ]
+    }
+  },
+  "size": 10,
+  "from": 0
+}
+
+ 		+ 
+{
+  "explain": true,
+  "fields": [
+    "*"
+  ],
+  "highlight": {},
+  "query": {
+    "match": "term",
+    "field":"dummyTerm"
+  },
+  "facets":{
+    "myDate":{
+      "field":"dummyDate",
+      "size":100000,
+      "date_ranges":[
+        {
+          "start":"2000-01-01T00:00:00" + i hour
+          "end":"2000-01-01T00:00:00" + (i+1) hour
+        }
+      ]
+    },
+    "myNum":{
+      "field":"dummyNumber",
+      "size":100000,
+      "numeric_ranges":[
+        {
+          "min": 1000-i,
+          "max": 1000-i+1
+        }
+      ]
+    }
+  },
+  "size": 10,
+  "from": 0
+}
+
+ 		+ 
+{
+  "explain": true,
+  "fields": [
+    "*"
+  ],
+  "highlight": {},
+  "query": {
+    "match": "term",
+    "field":"dummyTerm"
+  },
+  "facets":{
+    "myDate":{
+      "field":"dummyDate",
+      "size":100000,
+      "date_ranges":[
+        {
+          "start":"2000-01-01T01:00:00" + 1000 hour,
+          "end":"2000-01-01T02:00:00" + 1001 hour
+        }
+      ]
+    },
+    "myNum":{
+      "field":"dummyNumber",
+      "size":100000,
+      "numeric_ranges":[
+        {
+          "min": 0,
+          "max": 1
+        }
+      ]
+    }
+  },
+  "size": 10,
+  "from": 0
+}
+
+
+
+ + +
+ + + + + + + + +
Bleve index size growth with increase in indexed documentsTotal query time for 1000 queries with increase in number of indexed documents
indexSizeVsNumDocs.pngqueryTimevsNumDocs.png
+
+ +
+ + + + + + + + + +
Average increase in index size (in bytes) by enabling DocValuesAverage reduction in time taken to perform 1000 queries (in milliseconds) by enabling DocValues
7762.4727.034
+Even at this small scale, with a small document size and a very limited number of indexed documents, we still observe a noticeable tradeoff. With just a slight increase in the index size (an average of 7KB), we obtain a 20ms reduction in the total execution time, on average, for only 1000 queries. + +

Technical Information

+ +

When a search request involves facet or sorting operations on a field F, these operations occur after the main search query is executed. For instance, if the main query yields a result of 200 documents, the sorting and faceting processes will be applied to these 200 documents. However, the main query result only provides a set of document IDs, not the actual document contents.

+ +

Here's where docValues become essential. If the field mapping for F is docValue enabled, the system can directly access the values for the field from the stored docValue part in the index file. This means that for each document ID returned in the search result, the field values are readily available.

+ +

However, if docValues are not enabled for field F, the system must take a different approach. It needs to "fetch the document" from the index file, read the value for field F, and cache this field-document pair in memory for further processing. The issue becomes apparent in the latter scenario. By not enabling docValues for field F, you essentially retrieve all the documents in the search result (at the worst case), which can be a substantial amount of data. Moreover, you have to cache this information in memory, leading to increased memory usage. As a result, query latency significantly suffers because you're essentially fetching and processing all documents, which can be both time-consuming and resource-intensive. Enabling docValues for the relevant fields is, therefore, a crucial optimization to enhance query performance and reduce memory overhead in such situations.

diff --git a/docs/synonyms.md b/docs/synonyms.md new file mode 100644 index 0000000..4845b40 --- /dev/null +++ b/docs/synonyms.md @@ -0,0 +1,180 @@ +# Synonym search + +* *v2.5.0* (and after) will come with support for **synonym definition indexing and search**. +* We've achieved this by embedding synonym indexes within our bleve (scorch) indexes. +* Usage of zap file format: [v16](https://github.com/blevesearch/zapx/blob/master/zap.md). Here we co-locate text, vector, and synonym indexes as neighbors within segments, continuing to conform to the segmented architecture of *scorch*. + +## Supported + +* Indexing `Synonym Definitions` allows specifying equivalent terms that will be used to construct the synonym index. There are currently two types of `Synonym Definitions` supported: + + 1. Equivalent Mapping: + + In this type, all terms in the *synonyms* list are considered equal and can replace one another. Any of these terms can match a query or document containing any other term in the group, ensuring full synonym coverage. + + ```json + { + "synonyms": [ + "tranquil", + "peaceful", + "calm", + "relaxed", + "unruffled" + ] + } + ``` + + 2. Explicit Mapping: + + In this mapping, only the terms in the *input* list ("blazing") will have the terms in *synonyms* as their synonyms. The input terms are not equivalent to each other, and the synonym relationship is explicitly directional, applying only from the *input* to the *synonyms*. + + ```json + { + "input": [ + "blazing" + ], + "synonyms": [ + "intense", + "radiant", + "burning", + "fiery", + "glowing" + ] + } + ``` + +* The addition of `Synonym Sources` in the index mapping enables associating a set of `synonym definitions` (called a `synonym collection`) with a specific analyzer. This allows for preprocessing of terms in both the "input" and "synonyms" lists before the synonym index is created. By using an analyzer, you can normalize or transform terms (e.g., case folding, stemming) to improve synonym matching. + + ```json + { + "analysis": { + "synonym_sources": { + "english": { + "collection": "en_thesaurus", + "analyzer": "en" + }, + "german": { + "collection": "de_thesaurus", + "analyzer": "de" + } + } + } + } + ``` + + There are two `synonym sources` named "english" and "german," each associated with its respective `synonym collection` and analyzer. In any text field mapping, a `synonym source` can be specified to enable synonym expansion when the field is queried. The analyzer of the synonym source must match the analyzer of the field mapping to which it is applied. + +* Any text-based Bleve query (e.g., match, phrase, term, fuzzy, etc.) will use the `synonym source` (if available) for the queried field to expand the search terms using the thesaurus created from user-defined synonym definitions. The behavior for specific query types is as follows: + + 1. Queries with `fuzziness` parameter: For queries like match, phrase, and match-phrase that support the `fuzziness` parameter, the queried terms are fuzzily matched with the thesaurus's LHS terms to generate candidate terms. These terms are then combined with the results of fuzzy matching against the field dictionary, which contains the terms present in the queried field. + + 2. Wildcard, Regexp, and Prefix queries: These queries follow a similar approach. First, the thesaurus is used to expand terms (e.g., LHS terms that match the prefix or regex). The resulting terms are then combined with candidate terms from dictionary expansion. + +## Indexing + +Below is an example of using the Bleve API to define synonym sources, index synonym definitions, and associate them with a text field mapping: + +```go +// Define a document to be indexed. +doc := struct { + Text string `json:"text"` +}{ + Text: "hardworking employee", +} + +// Define a synonym definition where "hardworking" has equivalent terms. +synDef := &bleve.SynonymDefinition{ + Synonyms: []string{ + "hardworking", + "industrious", + "conscientious", + "persistent", + }, +} + +// Define the name of the `synonym collection`. +// This collection groups multiple synonym definitions. +synonymCollection := "collection1" + +// Define the name of the `synonym source`. +// This source will be associated with specific field mappings. +synonymSourceName := "english" + +// Define the analyzer to process terms in the synonym definitions. +// This analyzer must match the one applied to the field using the synonym source. +analyzer := "en" + +// Configure the synonym source by associating it with the synonym collection and analyzer. +synonymSourceConfig := map[string]interface{}{ + "collection": synonymCollection, + "analyzer": analyzer, +} + +// Create a new index mapping. +bleveMapping := bleve.NewIndexMapping() + +// Add the synonym source configuration to the index mapping. +err := bleveMapping.AddSynonymSource(synonymSourceName, synonymSourceConfig) +if err != nil { + panic(err) +} + +// Create a text field mapping with the specified analyzer and synonym source. +textFieldMapping := bleve.NewTextFieldMapping() +textFieldMapping.Analyzer = analyzer +textFieldMapping.SynonymSource = synonymSourceName + +// Associate the text field mapping with the "text" field in the default document mapping. +bleveMapping.DefaultMapping.AddFieldMappingsAt("text", textFieldMapping) + +// Create a new index with the specified mapping. +index, err := bleve.New("example.bleve", bleveMapping) +if err != nil { + panic(err) +} + +// Index the document into the created index. +err = index.Index("doc1", doc) +if err != nil { + panic(err) +} + +// Check if the index supports synonym indexing and add the synonym definition. +if synIndex, ok := index.(bleve.SynonymIndex); ok { + err = synIndex.IndexSynonym("synDoc1", synonymCollection, synDef) + if err != nil { + panic(err) + } +} else { + // If the index does not support synonym indexing, raise an error. + panic("expected synonym index") +} +``` + +## Querying + +```go +// Query the index created above. +// Create a match query for the term "persistent". +query := bleve.NewMatchQuery("persistent") + +// Specify the field to search within, in this case, the "text" field. +query.SetField("text") + +// Create a search request with the query and enable explanation to understand how results are scored. +searchRequest := bleve.NewSearchRequest(query) +searchRequest.Explain = true + +// Execute the search on the index. +searchResult, err := index.Search(searchRequest) +if err != nil { + // Handle any errors that occur during the search. + panic(err) +} + +// The search result will contain one match: "doc1". This document includes the term "hardworking", +// which is a synonym for the queried term "persistent". The synonym relationship is based on +// the user-defined thesaurus associated with the index. +// Print the search results, which will include the explanation for the match. +fmt.Println(searchResult) +``` diff --git a/docs/vectors.md b/docs/vectors.md new file mode 100644 index 0000000..ac625ea --- /dev/null +++ b/docs/vectors.md @@ -0,0 +1,347 @@ +# Approximate Nearest Neighbor Search (over vectors) + +* *v2.4.0* (and after) will come with support for **vector indexing and search**. +* We've achieved this by embedding [FAISS](https://github.com/facebookresearch/faiss) indexes within our bleve (scorch) indexes. +* Introduction of new zap file formats: [v16](https://github.com/blevesearch/zapx/blob/v16.x/zap.md), [v17](https://github.com/blevesearch/zapx/blob/master/zap.md) .. to co-locate text and vector indexes as neighbors within segments, continuing to conform to the segmented architecture of *scorch*. + +## Pre-requisite(s) + +* Induction of [FAISS](https://github.com/blevesearch/faiss) into our ecosystem, which is a fork of the original [facebookresearch/faiss](https://github.com/facebookresearch/faiss) +* FAISS is a C++ library that needs to be compiled and its shared libraries need to be situated at an accessible path for your application. +* A `vectors` GO TAG needs to be set for bleve to access all the supporting code. This TAG must be set only after the FAISS shared library is made available. Failure to do either will inhibit you from using this feature. +* Please follow these [instructions](#setup-instructions) below for any assistance in the area. +* Releases of `blevesearch/bleve` work with select checkpoints of `blevesearch/faiss` owing to API changes and improvements (tracking over the `bleve` branch): + + | bleve version(s) | blevesearch/faiss version | + | --- | --- | + | `v2.4.0` | [blevesearch/faiss@7b119f4](https://github.com/blevesearch/faiss/tree/7b119f4b9c408989b696b36f8cc53908e53de6db) (modified v1.7.4) | + | `v2.4.1`, `v2.4.2` | [blevesearch/faiss@d9db66a](https://github.com/blevesearch/faiss/tree/d9db66a38518d99eb334218697e1df0732f3fdf8) (modified v1.7.4) | + | `v2.4.3`, `v2.4.4` | [blevesearch/faiss@b747c55](https://github.com/blevesearch/faiss/tree/b747c55a93a9627039c34d44b081f375dca94e57) (modified v1.8.0) | + | `v2.5.0`, `v2.5.1` | [blevesearch/faiss@352484e](https://github.com/blevesearch/faiss/tree/352484e0fc9d1f8f46737841efe5f26e0f383f71) (modified v1.10.0) | + | `v2.5.2`, `v2.5.3`, `v2.5.4` | [blevesearch/faiss@b3d4e00](https://github.com/blevesearch/faiss/tree/b3d4e00a69425b95e0b283da7801efc9f66b580d) (modified v1.11.0) | + | `v2.5.5`, `v2.5.6`, `v2.5.7` | [blevesearch/faiss@8a59a0c](https://github.com/blevesearch/faiss/tree/8a59a0c552fa2d14fa871f6b6bc793de1d277f5e) (modified v1.12.0) | + | `v2.6.0` | [blevesearch/faiss@fff814d](https://github.com/blevesearch/faiss/tree/fff814dea2bdda020363506904979b204ee201aa) (modified v1.13.2) | + +## Features + +* The `vector` field type is an array that is to hold float32 values only. +* The `vector_base64` field type to support base64 encoded strings using little endian byte ordering (v2.4.1+) +* Supported similarity metrics are: [`"cosine"` (v2.4.3+), `"dot_product"`, `"l2_norm"`]. + * `cosine` paths will additionally normalize vectors before indexing and search. +* Supported dimensionality is between 1 and 2048 (v2.4.0), and up to **4096** (v2.4.1+). +* Supported vector index optimizations: + * `recall`, `latency` (v2.4.0+) + * Combination of Flat and IVF indexes with SQ8 quantization. + * `memory_efficient` (v2.4.1+) + * Combination of Flat and IVF indexes with SQ4 quantization. + * `bivf-flat`, `bivf-sq8` (v2.6.0+) + * Combination of BFlat and BIVF indexes with Binary quantization. + * Uses an additional index which is either a Flat index (`bivf-flat`) or an SQ8 index (`bivf-sq8`) for re-ranking. + * `ivf,rabitq` (v2.6.0+) + * Combination of Flat and IVF indexes with RaBitQ quantization. + * Works with [Fast Merge](https://github.com/blevesearch/bleve/blob/master/docs/fast_merge.md) - where a centroid index needs to be built/trained on a sample dataset prior to building the index for segments that deploy IVF indexes to reflect. + +--- + +* Vectors from documents that do not conform to the index mapping dimensionality are simply discarded at index time. +* The dimensionality of the query vector must match the dimensionality of the indexed vectors to obtain any results. +* Pure kNN searches can be performed, but the `query` attribute within the search request must be set -- to `{"match_none": {}}` in this case. The `query` attribute is made optional when `knn` is available with v2.4.1+. +* Hybrid searches are supported, where results from `query` are unioned with results from `knn`. The FTS scores from exact searches are simply summed with the similarity distances to determine the aggregate scores. + +```text +aggregate_score = (query_boost * query_hit_score) + (knn_boost * knn_hit_distance) +``` + +* Advanced score fusion strategies (v2.5.4+) are available if requested for - see [score fusion](score_fusion.md#score-fusion-for-hybrid-search). +* Multi kNN searches are supported - the `knn` object within the search request accepts an array of requests. These sub objects are unioned by default but this behavior can be overridden by setting `knn_operator` to `"and"`. +* Previously supported pagination settings will work as they were, with size/limit being applied over the top-K hits combined with any exact search hits. +* Pre-filtered vector and hybrid search (v2.4.3+): Apply any Bleve filter query first to narrow down candidates before running kNN search, making vector and hybrid searches faster and more relevant. +* Fields containing multiple vectors (v2.5.7+): + * A single document may contain multiple vectors within the same field, in the form of either: + * an array of vectors (multi-vector field) + * an array of objects each containing a vector (nested-vector field) + * For single-kNN queries, each document is scored using its single best-matching vector. + * For multi-kNN queries, the system selects the best-matching vector for each query vector within the document. + +--- + +* GPU-Accelerated vector search (v2.6.0+): + * Requires FAISS built with `-DFAISS_ENABLE_GPU=ON` CMake option (needs NVIDIA CUDA toolkit). + * Requires the `gpu` go tag in addition to the `vectors` tag when building bleve. + * GPU acceleration is enabled per vector field via the field mapping's GPU option. + * Bleve will use any available GPUs to offload training, indexing, and kNN searches for GPU-enabled vector fields. + * GPUs will be used for vector fields which are optimized for `recall`, `latency`, and `memory_efficient`. + * Multi-GPU support: when multiple GPUs are available, a load balancer distributes vector search workloads across devices. + * See [GPU setup instructions](#gpu-setup-instructions-v260) below for building FAISS with GPU support. + +## Indexing + +```go +// Example document with single-vector, multi-vector, and nested-vector fields +doc := struct { + Id string `json:"id"` + Text string `json:"text"` + Vec []float32 `json:"vec"` // Single-vector field + Embeddings [][]float32 `json:"embeddings"` // Multi-vector field: array of vectors (v2.5.7+) + Sections []struct { // Nested-vector field: array of objects with vectors (v2.5.7+) + Text string `json:"text"` + Vec []float32 `json:"vec"` + } `json:"sections"` +}{ + Id: "example", + Text: "hello from united states", + Vec: []float32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, // Single-vector field of dimensionality 10 + Embeddings: [][]float32{ // Multi-vector field containing 2 vectors of dimensionality 10 + {10, 11, 12, 13, 14, 15, 16, 17, 18, 19}, // First vector + {20, 21, 22, 23, 24, 25, 26, 27, 28, 29}, // Second vector + }, + Sections: []struct { // Nested-vector field containing 2 objects each with a vector of dimensionality 10 + Text string `json:"text"` + Vec []float32 `json:"vec"` + }{ + {Text: "first section", Vec: []float32{30, 31, 32, 33, 34, 35, 36, 37, 38, 39}}, + {Text: "second section", Vec: []float32{40, 41, 42, 43, 44, 45, 46, 47, 48, 49}}, + }, +} + +// Field mappings +textFieldMapping := bleve.NewTextFieldMapping() +vectorFieldMapping := bleve.NewVectorFieldMapping() +vectorFieldMapping.Dims = 10 // Set vector dimensionality +vectorFieldMapping.Similarity = index.CosineSimilarity // Set similarity metric +vectorFieldMapping.VectorIndexOptimizedFor = index.IndexOptimizedForRecall // Set vector index optimization type +vectorFieldMapping.GPU = true // Enable GPU acceleration + +// Sub-document mappings +sectionsMapping := bleve.NewDocumentMapping() +sectionsMapping.AddFieldMappingsAt("text", textFieldMapping) +sectionsMapping.AddFieldMappingsAt("vec", vectorFieldMapping) + +// Index mapping +bleveMapping := bleve.NewIndexMapping() +bleveMapping.DefaultMapping.AddFieldMappingsAt("text", textFieldMapping) +bleveMapping.DefaultMapping.AddFieldMappingsAt("vec", vectorFieldMapping) // Single-vector +bleveMapping.DefaultMapping.AddFieldMappingsAt("embeddings", vectorFieldMapping) // Multi-vector +bleveMapping.DefaultMapping.AddSubDocumentMapping("sections", sectionsMapping) // Nested-vector + +// Create the index +index, err := bleve.New("example.bleve", bleveMapping) +if err != nil { + panic(err) +} + +// Index the document +err = index.Index(doc.Id, doc) +if err != nil { + panic(err) +} +``` + +## Querying + +```go +// ------------------------------------ +// Single-vector field search (v2.4.0+) +// ------------------------------------ +searchRequest := bleve.NewSearchRequest(bleve.NewMatchNoneQuery()) +searchRequest.AddKNN( + "vec", // Vector field + []float32{0, 1, 1, 4, 4, 5, 7, 6, 8, 9}, // Query vector + 5, // top-k + 1, // boost +) +searchResult, err := index.Search(searchRequest) +if err != nil { + panic(err) +} +// Scores are 1 / squared L2 distance, e.g., score = 0.25 for squared distance of 4 +fmt.Printf("Single-vector field kNN result:\n%s\n", searchResult) +``` + +```go +// ----------------------------------- +// Multi-vector field search (v2.5.7+) +// ----------------------------------- +searchRequest = bleve.NewSearchRequest(bleve.NewMatchNoneQuery()) +searchRequest.AddKNN( + "embeddings", + []float32{0, 1, 1, 4, 4, 5, 7, 6, 8, 9}, + 5, + 1.0, +) +searchResult, err = index.Search(searchRequest) +if err != nil { + panic(err) +} +// Scores are based on the **best-matching vector** from the multi-vector field. +// Example: distances to doc vectors {10..19} and {20..29} → pick the closer one (smaller squared L2), +// then score = 1 / squared L2 distance. +fmt.Printf("Multi-vector field kNN result:\n%s\n", searchResult) +``` + +```go +// ------------------------------------ +// Nested-vector field search (v2.5.7+) +// ------------------------------------ +searchRequest = bleve.NewSearchRequest(bleve.NewMatchNoneQuery()) +searchRequest.AddKNN( + "sections.vec", + []float32{0, 1, 1, 4, 4, 5, 7, 6, 8, 9}, + 5, + 1.0, +) +searchResult, err = index.Search(searchRequest) +if err != nil { + panic(err) +} +// Scores are based on the **best-matching vector** from the nested-vector field. +// Example: distances to doc vectors {30..39} and {40..49} → pick the closer one (smaller squared L2), +// then score = 1 / squared L2 distance. +fmt.Printf("Nested-vector field kNN result:\n%s\n", searchResult) +``` + +```go +// ----------------------------------------------------- +// Multi kNN queries on multi-vector documents (v2.5.7+) +// ----------------------------------------------------- +searchRequest = bleve.NewSearchRequest(bleve.NewMatchNoneQuery()) +searchRequest.AddKNN( + "embeddings", + []float32{0, 1, 1, 4, 4, 5, 7, 6, 8, 9}, + 5, + 1.0, +) +searchRequest.AddKNN( + "embeddings", + []float32{1, 2, 2, 5, 5, 6, 8, 7, 9, 10}, + 8, + 1.0, +) +searchResult, err = index.Search(searchRequest) +if err != nil { + panic(err) +} +// Document score explanation: +// - For each query vector, Bleve selects the **closest vector** in the multi-vector field. +// - Scores from multiple queries are then **normalized and summed** to get the final document score. +// For example, if the closest vector to the first query has squared L2 distance 4 (score 0.25) +// and the closest vector to the second query has squared L2 distance 1 (score 1.0), +// and both queries use equal boost values of 1.0, the normalization factor is 1/√2 (where 2 is the number of kNN queries). +// Then the total document score = 1/√2 * 0.25 + 1/√2 * 1.0 = 0.1768 + 0.7071 = 0.8839. +// Note: If the boost values differ, or if more queries are used, the normalization factor and score calculation will change accordingly. +fmt.Printf("Multi kNN queries result:\n%s\n", searchResult) +``` + +```go +// -------------------------------------- +// Hybrid search: text + vector (v2.4.0+) +// -------------------------------------- +searchRequest = bleve.NewSearchRequest(bleve.NewMatchQuery("united states")) +searchRequest.AddKNN( + "vec", + []float32{0, 1, 1, 4, 4, 5, 7, 6, 8, 9}, + 5, + 1, +) +searchResult, err = index.Search(searchRequest) +if err != nil { + panic(err) +} +// Score = sum of text relevance score + kNN vector score +// Example: text score 0.5 + vector score 0.25 = total score 0.75 +fmt.Printf("Hybrid search result:\n%s\n", searchResult) +``` + +## Querying with filters (v2.4.3+) + +```go +// Pre-filtered vector/hybrid search: filter query narrows candidates before KNN search +searchRequest = bleve.NewSearchRequest(bleve.NewMatchNoneQuery()) // replace with any Bleve query for Pre-filtered Hybrid Search +filterQuery := bleve.NewTermQuery("hello") // Filter query to narrow candidates +searchRequest.AddKNNWithFilter( + "vec", // Vector field name + []float32{0, 1, 1, 4, 4, 5, 7, 6, 8, 9}, // Query vector (must match indexed vector dims) + 5, // Number of nearest neighbors to return (k) + 1, // Boost factor for KNN score + filterQuery, // Filter query applied before KNN search +) +searchResult, err = index.Search(searchRequest) +if err != nil { + panic(err) +} +// Scores are computed only among documents matching the filter query +// Example: if only one document matches the filter and has squared L2 distance 4 to the query vector, +// its score will be 0.25 (1 / 4) and returned as the top result. +fmt.Printf("Pre-filtered kNN search result:\n%s\n", searchResult) +``` + +## Setup Instructions + +* Using `cmake` is a recommended approach by FAISS authors. +* More details here - [faiss/INSTALL](https://github.com/blevesearch/faiss/blob/main/INSTALL.md). + +### Linux + +Also documented here - [go-faiss/README](https://github.com/blevesearch/go-faiss/blob/master/README.md). + +```shell +git clone https://github.com/blevesearch/faiss.git +cd faiss +cmake -B build -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_C_API=ON -DBUILD_SHARED_LIBS=ON . +make -C build +sudo make -C build install +``` + +Building will produce the dynamic library `faiss_c`. You will need to install it in a place where your system will find it (e.g. /usr/lib). You can do this with: + +```shell +sudo cp build/c_api/libfaiss_c.so /usr/local/lib +``` + +### OSX + +While you shouldn't need to do anything different for macOS x86_64, with aarch64 - some instructions need adjusting (see [facebookresearch/faiss#2111](https://github.com/facebookresearch/faiss/issues/2111)) .. + +```shell +LDFLAGS="-L/opt/homebrew/opt/llvm/lib" CPPFLAGS="-I/opt/homebrew/opt/llvm/include" CXX=/opt/homebrew/opt/llvm/bin/clang++ CC=/opt/homebrew/opt/llvm/bin/clang cmake -B build -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_C_API=ON -DBUILD_SHARED_LIBS=ON -DFAISS_ENABLE_PYTHON=OFF . +make -C build +sudo make -C build install +sudo cp build/c_api/libfaiss_c.dylib /usr/local/lib +``` + +### Sanity check + +Once the supporting library is built and made available, a sanity run is recommended to make sure all unit tests and especially those accessing the vectors' code pass. Here's how .. + +```shell +go test -ldflags "-r /usr/local/lib" ./... -tags=vectors +``` + +## GPU Setup Instructions (v2.6.0+) + +GPU-accelerated vector search requires FAISS to be compiled with CUDA support and the `gpu` go tag to be set when building bleve. + +### Pre-requisites + +* An NVIDIA GPU with CUDA support (Pascal architecture / CC 6.0+ recommended). +* [NVIDIA CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit) installed and available on your system. +* The FAISS C API shared library built with GPU support. + +### Linux (GPU) + +```shell +git clone https://github.com/blevesearch/faiss.git +cd faiss +cmake -B build -DFAISS_ENABLE_GPU=ON -DFAISS_ENABLE_C_API=ON -DBUILD_SHARED_LIBS=ON . +make -C build +sudo make -C build install +sudo cp build/c_api/libfaiss_c.so /usr/local/lib +``` + +### Sanity check (GPU) + +Run the full test suite with both the `vectors` and `gpu` build tags: + +```shell +go test -ldflags "-r /usr/local/lib" ./... -tags=vectors,gpu +``` diff --git a/document/document.go b/document/document.go new file mode 100644 index 0000000..cb3f537 --- /dev/null +++ b/document/document.go @@ -0,0 +1,181 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "fmt" + "reflect" + + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeDocument int + +func init() { + var d Document + reflectStaticSizeDocument = int(reflect.TypeOf(d).Size()) +} + +type Document struct { + id string + Fields []Field `json:"fields"` + NestedDocuments []*Document `json:"nested_documents"` + CompositeFields []*CompositeField + StoredFieldsSize uint64 + indexed bool +} + +func (d *Document) StoredFieldsBytes() uint64 { + return d.StoredFieldsSize +} + +func NewDocument(id string) *Document { + return &Document{ + id: id, + Fields: make([]Field, 0), + CompositeFields: make([]*CompositeField, 0), + } +} + +func NewSynonymDocument(id string) *Document { + return &Document{ + id: id, + Fields: make([]Field, 0), + } +} + +func (d *Document) Size() int { + sizeInBytes := reflectStaticSizeDocument + size.SizeOfPtr + + len(d.id) + + for _, entry := range d.Fields { + sizeInBytes += entry.Size() + } + + for _, entry := range d.CompositeFields { + sizeInBytes += entry.Size() + } + + for _, entry := range d.NestedDocuments { + if entry != nil { + sizeInBytes += entry.Size() + } + } + + return sizeInBytes +} + +func (d *Document) AddField(f Field) *Document { + switch f := f.(type) { + case *CompositeField: + d.CompositeFields = append(d.CompositeFields, f) + default: + d.Fields = append(d.Fields, f) + } + return d +} + +func (d *Document) GoString() string { + fields := "" + for i, field := range d.Fields { + if i != 0 { + fields += ", " + } + fields += fmt.Sprintf("%#v", field) + } + compositeFields := "" + for i, field := range d.CompositeFields { + if i != 0 { + compositeFields += ", " + } + compositeFields += fmt.Sprintf("%#v", field) + } + return fmt.Sprintf("&document.Document{ID:%s, Fields: %s, CompositeFields: %s}", d.ID(), fields, compositeFields) +} + +func (d *Document) NumPlainTextBytes() uint64 { + rv := uint64(0) + for _, field := range d.Fields { + rv += field.NumPlainTextBytes() + } + for _, compositeField := range d.CompositeFields { + for _, field := range d.Fields { + if compositeField.includesField(field.Name()) { + rv += field.NumPlainTextBytes() + } + } + } + for _, nestedDoc := range d.NestedDocuments { + if nestedDoc != nil { + rv += nestedDoc.NumPlainTextBytes() + } + } + return rv +} + +func (d *Document) ID() string { + return d.id +} + +func (d *Document) SetID(id string) { + d.id = id +} + +func (d *Document) AddIDField() { + d.AddField(NewTextFieldCustom("_id", nil, []byte(d.ID()), index.IndexField|index.StoreField, nil)) +} + +func (d *Document) VisitFields(visitor index.FieldVisitor) { + for _, f := range d.Fields { + visitor(f) + } +} + +func (d *Document) VisitComposite(visitor index.CompositeFieldVisitor) { + for _, f := range d.CompositeFields { + visitor(f) + } +} + +func (d *Document) HasComposite() bool { + return len(d.CompositeFields) > 0 +} + +func (d *Document) VisitSynonymFields(visitor index.SynonymFieldVisitor) { + for _, f := range d.Fields { + if sf, ok := f.(index.SynonymField); ok { + visitor(sf) + } + } +} + +func (d *Document) SetIndexed() { + d.indexed = true +} + +func (d *Document) Indexed() bool { + return d.indexed +} + +func (d *Document) AddNestedDocument(doc *Document) { + d.NestedDocuments = append(d.NestedDocuments, doc) +} + +func (d *Document) VisitNestedDocuments(visitor func(doc index.Document)) { + for _, doc := range d.NestedDocuments { + visitor(doc) + } +} diff --git a/document/document_test.go b/document/document_test.go new file mode 100644 index 0000000..22aff7a --- /dev/null +++ b/document/document_test.go @@ -0,0 +1,73 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "testing" +) + +func TestDocumentNumPlainTextBytes(t *testing.T) { + + tests := []struct { + doc *Document + num uint64 + }{ + { + doc: NewDocument("a"), + num: 0, + }, + { + doc: NewDocument("b"). + AddField(NewTextField("name", nil, []byte("hello"))), + num: 5, + }, + { + doc: NewDocument("c"). + AddField(NewTextField("name", nil, []byte("hello"))). + AddField(NewTextField("desc", nil, []byte("x"))), + num: 6, + }, + { + doc: NewDocument("d"). + AddField(NewTextField("name", nil, []byte("hello"))). + AddField(NewTextField("desc", nil, []byte("x"))). + AddField(NewNumericField("age", nil, 1.0)), + num: 14, + }, + { + doc: NewDocument("e"). + AddField(NewTextField("name", nil, []byte("hello"))). + AddField(NewTextField("desc", nil, []byte("x"))). + AddField(NewNumericField("age", nil, 1.0)). + AddField(NewCompositeField("_all", true, nil, nil)), + num: 28, + }, + { + doc: NewDocument("e"). + AddField(NewTextField("name", nil, []byte("hello"))). + AddField(NewTextField("desc", nil, []byte("x"))). + AddField(NewNumericField("age", nil, 1.0)). + AddField(NewCompositeField("_all", true, nil, []string{"age"})), + num: 20, + }, + } + + for _, test := range tests { + actual := test.doc.NumPlainTextBytes() + if actual != test.num { + t.Errorf("expected doc '%s' to have %d plain text bytes, got %d", test.doc.ID(), test.num, actual) + } + } +} diff --git a/document/field.go b/document/field.go new file mode 100644 index 0000000..eb104e2 --- /dev/null +++ b/document/field.go @@ -0,0 +1,45 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + index "github.com/blevesearch/bleve_index_api" +) + +type Field interface { + // Name returns the path of the field from the root DocumentMapping. + // A root field path is "field", a subdocument field is "parent.field". + Name() string + // ArrayPositions returns the intermediate document and field indices + // required to resolve the field value in the document. For example, if the + // field path is "doc1.doc2.field" where doc1 and doc2 are slices or + // arrays, ArrayPositions returns 2 indices used to resolve "doc2" value in + // "doc1", then "field" in "doc2". + ArrayPositions() []uint64 + Options() index.FieldIndexingOptions + Analyze() + Value() []byte + + // NumPlainTextBytes should return the number of plain text bytes + // that this field represents - this is a common metric for tracking + // the rate of indexing + NumPlainTextBytes() uint64 + + Size() int + + EncodedFieldType() byte + AnalyzedLength() int + AnalyzedTokenFrequencies() index.TokenFrequencies +} diff --git a/document/field_boolean.go b/document/field_boolean.go new file mode 100644 index 0000000..fdd14ce --- /dev/null +++ b/document/field_boolean.go @@ -0,0 +1,142 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "fmt" + "reflect" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeBooleanField int + +func init() { + var f BooleanField + reflectStaticSizeBooleanField = int(reflect.TypeOf(f).Size()) +} + +const DefaultBooleanIndexingOptions = index.StoreField | index.IndexField | index.DocValues + +type BooleanField struct { + name string + arrayPositions []uint64 + options index.FieldIndexingOptions + value []byte + numPlainTextBytes uint64 + length int + frequencies index.TokenFrequencies +} + +func (b *BooleanField) Size() int { + var freqSize int + if b.frequencies != nil { + freqSize = b.frequencies.Size() + } + return reflectStaticSizeBooleanField + size.SizeOfPtr + + len(b.name) + + len(b.arrayPositions)*size.SizeOfUint64 + + len(b.value) + + freqSize +} + +func (b *BooleanField) Name() string { + return b.name +} + +func (b *BooleanField) ArrayPositions() []uint64 { + return b.arrayPositions +} + +func (b *BooleanField) Options() index.FieldIndexingOptions { + return b.options +} + +func (b *BooleanField) Analyze() { + tokens := make(analysis.TokenStream, 0) + tokens = append(tokens, &analysis.Token{ + Start: 0, + End: len(b.value), + Term: b.value, + Position: 1, + Type: analysis.Boolean, + }) + + b.length = len(tokens) + b.frequencies = analysis.TokenFrequency(tokens, b.arrayPositions, b.options) +} + +func (b *BooleanField) Value() []byte { + return b.value +} + +func (b *BooleanField) Boolean() (bool, error) { + if len(b.value) == 1 { + return b.value[0] == 'T', nil + } + return false, fmt.Errorf("boolean field has %d bytes", len(b.value)) +} + +func (b *BooleanField) GoString() string { + return fmt.Sprintf("&document.BooleanField{Name:%s, Options: %s, Value: %s}", b.name, b.options, b.value) +} + +func (b *BooleanField) NumPlainTextBytes() uint64 { + return b.numPlainTextBytes +} + +func (b *BooleanField) EncodedFieldType() byte { + return 'b' +} + +func (b *BooleanField) AnalyzedLength() int { + return b.length +} + +func (b *BooleanField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return b.frequencies +} + +func NewBooleanFieldFromBytes(name string, arrayPositions []uint64, value []byte) *BooleanField { + return &BooleanField{ + name: name, + arrayPositions: arrayPositions, + value: value, + options: DefaultBooleanIndexingOptions, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewBooleanField(name string, arrayPositions []uint64, b bool) *BooleanField { + return NewBooleanFieldWithIndexingOptions(name, arrayPositions, b, DefaultBooleanIndexingOptions) +} + +func NewBooleanFieldWithIndexingOptions(name string, arrayPositions []uint64, b bool, options index.FieldIndexingOptions) *BooleanField { + numPlainTextBytes := 5 + v := []byte("F") + if b { + numPlainTextBytes = 4 + v = []byte("T") + } + return &BooleanField{ + name: name, + arrayPositions: arrayPositions, + value: v, + options: options, + numPlainTextBytes: uint64(numPlainTextBytes), + } +} diff --git a/document/field_composite.go b/document/field_composite.go new file mode 100644 index 0000000..e0ba8af --- /dev/null +++ b/document/field_composite.go @@ -0,0 +1,138 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "reflect" + + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeCompositeField int + +func init() { + var cf CompositeField + reflectStaticSizeCompositeField = int(reflect.TypeOf(cf).Size()) +} + +const DefaultCompositeIndexingOptions = index.IndexField + +type CompositeField struct { + name string + includedFields map[string]bool + excludedFields map[string]bool + defaultInclude bool + options index.FieldIndexingOptions + totalLength int + compositeFrequencies index.TokenFrequencies +} + +func NewCompositeField(name string, defaultInclude bool, include []string, exclude []string) *CompositeField { + return NewCompositeFieldWithIndexingOptions(name, defaultInclude, include, exclude, DefaultCompositeIndexingOptions) +} + +func NewCompositeFieldWithIndexingOptions(name string, defaultInclude bool, include []string, exclude []string, options index.FieldIndexingOptions) *CompositeField { + rv := &CompositeField{ + name: name, + options: options, + defaultInclude: defaultInclude, + includedFields: make(map[string]bool, len(include)), + excludedFields: make(map[string]bool, len(exclude)), + compositeFrequencies: make(index.TokenFrequencies), + } + + for _, i := range include { + rv.includedFields[i] = true + } + for _, e := range exclude { + rv.excludedFields[e] = true + } + + return rv +} + +func (c *CompositeField) Size() int { + sizeInBytes := reflectStaticSizeCompositeField + size.SizeOfPtr + + len(c.name) + + for k := range c.includedFields { + sizeInBytes += size.SizeOfString + len(k) + size.SizeOfBool + } + + for k := range c.excludedFields { + sizeInBytes += size.SizeOfString + len(k) + size.SizeOfBool + } + if c.compositeFrequencies != nil { + sizeInBytes += c.compositeFrequencies.Size() + } + + return sizeInBytes +} + +func (c *CompositeField) Name() string { + return c.name +} + +func (c *CompositeField) ArrayPositions() []uint64 { + return []uint64{} +} + +func (c *CompositeField) Options() index.FieldIndexingOptions { + return c.options +} + +func (c *CompositeField) Analyze() { +} + +func (c *CompositeField) Value() []byte { + return []byte{} +} + +func (c *CompositeField) NumPlainTextBytes() uint64 { + return 0 +} + +func (c *CompositeField) includesField(field string) bool { + shouldInclude := c.defaultInclude + _, fieldShouldBeIncluded := c.includedFields[field] + if fieldShouldBeIncluded { + shouldInclude = true + } + _, fieldShouldBeExcluded := c.excludedFields[field] + if fieldShouldBeExcluded { + shouldInclude = false + } + return shouldInclude +} + +func (c *CompositeField) Compose(field string, length int, freq index.TokenFrequencies) { + if c.includesField(field) { + c.totalLength += length + c.compositeFrequencies.MergeAll(field, freq) + } +} + +func (c *CompositeField) EncodedFieldType() byte { + return 'c' +} + +func (c *CompositeField) AnalyzedLength() int { + return c.totalLength +} + +func (c *CompositeField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return c.compositeFrequencies +} diff --git a/document/field_datetime.go b/document/field_datetime.go new file mode 100644 index 0000000..f3b859c --- /dev/null +++ b/document/field_datetime.go @@ -0,0 +1,202 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "bytes" + "fmt" + "math" + "reflect" + "time" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var dateTimeValueSeperator = []byte{'\xff'} + +var reflectStaticSizeDateTimeField int + +func init() { + var f DateTimeField + reflectStaticSizeDateTimeField = int(reflect.TypeOf(f).Size()) +} + +const DefaultDateTimeIndexingOptions = index.StoreField | index.IndexField | index.DocValues +const DefaultDateTimePrecisionStep uint = 4 + +var MinTimeRepresentable = time.Unix(0, math.MinInt64) +var MaxTimeRepresentable = time.Unix(0, math.MaxInt64) + +type DateTimeField struct { + name string + arrayPositions []uint64 + options index.FieldIndexingOptions + value numeric.PrefixCoded + numPlainTextBytes uint64 + length int + frequencies index.TokenFrequencies +} + +func (n *DateTimeField) Size() int { + var freqSize int + if n.frequencies != nil { + freqSize = n.frequencies.Size() + } + return reflectStaticSizeDateTimeField + size.SizeOfPtr + + len(n.name) + + len(n.arrayPositions)*size.SizeOfUint64 + + len(n.value) + + freqSize +} + +func (n *DateTimeField) Name() string { + return n.name +} + +func (n *DateTimeField) ArrayPositions() []uint64 { + return n.arrayPositions +} + +func (n *DateTimeField) Options() index.FieldIndexingOptions { + return n.options +} + +func (n *DateTimeField) EncodedFieldType() byte { + return 'd' +} + +func (n *DateTimeField) AnalyzedLength() int { + return n.length +} + +func (n *DateTimeField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return n.frequencies +} + +// split the value into the prefix coded date and the layout +// using the dateTimeValueSeperator as the split point +func (n *DateTimeField) splitValue() (numeric.PrefixCoded, string) { + parts := bytes.SplitN(n.value, dateTimeValueSeperator, 2) + if len(parts) == 1 { + return numeric.PrefixCoded(parts[0]), "" + } + return numeric.PrefixCoded(parts[0]), string(parts[1]) +} + +func (n *DateTimeField) Analyze() { + valueWithoutLayout, _ := n.splitValue() + tokens := make(analysis.TokenStream, 0) + tokens = append(tokens, &analysis.Token{ + Start: 0, + End: len(valueWithoutLayout), + Term: valueWithoutLayout, + Position: 1, + Type: analysis.DateTime, + }) + + original, err := valueWithoutLayout.Int64() + if err == nil { + + shift := DefaultDateTimePrecisionStep + for shift < 64 { + shiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift) + if err != nil { + break + } + token := analysis.Token{ + Start: 0, + End: len(shiftEncoded), + Term: shiftEncoded, + Position: 1, + Type: analysis.DateTime, + } + tokens = append(tokens, &token) + shift += DefaultDateTimePrecisionStep + } + } + + n.length = len(tokens) + n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options) +} + +func (n *DateTimeField) Value() []byte { + return n.value +} + +func (n *DateTimeField) DateTime() (time.Time, string, error) { + date, layout := n.splitValue() + i64, err := date.Int64() + if err != nil { + return time.Time{}, "", err + } + return time.Unix(0, i64).UTC(), layout, nil +} + +func (n *DateTimeField) GoString() string { + return fmt.Sprintf("&document.DateField{Name:%s, Options: %s, Value: %s}", n.name, n.options, n.value) +} + +func (n *DateTimeField) NumPlainTextBytes() uint64 { + return n.numPlainTextBytes +} + +func NewDateTimeFieldFromBytes(name string, arrayPositions []uint64, value []byte) *DateTimeField { + return &DateTimeField{ + name: name, + arrayPositions: arrayPositions, + value: value, + options: DefaultDateTimeIndexingOptions, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewDateTimeField(name string, arrayPositions []uint64, dt time.Time, layout string) (*DateTimeField, error) { + return NewDateTimeFieldWithIndexingOptions(name, arrayPositions, dt, layout, DefaultDateTimeIndexingOptions) +} + +func NewDateTimeFieldWithIndexingOptions(name string, arrayPositions []uint64, dt time.Time, layout string, options index.FieldIndexingOptions) (*DateTimeField, error) { + if canRepresent(dt) { + dtInt64 := dt.UnixNano() + prefixCoded := numeric.MustNewPrefixCodedInt64(dtInt64, 0) + // The prefixCoded value is combined with the layout. + // This is necessary because the storage layer stores a fields value as a byte slice + // without storing extra information like layout. So by making value = prefixCoded + layout, + // both pieces of information are stored in the byte slice. + // During a query, the layout is extracted from the byte slice stored to correctly + // format the prefixCoded value. + valueWithLayout := append(prefixCoded, dateTimeValueSeperator...) + valueWithLayout = append(valueWithLayout, []byte(layout)...) + return &DateTimeField{ + name: name, + arrayPositions: arrayPositions, + value: valueWithLayout, + options: options, + // not correct, just a place holder until we revisit how fields are + // represented and can fix this better + numPlainTextBytes: uint64(8), + }, nil + } + return nil, fmt.Errorf("cannot represent %s in this type", dt) +} + +func canRepresent(dt time.Time) bool { + if dt.Before(MinTimeRepresentable) || dt.After(MaxTimeRepresentable) { + return false + } + return true +} diff --git a/document/field_geopoint.go b/document/field_geopoint.go new file mode 100644 index 0000000..ef8938f --- /dev/null +++ b/document/field_geopoint.go @@ -0,0 +1,208 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "fmt" + "reflect" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeGeoPointField int + +func init() { + var f GeoPointField + reflectStaticSizeGeoPointField = int(reflect.TypeOf(f).Size()) +} + +var GeoPrecisionStep uint = 9 + +type GeoPointField struct { + name string + arrayPositions []uint64 + options index.FieldIndexingOptions + value numeric.PrefixCoded + numPlainTextBytes uint64 + length int + frequencies index.TokenFrequencies + + spatialplugin index.SpatialAnalyzerPlugin +} + +func (n *GeoPointField) Size() int { + var freqSize int + if n.frequencies != nil { + freqSize = n.frequencies.Size() + } + return reflectStaticSizeGeoPointField + size.SizeOfPtr + + len(n.name) + + len(n.arrayPositions)*size.SizeOfUint64 + + len(n.value) + + freqSize +} + +func (n *GeoPointField) Name() string { + return n.name +} + +func (n *GeoPointField) ArrayPositions() []uint64 { + return n.arrayPositions +} + +func (n *GeoPointField) Options() index.FieldIndexingOptions { + return n.options +} + +func (n *GeoPointField) EncodedFieldType() byte { + return 'g' +} + +func (n *GeoPointField) AnalyzedLength() int { + return n.length +} + +func (n *GeoPointField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return n.frequencies +} + +func (n *GeoPointField) Analyze() { + tokens := make(analysis.TokenStream, 0, 8) + tokens = append(tokens, &analysis.Token{ + Start: 0, + End: len(n.value), + Term: n.value, + Position: 1, + Type: analysis.Numeric, + }) + + if n.spatialplugin != nil { + lat, _ := n.Lat() + lon, _ := n.Lon() + p := &geo.Point{Lat: lat, Lon: lon} + terms := n.spatialplugin.GetIndexTokens(p) + + for _, term := range terms { + token := analysis.Token{ + Start: 0, + End: len(term), + Term: []byte(term), + Position: 1, + Type: analysis.AlphaNumeric, + } + tokens = append(tokens, &token) + } + } else { + original, err := n.value.Int64() + if err == nil { + + shift := GeoPrecisionStep + for shift < 64 { + shiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift) + if err != nil { + break + } + token := analysis.Token{ + Start: 0, + End: len(shiftEncoded), + Term: shiftEncoded, + Position: 1, + Type: analysis.Numeric, + } + tokens = append(tokens, &token) + shift += GeoPrecisionStep + } + } + } + + n.length = len(tokens) + n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options) +} + +func (n *GeoPointField) Value() []byte { + return n.value +} + +func (n *GeoPointField) Lon() (float64, error) { + i64, err := n.value.Int64() + if err != nil { + return 0.0, err + } + return geo.MortonUnhashLon(uint64(i64)), nil +} + +func (n *GeoPointField) Lat() (float64, error) { + i64, err := n.value.Int64() + if err != nil { + return 0.0, err + } + return geo.MortonUnhashLat(uint64(i64)), nil +} + +func (n *GeoPointField) GoString() string { + return fmt.Sprintf("&document.GeoPointField{Name:%s, Options: %s, Value: %s}", n.name, n.options, n.value) +} + +func (n *GeoPointField) NumPlainTextBytes() uint64 { + return n.numPlainTextBytes +} + +func NewGeoPointFieldFromBytes(name string, arrayPositions []uint64, value []byte) *GeoPointField { + return &GeoPointField{ + name: name, + arrayPositions: arrayPositions, + value: value, + options: DefaultNumericIndexingOptions, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewGeoPointField(name string, arrayPositions []uint64, lon, lat float64) *GeoPointField { + return NewGeoPointFieldWithIndexingOptions(name, arrayPositions, lon, lat, DefaultNumericIndexingOptions) +} + +func NewGeoPointFieldWithIndexingOptions(name string, arrayPositions []uint64, lon, lat float64, options index.FieldIndexingOptions) *GeoPointField { + mhash := geo.MortonHash(lon, lat) + prefixCoded := numeric.MustNewPrefixCodedInt64(int64(mhash), 0) + + // docvalues are always enabled for geopoint fields, even if the + // indexing options are set to not include docvalues. + // snappy compression and chunking are always skipped for geopoint + // to avoid mem copies and faster lookups. + options |= index.DocValues + options |= index.SkipDVChunking + options |= index.SkipDVCompression + + return &GeoPointField{ + name: name, + arrayPositions: arrayPositions, + value: prefixCoded, + options: options, + // not correct, just a place holder until we revisit how fields are + // represented and can fix this better + numPlainTextBytes: uint64(8), + } +} + +// SetSpatialAnalyzerPlugin implements the +// index.TokenisableSpatialField interface. +func (n *GeoPointField) SetSpatialAnalyzerPlugin( + plugin index.SpatialAnalyzerPlugin) { + n.spatialplugin = plugin +} diff --git a/document/field_geopoint_test.go b/document/field_geopoint_test.go new file mode 100644 index 0000000..2860264 --- /dev/null +++ b/document/field_geopoint_test.go @@ -0,0 +1,16 @@ +package document + +import "testing" + +func TestGeoPointField(t *testing.T) { + gf := NewGeoPointField("loc", []uint64{}, 0.0015, 0.0015) + gf.Analyze() + numTokens := gf.AnalyzedLength() + tokenFreqs := gf.AnalyzedTokenFrequencies() + if numTokens != 8 { + t.Errorf("expected 8 tokens, got %d", numTokens) + } + if len(tokenFreqs) != 8 { + t.Errorf("expected 8 token freqs") + } +} diff --git a/document/field_geoshape.go b/document/field_geoshape.go new file mode 100644 index 0000000..2eb7aa3 --- /dev/null +++ b/document/field_geoshape.go @@ -0,0 +1,273 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "fmt" + "reflect" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/geo/geojson" +) + +var reflectStaticSizeGeoShapeField int + +func init() { + var f GeoShapeField + reflectStaticSizeGeoShapeField = int(reflect.TypeOf(f).Size()) +} + +const DefaultGeoShapeIndexingOptions = index.IndexField | index.DocValues + +type GeoShapeField struct { + name string + shape index.GeoJSON + arrayPositions []uint64 + options index.FieldIndexingOptions + numPlainTextBytes uint64 + length int + encodedValue []byte + value []byte + + frequencies index.TokenFrequencies +} + +func (n *GeoShapeField) Size() int { + var freqSize int + if n.frequencies != nil { + freqSize = n.frequencies.Size() + } + return reflectStaticSizeGeoShapeField + size.SizeOfPtr + + len(n.name) + + len(n.arrayPositions)*size.SizeOfUint64 + + len(n.encodedValue) + + len(n.value) + + freqSize +} + +func (n *GeoShapeField) Name() string { + return n.name +} + +func (n *GeoShapeField) ArrayPositions() []uint64 { + return n.arrayPositions +} + +func (n *GeoShapeField) Options() index.FieldIndexingOptions { + return n.options +} + +func (n *GeoShapeField) EncodedFieldType() byte { + return 's' +} + +func (n *GeoShapeField) AnalyzedLength() int { + return n.length +} + +func (n *GeoShapeField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return n.frequencies +} + +func (n *GeoShapeField) Analyze() { + // compute the bytes representation for the coordinates + tokens := make(analysis.TokenStream, 0) + + rti := geo.GetSpatialAnalyzerPlugin("s2") + terms := rti.GetIndexTokens(n.shape) + + for _, term := range terms { + token := analysis.Token{ + Start: 0, + End: len(term), + Term: []byte(term), + Position: 1, + Type: analysis.AlphaNumeric, + } + tokens = append(tokens, &token) + } + + n.length = len(tokens) + n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options) +} + +func (n *GeoShapeField) Value() []byte { + return n.value +} + +func (n *GeoShapeField) GoString() string { + return fmt.Sprintf("&document.GeoShapeField{Name:%s, Options: %s, Value: %s}", + n.name, n.options, n.value) +} + +func (n *GeoShapeField) NumPlainTextBytes() uint64 { + return n.numPlainTextBytes +} + +func (n *GeoShapeField) EncodedShape() []byte { + return n.encodedValue +} + +func NewGeoShapeField(name string, arrayPositions []uint64, + coordinates [][][][]float64, typ string) *GeoShapeField { + return NewGeoShapeFieldWithIndexingOptions(name, arrayPositions, + coordinates, typ, DefaultGeoShapeIndexingOptions) +} + +func NewGeoShapeFieldFromBytes(name string, arrayPositions []uint64, + value []byte) *GeoShapeField { + return &GeoShapeField{ + name: name, + arrayPositions: arrayPositions, + value: value, + options: DefaultGeoShapeIndexingOptions, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewGeoShapeFieldWithIndexingOptions(name string, arrayPositions []uint64, + coordinates [][][][]float64, typ string, + options index.FieldIndexingOptions) *GeoShapeField { + shape := &geojson.GeoShape{ + Coordinates: coordinates, + Type: typ, + } + + return NewGeoShapeFieldFromShapeWithIndexingOptions(name, + arrayPositions, shape, options) +} + +func NewGeoShapeFieldFromShapeWithIndexingOptions(name string, arrayPositions []uint64, + geoShape *geojson.GeoShape, options index.FieldIndexingOptions) *GeoShapeField { + + var shape index.GeoJSON + var encodedValue []byte + var err error + + if geoShape.Type == geo.CircleType { + shape, encodedValue, err = geo.NewGeoCircleShape(geoShape.Center, geoShape.Radius) + } else { + shape, encodedValue, err = geo.NewGeoJsonShape(geoShape.Coordinates, geoShape.Type) + } + if err != nil { + return nil + } + + // extra glue bytes to work around the term splitting logic from interfering + // the custom encoding of the geoshape coordinates inside the docvalues. + encodedValue = append(geo.GlueBytes, append(encodedValue, geo.GlueBytes...)...) + + // get the byte value for the geoshape. + value, err := shape.Value() + if err != nil { + return nil + } + + // docvalues are always enabled for geoshape fields, even if the + // indexing options are set to not include docvalues. + // snappy compression and chunking are always skipped for geoshape + // to avoid mem copies and faster lookups. + options |= index.DocValues + options |= index.SkipDVChunking + options |= index.SkipDVCompression + + return &GeoShapeField{ + shape: shape, + name: name, + arrayPositions: arrayPositions, + options: options, + encodedValue: encodedValue, + value: value, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewGeometryCollectionFieldWithIndexingOptions(name string, + arrayPositions []uint64, coordinates [][][][][]float64, types []string, + options index.FieldIndexingOptions) *GeoShapeField { + if len(coordinates) != len(types) { + return nil + } + + shapes := make([]*geojson.GeoShape, len(types)) + for i := range coordinates { + shapes[i] = &geojson.GeoShape{ + Coordinates: coordinates[i], + Type: types[i], + } + } + + return NewGeometryCollectionFieldFromShapesWithIndexingOptions(name, + arrayPositions, shapes, options) +} + +func NewGeometryCollectionFieldFromShapesWithIndexingOptions(name string, + arrayPositions []uint64, geoShapes []*geojson.GeoShape, + options index.FieldIndexingOptions) *GeoShapeField { + shape, encodedValue, err := geo.NewGeometryCollectionFromShapes(geoShapes) + if err != nil { + return nil + } + + // extra glue bytes to work around the term splitting logic from interfering + // the custom encoding of the geoshape coordinates inside the docvalues. + encodedValue = append(geo.GlueBytes, append(encodedValue, geo.GlueBytes...)...) + + // get the byte value for the geometryCollection. + value, err := shape.Value() + if err != nil { + return nil + } + + // docvalues are always enabled for geoshape fields, even if the + // indexing options are set to not include docvalues. + // snappy compression and chunking are always skipped for geoshape + // to avoid mem copies and faster lookups. + options |= index.DocValues + options |= index.SkipDVChunking + options |= index.SkipDVCompression + + return &GeoShapeField{ + shape: shape, + name: name, + arrayPositions: arrayPositions, + options: options, + encodedValue: encodedValue, + value: value, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewGeoCircleFieldWithIndexingOptions(name string, arrayPositions []uint64, + centerPoint []float64, radius string, + options index.FieldIndexingOptions) *GeoShapeField { + + shape := &geojson.GeoShape{ + Center: centerPoint, + Radius: radius, + Type: geo.CircleType, + } + + return NewGeoShapeFieldFromShapeWithIndexingOptions(name, + arrayPositions, shape, options) +} + +// GeoShape is an implementation of the index.GeoShapeField interface. +func (n *GeoShapeField) GeoShape() (index.GeoJSON, error) { + return geojson.ParseGeoJSONShape(n.value) +} diff --git a/document/field_geoshape_v2.go b/document/field_geoshape_v2.go new file mode 100644 index 0000000..60614d4 --- /dev/null +++ b/document/field_geoshape_v2.go @@ -0,0 +1,162 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "reflect" + + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/geov2" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/geo/geojson" +) + +var reflectStaticSizeGeoShapeV2Field int + +func init() { + var f GeoShapeV2Field + reflectStaticSizeGeoShapeV2Field = int(reflect.TypeOf(f).Size()) +} + +type GeoShapeV2Field struct { + name string + + shape index.GeoJSON + inner []uint64 + cross []uint64 + scoreInner uint64 + scoreCross uint64 + bBoxBytes []byte + shapeBytes []byte + + options index.FieldIndexingOptions +} + +func (f *GeoShapeV2Field) Name() string { + return f.name +} + +func (f *GeoShapeV2Field) ArrayPositions() []uint64 { + return nil +} + +func (f *GeoShapeV2Field) Options() index.FieldIndexingOptions { + return f.options +} + +func (f *GeoShapeV2Field) Analyze() { + f.inner, f.cross = f.shape.IndexCells() + f.scoreInner = geov2.CalcCellsScore(f.inner) + f.scoreCross = geov2.CalcCellsScore(f.cross) + + if bBox, ok := f.shape.BoundingBox().(*geojson.Envelope); ok { + bBoxBytes, err := bBox.Marshal() + if err != nil { + return + } + f.bBoxBytes = bBoxBytes + } +} + +func (f *GeoShapeV2Field) Value() []byte { + return []byte{} +} + +func (f *GeoShapeV2Field) NumPlainTextBytes() uint64 { + return 0 +} + +func (f *GeoShapeV2Field) Size() int { + return reflectStaticSizeGeoShapeV2Field + size.SizeOfPtr + + len(f.name) + + len(f.inner)*size.SizeOfUint64 + + len(f.cross)*size.SizeOfUint64 + + len(f.bBoxBytes) + + len(f.shapeBytes) +} + +func (f *GeoShapeV2Field) EncodedFieldType() byte { + return 'o' +} + +func (f *GeoShapeV2Field) AnalyzedLength() int { + return 0 +} + +func (f *GeoShapeV2Field) AnalyzedTokenFrequencies() index.TokenFrequencies { + return nil +} + +func (f *GeoShapeV2Field) InnerCells() []uint64 { + return f.inner +} + +func (f *GeoShapeV2Field) CrossCells() []uint64 { + return f.cross +} + +func (f *GeoShapeV2Field) EncodedBoundingBox() []byte { + return f.bBoxBytes +} + +func (f *GeoShapeV2Field) EncodedShape() []byte { + return f.shapeBytes +} + +func (f *GeoShapeV2Field) Scores() (uint64, uint64) { + return f.scoreInner, f.scoreCross +} + +func NewGeoShapeV2FieldFromShapeWithIndexingOptions(name string, geoShape *geojson.GeoShape, + options index.FieldIndexingOptions) *GeoShapeV2Field { + + var shape index.GeoJSON + var shapeBytes []byte + var err error + + if geoShape.Type == geo.CircleType { + shape, shapeBytes, err = geo.NewGeoCircleShape(geoShape.Center, + geoShape.Radius) + } else { + shape, shapeBytes, err = geo.NewGeoJsonShape(geoShape.Coordinates, + geoShape.Type) + } + if err != nil { + return nil + } + + return &GeoShapeV2Field{ + name: name, + shape: shape, + options: options, + shapeBytes: shapeBytes, + } +} + +func NewGeometryCollectionV2FieldFromShapesWithIndexingOptions(name string, + geoShapes []*geojson.GeoShape, options index.FieldIndexingOptions) *GeoShapeV2Field { + shape, shapeBytes, err := geo.NewGeometryCollectionFromShapes(geoShapes) + if err != nil { + return nil + } + + return &GeoShapeV2Field{ + name: name, + shape: shape, + options: options, + shapeBytes: shapeBytes, + } +} diff --git a/document/field_ip.go b/document/field_ip.go new file mode 100644 index 0000000..3a5ab37 --- /dev/null +++ b/document/field_ip.go @@ -0,0 +1,137 @@ +// Copyright (c) 2021 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "fmt" + "net" + "reflect" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeIPField int + +func init() { + var f IPField + reflectStaticSizeIPField = int(reflect.TypeOf(f).Size()) +} + +const DefaultIPIndexingOptions = index.StoreField | index.IndexField | index.DocValues + +type IPField struct { + name string + arrayPositions []uint64 + options index.FieldIndexingOptions + value net.IP + numPlainTextBytes uint64 + length int + frequencies index.TokenFrequencies +} + +func (b *IPField) Size() int { + var freqSize int + if b.frequencies != nil { + freqSize = b.frequencies.Size() + } + return reflectStaticSizeIPField + size.SizeOfPtr + + len(b.name) + + len(b.arrayPositions)*size.SizeOfUint64 + + len(b.value) + + freqSize +} + +func (b *IPField) Name() string { + return b.name +} + +func (b *IPField) ArrayPositions() []uint64 { + return b.arrayPositions +} + +func (b *IPField) Options() index.FieldIndexingOptions { + return b.options +} + +func (n *IPField) EncodedFieldType() byte { + return 'i' +} + +func (n *IPField) AnalyzedLength() int { + return n.length +} + +func (n *IPField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return n.frequencies +} + +func (b *IPField) Analyze() { + + tokens := analysis.TokenStream{ + &analysis.Token{ + Start: 0, + End: len(b.value), + Term: b.value, + Position: 1, + Type: analysis.IP, + }, + } + b.length = 1 + b.frequencies = analysis.TokenFrequency(tokens, b.arrayPositions, b.options) +} + +func (b *IPField) Value() []byte { + return b.value +} + +func (b *IPField) IP() (net.IP, error) { + return net.IP(b.value), nil +} + +func (b *IPField) GoString() string { + return fmt.Sprintf("&document.IPField{Name:%s, Options: %s, Value: %s}", b.name, b.options, net.IP(b.value)) +} + +func (b *IPField) NumPlainTextBytes() uint64 { + return b.numPlainTextBytes +} + +func NewIPFieldFromBytes(name string, arrayPositions []uint64, value []byte) *IPField { + return &IPField{ + name: name, + arrayPositions: arrayPositions, + value: value, + options: DefaultIPIndexingOptions, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewIPField(name string, arrayPositions []uint64, v net.IP) *IPField { + return NewIPFieldWithIndexingOptions(name, arrayPositions, v, DefaultIPIndexingOptions) +} + +func NewIPFieldWithIndexingOptions(name string, arrayPositions []uint64, b net.IP, options index.FieldIndexingOptions) *IPField { + v := b.To16() + + return &IPField{ + name: name, + arrayPositions: arrayPositions, + value: v, + options: options, + numPlainTextBytes: net.IPv6len, + } +} diff --git a/document/field_ip_test.go b/document/field_ip_test.go new file mode 100644 index 0000000..e5e8bf2 --- /dev/null +++ b/document/field_ip_test.go @@ -0,0 +1,38 @@ +// Copyright (c) 2021 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "bytes" + "net" + "testing" +) + +func TestIPField(t *testing.T) { + nf := NewIPField("ip", []uint64{}, net.IPv4(192, 168, 1, 1)) + nf.Analyze() + if nf.length != 1 { + t.Errorf("expected 1 token") + } + if len(nf.value) != 16 { + t.Errorf("stored value should be in 16 byte ipv6 format") + } + if !bytes.Equal(nf.value, []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 192, 168, 1, 1}) { + t.Errorf("wrong value stored, expected 192.168.1.1, got %q", nf.value.String()) + } + if len(nf.frequencies) != 1 { + t.Errorf("expected 1 token freqs") + } +} diff --git a/document/field_numeric.go b/document/field_numeric.go new file mode 100644 index 0000000..1ee7b75 --- /dev/null +++ b/document/field_numeric.go @@ -0,0 +1,165 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "fmt" + "reflect" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeNumericField int + +func init() { + var f NumericField + reflectStaticSizeNumericField = int(reflect.TypeOf(f).Size()) +} + +const DefaultNumericIndexingOptions = index.StoreField | index.IndexField | index.DocValues + +const DefaultPrecisionStep uint = 4 + +type NumericField struct { + name string + arrayPositions []uint64 + options index.FieldIndexingOptions + value numeric.PrefixCoded + numPlainTextBytes uint64 + length int + frequencies index.TokenFrequencies +} + +func (n *NumericField) Size() int { + var freqSize int + if n.frequencies != nil { + freqSize = n.frequencies.Size() + } + return reflectStaticSizeNumericField + size.SizeOfPtr + + len(n.name) + + len(n.arrayPositions)*size.SizeOfUint64 + + len(n.value) + + freqSize +} + +func (n *NumericField) Name() string { + return n.name +} + +func (n *NumericField) ArrayPositions() []uint64 { + return n.arrayPositions +} + +func (n *NumericField) Options() index.FieldIndexingOptions { + return n.options +} + +func (n *NumericField) EncodedFieldType() byte { + return 'n' +} + +func (n *NumericField) AnalyzedLength() int { + return n.length +} + +func (n *NumericField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return n.frequencies +} + +func (n *NumericField) Analyze() { + tokens := make(analysis.TokenStream, 0) + tokens = append(tokens, &analysis.Token{ + Start: 0, + End: len(n.value), + Term: n.value, + Position: 1, + Type: analysis.Numeric, + }) + + original, err := n.value.Int64() + if err == nil { + + shift := DefaultPrecisionStep + for shift < 64 { + shiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift) + if err != nil { + break + } + token := analysis.Token{ + Start: 0, + End: len(shiftEncoded), + Term: shiftEncoded, + Position: 1, + Type: analysis.Numeric, + } + tokens = append(tokens, &token) + shift += DefaultPrecisionStep + } + } + + n.length = len(tokens) + n.frequencies = analysis.TokenFrequency(tokens, n.arrayPositions, n.options) +} + +func (n *NumericField) Value() []byte { + return n.value +} + +func (n *NumericField) Number() (float64, error) { + i64, err := n.value.Int64() + if err != nil { + return 0.0, err + } + return numeric.Int64ToFloat64(i64), nil +} + +func (n *NumericField) GoString() string { + return fmt.Sprintf("&document.NumericField{Name:%s, Options: %s, Value: %s}", n.name, n.options, n.value) +} + +func (n *NumericField) NumPlainTextBytes() uint64 { + return n.numPlainTextBytes +} + +func NewNumericFieldFromBytes(name string, arrayPositions []uint64, value []byte) *NumericField { + return &NumericField{ + name: name, + arrayPositions: arrayPositions, + value: value, + options: DefaultNumericIndexingOptions, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewNumericField(name string, arrayPositions []uint64, number float64) *NumericField { + return NewNumericFieldWithIndexingOptions(name, arrayPositions, number, DefaultNumericIndexingOptions) +} + +func NewNumericFieldWithIndexingOptions(name string, arrayPositions []uint64, number float64, options index.FieldIndexingOptions) *NumericField { + numberInt64 := numeric.Float64ToInt64(number) + prefixCoded := numeric.MustNewPrefixCodedInt64(numberInt64, 0) + return &NumericField{ + name: name, + arrayPositions: arrayPositions, + value: prefixCoded, + options: options, + // not correct, just a place holder until we revisit how fields are + // represented and can fix this better + numPlainTextBytes: uint64(8), + } +} diff --git a/document/field_numeric_test.go b/document/field_numeric_test.go new file mode 100644 index 0000000..5f36d68 --- /dev/null +++ b/document/field_numeric_test.go @@ -0,0 +1,32 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "testing" +) + +func TestNumericField(t *testing.T) { + nf := NewNumericField("age", []uint64{}, 3.4) + nf.Analyze() + numTokens := nf.AnalyzedLength() + tokenFreqs := nf.AnalyzedTokenFrequencies() + if numTokens != 16 { + t.Errorf("expected 16 tokens") + } + if len(tokenFreqs) != 16 { + t.Errorf("expected 16 token freqs") + } +} diff --git a/document/field_synonym.go b/document/field_synonym.go new file mode 100644 index 0000000..c34b481 --- /dev/null +++ b/document/field_synonym.go @@ -0,0 +1,149 @@ +// Copyright (c) 2024 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "reflect" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeSynonymField int + +func init() { + var f SynonymField + reflectStaticSizeSynonymField = int(reflect.TypeOf(f).Size()) +} + +const DefaultSynonymIndexingOptions = index.IndexField + +type SynonymField struct { + name string + analyzer analysis.Analyzer + options index.FieldIndexingOptions + input []string + synonyms []string + numPlainTextBytes uint64 + + // populated during analysis + synonymMap map[string][]string +} + +func (s *SynonymField) Size() int { + return reflectStaticSizeSynonymField + size.SizeOfPtr + + len(s.name) +} + +func (s *SynonymField) Name() string { + return s.name +} + +func (s *SynonymField) ArrayPositions() []uint64 { + return nil +} + +func (s *SynonymField) Options() index.FieldIndexingOptions { + return s.options +} + +func (s *SynonymField) NumPlainTextBytes() uint64 { + return s.numPlainTextBytes +} + +func (s *SynonymField) AnalyzedLength() int { + return 0 +} + +func (s *SynonymField) EncodedFieldType() byte { + return 'y' +} + +func (s *SynonymField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return nil +} + +func (s *SynonymField) Analyze() { + var analyzedInput []string + if len(s.input) > 0 { + analyzedInput = make([]string, 0, len(s.input)) + for _, term := range s.input { + analyzedTerm := analyzeSynonymTerm(term, s.analyzer) + if analyzedTerm != "" { + analyzedInput = append(analyzedInput, analyzedTerm) + } + } + } + analyzedSynonyms := make([]string, 0, len(s.synonyms)) + for _, syn := range s.synonyms { + analyzedTerm := analyzeSynonymTerm(syn, s.analyzer) + if analyzedTerm != "" { + analyzedSynonyms = append(analyzedSynonyms, analyzedTerm) + } + } + s.synonymMap = processSynonymData(analyzedInput, analyzedSynonyms) +} + +func (s *SynonymField) Value() []byte { + return nil +} + +func (s *SynonymField) IterateSynonyms(visitor func(term string, synonyms []string)) { + for term, synonyms := range s.synonymMap { + visitor(term, synonyms) + } +} + +func NewSynonymField(name string, analyzer analysis.Analyzer, input []string, synonyms []string) *SynonymField { + return &SynonymField{ + name: name, + analyzer: analyzer, + options: DefaultSynonymIndexingOptions, + input: input, + synonyms: synonyms, + } +} + +func processSynonymData(input []string, synonyms []string) map[string][]string { + var synonymMap map[string][]string + if len(input) > 0 { + // Map each term to the same list of synonyms. + synonymMap = make(map[string][]string, len(input)) + for _, term := range input { + synonymMap[term] = synonyms + } + } else { + synonymMap = make(map[string][]string, len(synonyms)) + // Precompute a map where each synonym points to all other synonyms. + for i, elem := range synonyms { + synonymMap[elem] = make([]string, 0, len(synonyms)-1) + for j, otherElem := range synonyms { + if i != j { + synonymMap[elem] = append(synonymMap[elem], otherElem) + } + } + } + } + return synonymMap +} + +func analyzeSynonymTerm(term string, analyzer analysis.Analyzer) string { + tokenStream := analyzer.Analyze([]byte(term)) + if len(tokenStream) == 1 { + return string(tokenStream[0].Term) + } + return "" +} diff --git a/document/field_text.go b/document/field_text.go new file mode 100644 index 0000000..d35e747 --- /dev/null +++ b/document/field_text.go @@ -0,0 +1,162 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "fmt" + "reflect" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeTextField int + +func init() { + var f TextField + reflectStaticSizeTextField = int(reflect.TypeOf(f).Size()) +} + +const DefaultTextIndexingOptions = index.IndexField | index.DocValues + +type TextField struct { + name string + arrayPositions []uint64 + options index.FieldIndexingOptions + analyzer analysis.Analyzer + value []byte + numPlainTextBytes uint64 + length int + frequencies index.TokenFrequencies +} + +func (t *TextField) Size() int { + var freqSize int + if t.frequencies != nil { + freqSize = t.frequencies.Size() + } + return reflectStaticSizeTextField + size.SizeOfPtr + + len(t.name) + + len(t.arrayPositions)*size.SizeOfUint64 + + len(t.value) + + freqSize +} + +func (t *TextField) Name() string { + return t.name +} + +func (t *TextField) ArrayPositions() []uint64 { + return t.arrayPositions +} + +func (t *TextField) Options() index.FieldIndexingOptions { + return t.options +} + +func (t *TextField) EncodedFieldType() byte { + return 't' +} + +func (t *TextField) AnalyzedLength() int { + return t.length +} + +func (t *TextField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return t.frequencies +} + +func (t *TextField) Analyze() { + var tokens analysis.TokenStream + if t.analyzer != nil { + bytesToAnalyze := t.Value() + if t.options.IsStored() { + // need to copy + bytesCopied := make([]byte, len(bytesToAnalyze)) + copy(bytesCopied, bytesToAnalyze) + bytesToAnalyze = bytesCopied + } + tokens = t.analyzer.Analyze(bytesToAnalyze) + } else { + tokens = analysis.TokenStream{ + &analysis.Token{ + Start: 0, + End: len(t.value), + Term: t.value, + Position: 1, + Type: analysis.AlphaNumeric, + }, + } + } + t.length = len(tokens) // number of tokens in this doc field + t.frequencies = analysis.TokenFrequency(tokens, t.arrayPositions, t.options) +} + +func (t *TextField) Analyzer() analysis.Analyzer { + return t.analyzer +} + +func (t *TextField) Value() []byte { + return t.value +} + +func (t *TextField) Text() string { + return string(t.value) +} + +func (t *TextField) GoString() string { + return fmt.Sprintf("&document.TextField{Name:%s, Options: %s, Analyzer: %v, Value: %s, ArrayPositions: %v}", t.name, t.options, t.analyzer, t.value, t.arrayPositions) +} + +func (t *TextField) NumPlainTextBytes() uint64 { + return t.numPlainTextBytes +} + +func NewTextField(name string, arrayPositions []uint64, value []byte) *TextField { + return NewTextFieldWithIndexingOptions(name, arrayPositions, value, DefaultTextIndexingOptions) +} + +func NewTextFieldWithIndexingOptions(name string, arrayPositions []uint64, value []byte, options index.FieldIndexingOptions) *TextField { + return &TextField{ + name: name, + arrayPositions: arrayPositions, + options: options, + value: value, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewTextFieldWithAnalyzer(name string, arrayPositions []uint64, value []byte, analyzer analysis.Analyzer) *TextField { + return &TextField{ + name: name, + arrayPositions: arrayPositions, + options: DefaultTextIndexingOptions, + analyzer: analyzer, + value: value, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewTextFieldCustom(name string, arrayPositions []uint64, value []byte, options index.FieldIndexingOptions, analyzer analysis.Analyzer) *TextField { + return &TextField{ + name: name, + arrayPositions: arrayPositions, + options: options, + analyzer: analyzer, + value: value, + numPlainTextBytes: uint64(len(value)), + } +} diff --git a/document/field_vector.go b/document/field_vector.go new file mode 100644 index 0000000..136b79b --- /dev/null +++ b/document/field_vector.go @@ -0,0 +1,157 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package document + +import ( + "fmt" + "reflect" + + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeVectorField int + +func init() { + var f VectorField + reflectStaticSizeVectorField = int(reflect.TypeOf(f).Size()) +} + +const DefaultVectorIndexingOptions = index.IndexField + +type VectorField struct { + name string + dims int // Dimensionality of the vector + similarity string // Similarity metric to use for scoring + options index.FieldIndexingOptions + value []float32 + numPlainTextBytes uint64 + vectorIndexOptimizedFor string // Optimization applied to this index. +} + +func (n *VectorField) Size() int { + return reflectStaticSizeVectorField + size.SizeOfPtr + + len(n.name) + + len(n.similarity) + + len(n.vectorIndexOptimizedFor) + + int(numBytesFloat32s(n.value)) +} + +func (n *VectorField) Name() string { + return n.name +} + +func (n *VectorField) ArrayPositions() []uint64 { + return nil +} + +func (n *VectorField) Options() index.FieldIndexingOptions { + return n.options +} + +func (n *VectorField) NumPlainTextBytes() uint64 { + return n.numPlainTextBytes +} + +func (n *VectorField) AnalyzedLength() int { + // vectors aren't analyzed + return 0 +} + +func (n *VectorField) EncodedFieldType() byte { + return 'v' +} + +func (n *VectorField) AnalyzedTokenFrequencies() index.TokenFrequencies { + // vectors aren't analyzed + return nil +} + +func (n *VectorField) Analyze() { + // vectors aren't analyzed +} + +func (n *VectorField) Value() []byte { + return nil +} + +func (n *VectorField) GoString() string { + return fmt.Sprintf("&document.VectorField{Name:%s, Options: %s, "+ + "Value: %+v}", n.name, n.options, n.value) +} + +// For the sake of not polluting the API, we are keeping arrayPositions as a +// parameter, but it is not used. +func NewVectorField(name string, arrayPositions []uint64, + vector []float32, dims int, similarity, vectorIndexOptimizedFor string) *VectorField { + return NewVectorFieldWithIndexingOptions(name, arrayPositions, + vector, dims, similarity, vectorIndexOptimizedFor, + DefaultVectorIndexingOptions) +} + +// For the sake of not polluting the API, we are keeping arrayPositions as a +// parameter, but it is not used. +func NewVectorFieldWithIndexingOptions(name string, arrayPositions []uint64, + vector []float32, dims int, similarity, vectorIndexOptimizedFor string, + options index.FieldIndexingOptions) *VectorField { + // ensure the options are set to not store/index term vectors/doc values + options &^= index.StoreField | index.IncludeTermVectors | index.DocValues + // skip freq/norms for vector field + options |= index.SkipFreqNorm + + // bivf-sq8 indexes only supports hamming distance for the primary + // binary index. Similarity here is used for the backing flat index, + // which is set to cosine similarity for recall reasons + if index.OptimizationRequiresBinaryIndex(vectorIndexOptimizedFor) { + similarity = index.CosineSimilarity + } + + return &VectorField{ + name: name, + dims: dims, + similarity: similarity, + options: options, + value: vector, + numPlainTextBytes: numBytesFloat32s(vector), + vectorIndexOptimizedFor: vectorIndexOptimizedFor, + } +} + +func numBytesFloat32s(value []float32) uint64 { + return uint64(len(value) * size.SizeOfFloat32) +} + +// ----------------------------------------------------------------------------- +// Following methods help in implementing the bleve_index_api's VectorField +// interface. + +func (n *VectorField) Vector() []float32 { + return n.value +} + +func (n *VectorField) Dims() int { + return n.dims +} + +func (n *VectorField) Similarity() string { + return n.similarity +} + +func (n *VectorField) IndexOptimizedFor() string { + return n.vectorIndexOptimizedFor +} diff --git a/document/field_vector_base64.go b/document/field_vector_base64.go new file mode 100644 index 0000000..31d6cbf --- /dev/null +++ b/document/field_vector_base64.go @@ -0,0 +1,163 @@ +// Copyright (c) 2024 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package document + +import ( + "encoding/base64" + "encoding/binary" + "fmt" + "math" + "reflect" + + "github.com/blevesearch/bleve/v2/size" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeVectorBase64Field int + +func init() { + var f VectorBase64Field + reflectStaticSizeVectorBase64Field = int(reflect.TypeOf(f).Size()) +} + +type VectorBase64Field struct { + vectorField *VectorField + base64Encoding string +} + +func (n *VectorBase64Field) Size() int { + var vecFieldSize int + if n.vectorField != nil { + vecFieldSize = n.vectorField.Size() + } + return reflectStaticSizeVectorBase64Field + size.SizeOfPtr + + len(n.base64Encoding) + + vecFieldSize +} + +func (n *VectorBase64Field) Name() string { + return n.vectorField.Name() +} + +func (n *VectorBase64Field) ArrayPositions() []uint64 { + return n.vectorField.ArrayPositions() +} + +func (n *VectorBase64Field) Options() index.FieldIndexingOptions { + return n.vectorField.Options() +} + +func (n *VectorBase64Field) NumPlainTextBytes() uint64 { + return n.vectorField.NumPlainTextBytes() +} + +func (n *VectorBase64Field) AnalyzedLength() int { + return n.vectorField.AnalyzedLength() +} + +func (n *VectorBase64Field) EncodedFieldType() byte { + return 'e' +} + +func (n *VectorBase64Field) AnalyzedTokenFrequencies() index.TokenFrequencies { + return n.vectorField.AnalyzedTokenFrequencies() +} + +func (n *VectorBase64Field) Analyze() { +} + +func (n *VectorBase64Field) Value() []byte { + return n.vectorField.Value() +} + +func (n *VectorBase64Field) GoString() string { + return fmt.Sprintf("&document.vectorFieldBase64Field{Name:%s, Options: %s, "+ + "Value: %+v}", n.vectorField.Name(), n.vectorField.Options(), n.vectorField.Value()) +} + +// For the sake of not polluting the API, we are keeping arrayPositions as a +// parameter, but it is not used. +func NewVectorBase64Field(name string, arrayPositions []uint64, vectorBase64 string, + dims int, similarity, vectorIndexOptimizedFor string) (*VectorBase64Field, error) { + + decodedVector, err := DecodeVector(vectorBase64) + if err != nil { + return nil, err + } + + return &VectorBase64Field{ + vectorField: NewVectorFieldWithIndexingOptions(name, arrayPositions, + decodedVector, dims, similarity, + vectorIndexOptimizedFor, DefaultVectorIndexingOptions), + + base64Encoding: vectorBase64, + }, nil +} + +// This function takes a base64 encoded string and decodes it into +// a vector. +func DecodeVector(encodedValue string) ([]float32, error) { + // We first decode the encoded string into a byte array. + decodedString, err := base64.StdEncoding.DecodeString(encodedValue) + if err != nil { + return nil, err + } + + // The array is expected to be divisible by 4 because each float32 + // should occupy 4 bytes + if len(decodedString)%size.SizeOfFloat32 != 0 { + return nil, fmt.Errorf("decoded byte array not divisible by %d", size.SizeOfFloat32) + } + dims := int(len(decodedString) / size.SizeOfFloat32) + + if dims <= 0 { + return nil, fmt.Errorf("unable to decode encoded vector") + } + + decodedVector := make([]float32, dims) + + // We iterate through the array 4 bytes at a time and convert each of + // them to a float32 value by reading them in a little endian notation + for i := 0; i < dims; i++ { + bytes := decodedString[i*size.SizeOfFloat32 : (i+1)*size.SizeOfFloat32] + entry := math.Float32frombits(binary.LittleEndian.Uint32(bytes)) + if !util.IsValidFloat32(float64(entry)) { + return nil, fmt.Errorf("invalid float32 value: %f", entry) + } + decodedVector[i] = entry + } + + return decodedVector, nil +} + +func (n *VectorBase64Field) Vector() []float32 { + return n.vectorField.Vector() +} + +func (n *VectorBase64Field) Dims() int { + return n.vectorField.Dims() +} + +func (n *VectorBase64Field) Similarity() string { + return n.vectorField.Similarity() +} + +func (n *VectorBase64Field) IndexOptimizedFor() string { + return n.vectorField.IndexOptimizedFor() +} diff --git a/document/field_vector_base64_test.go b/document/field_vector_base64_test.go new file mode 100644 index 0000000..91212b0 --- /dev/null +++ b/document/field_vector_base64_test.go @@ -0,0 +1,112 @@ +// Copyright (c) 2024 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package document + +import ( + "bytes" + "encoding/base64" + "encoding/binary" + "fmt" + "math/rand" + "testing" +) + +func TestDecodeVector(t *testing.T) { + vec := make([]float32, 2048) + for i := range vec { + vec[i] = rand.Float32() + } + + vecBytes := bytifyVec(vec) + encodedVec := base64.StdEncoding.EncodeToString(vecBytes) + + decodedVector, err := DecodeVector(encodedVec) + if err != nil { + t.Error(err) + } + if len(decodedVector) != len(vec) { + t.Errorf("Decoded vector dimensions not same as original vector dimensions") + } + + for i := range vec { + if vec[i] != decodedVector[i] { + t.Fatalf("Decoded vector not the same as original vector %v != %v", vec[i], decodedVector[i]) + } + } +} + +func BenchmarkDecodeVector128(b *testing.B) { + vec := make([]float32, 128) + for i := range vec { + vec[i] = rand.Float32() + } + + vecBytes := bytifyVec(vec) + encodedVec := base64.StdEncoding.EncodeToString(vecBytes) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = DecodeVector(encodedVec) + } +} + +func BenchmarkDecodeVector784(b *testing.B) { + vec := make([]float32, 784) + for i := range vec { + vec[i] = rand.Float32() + } + + vecBytes := bytifyVec(vec) + encodedVec := base64.StdEncoding.EncodeToString(vecBytes) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = DecodeVector(encodedVec) + } +} + +func BenchmarkDecodeVector1536(b *testing.B) { + vec := make([]float32, 1536) + for i := range vec { + vec[i] = rand.Float32() + } + + vecBytes := bytifyVec(vec) + encodedVec := base64.StdEncoding.EncodeToString(vecBytes) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _ = DecodeVector(encodedVec) + } +} + +func bytifyVec(vec []float32) []byte { + buf := new(bytes.Buffer) + + for _, v := range vec { + err := binary.Write(buf, binary.LittleEndian, v) + if err != nil { + fmt.Println(err) + } + } + + return buf.Bytes() +} diff --git a/earlystop_test.go b/earlystop_test.go new file mode 100644 index 0000000..150dba7 --- /dev/null +++ b/earlystop_test.go @@ -0,0 +1,184 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "fmt" + "os" + "testing" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search/query" +) + +// buildEarlyStopIndex indexes n documents that all match "common", so a bounded +// scan has plenty of matches left undrained. +func buildEarlyStopIndex(t *testing.T, n int) Index { + t.Helper() + + dir, err := os.MkdirTemp("", "earlystop") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + + im := mapping.NewIndexMapping() + im.DefaultAnalyzer = "standard" + idx, err := New(dir+"/i.bleve", im) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = idx.Close() }) + + batch := idx.NewBatch() + for i := 0; i < n; i++ { + if err := batch.Index(fmt.Sprintf("d%05d", i), map[string]interface{}{ + "body": fmt.Sprintf("common tag%d", i%10), + "num": float64(i), + }); err != nil { + t.Fatal(err) + } + } + if err := idx.Batch(batch); err != nil { + t.Fatal(err) + } + return idx +} + +func earlyStopQuery() query.Query { + q := query.NewTermQuery("common") + q.SetField("body") + return q +} + +// TestEarlyStopBoundedScan is the contract for the bounded scan: with +// Score="none" and a bounded Size, the request means "any Size+From matching +// docs", so collection may stop once that many hits are in hand instead of +// draining every match. +// +// Two things must hold. The caller must still get Size hits — stopping early must +// not lose results it asked for. And Total must be reported as a lower bound +// (TotalRelation "gte"), because the scan genuinely did not count the rest; a +// caller that reads Total as exact would otherwise be silently misled. +func TestEarlyStopBoundedScan(t *testing.T) { + const n = 5000 + idx := buildEarlyStopIndex(t, n) + + for _, size := range []int{1, 10, 100} { + req := NewSearchRequest(earlyStopQuery()) + req.Size = size + req.Score = ScoreNone + res, err := idx.Search(req) + if err != nil { + t.Fatal(err) + } + if len(res.Hits) != size { + t.Errorf("size=%d: got %d hits, want %d — early stop dropped results the "+ + "caller asked for", size, len(res.Hits), size) + } + if res.TotalRelation != TotalRelationGte { + t.Errorf("size=%d: TotalRelation=%q, want %q — Total is a lower bound once "+ + "the scan stops early, and saying otherwise misleads the caller", + size, res.TotalRelation, TotalRelationGte) + } + if res.Total > uint64(n) { + t.Errorf("size=%d: Total=%d exceeds the corpus size %d", size, res.Total, n) + } + } +} + +// TestEarlyStopDoesNotEngageWhenUnsafe pins the preconditions. Each of these +// requests depends on documents the bounded scan would never look at, so it must +// keep draining and report an exact Total. Getting any of these wrong is a silent +// wrong-answer bug, not a slowdown: +// +// facets every match must be counted into the facet buckets +// field sort the top-k by field value can lie anywhere in the match set +// SearchAfter the cursor position depends on the full ordering +// scoring Score != "none" means order depends on scores, not arrival +func TestEarlyStopDoesNotEngageWhenUnsafe(t *testing.T) { + const n = 2000 + idx := buildEarlyStopIndex(t, n) + + cases := []struct { + name string + tweak func(*SearchRequest) + }{ + {"facets", func(r *SearchRequest) { + r.Score = ScoreNone + r.AddFacet("tags", NewFacetRequest("body", 5)) + }}, + {"field-sort", func(r *SearchRequest) { + r.Score = ScoreNone + r.SortBy([]string{"num"}) + }}, + {"search-after", func(r *SearchRequest) { + r.Score = ScoreNone + r.SortBy([]string{"_id"}) + r.SearchAfter = []string{"d00010"} + }}, + {"scoring-enabled", func(r *SearchRequest) { + // Score defaults to full scoring; ordering depends on scores. + }}, + } + + for _, tc := range cases { + req := NewSearchRequest(earlyStopQuery()) + req.Size = 10 + tc.tweak(req) + res, err := idx.Search(req) + if err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + if res.TotalRelation != TotalRelationEq { + t.Errorf("%s: TotalRelation=%q, want %q — the bounded scan engaged on a "+ + "request whose result depends on documents it would not visit", + tc.name, res.TotalRelation, TotalRelationEq) + } + if res.Total != uint64(n) { + t.Errorf("%s: Total=%d, want the exact %d", tc.name, res.Total, n) + } + } +} + +// TestEarlyStopHitsAreRealMatches guards the cheapest way for a bounded scan to be +// wrong: returning documents that do not match. Order is not part of the contract +// under Score="none", but membership is. +func TestEarlyStopHitsAreRealMatches(t *testing.T) { + idx := buildEarlyStopIndex(t, 1000) + + q := query.NewTermQuery("tag3") + q.SetField("body") + req := NewSearchRequest(q) + req.Size = 20 + req.Score = ScoreNone + res, err := idx.Search(req) + if err != nil { + t.Fatal(err) + } + if len(res.Hits) != 20 { + t.Fatalf("got %d hits, want 20", len(res.Hits)) + } + // tag3 was indexed on every doc where i%10 == 3. + for _, h := range res.Hits { + var i int + if _, err := fmt.Sscanf(h.ID, "d%05d", &i); err != nil { + t.Fatalf("unexpected id %q: %v", h.ID, err) + } + if i%10 != 3 { + t.Errorf("id %s does not match tag3 — the bounded scan returned a non-match", h.ID) + } + } +} diff --git a/error.go b/error.go new file mode 100644 index 0000000..e5ffe07 --- /dev/null +++ b/error.go @@ -0,0 +1,55 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +// Constant Error values which can be compared to determine the type of error +const ( + ErrorIndexPathExists Error = iota + ErrorIndexPathDoesNotExist + ErrorIndexMetaMissing + ErrorIndexMetaCorrupt + ErrorIndexClosed + ErrorAliasMulti + ErrorAliasEmpty + ErrorUnknownIndexType + ErrorEmptyID + ErrorIndexReadInconsistency + ErrorTwoPhaseSearchInconsistency + ErrorSynonymSearchNotSupported + ErrorTrainingNotSupported +) + +// Error represents a more strongly typed bleve error for detecting +// and handling specific types of errors. +type Error int + +func (e Error) Error() string { + return errorMessages[e] +} + +var errorMessages = map[Error]string{ + ErrorIndexPathExists: "cannot create new index, path already exists", + ErrorIndexPathDoesNotExist: "cannot open index, path does not exist", + ErrorIndexMetaMissing: "cannot open index, metadata missing", + ErrorIndexMetaCorrupt: "cannot open index, metadata corrupt", + ErrorIndexClosed: "index is closed", + ErrorAliasMulti: "cannot perform single index operation on multiple index alias", + ErrorAliasEmpty: "cannot perform operation on empty alias", + ErrorUnknownIndexType: "unknown index type", + ErrorEmptyID: "document ID cannot be empty", + ErrorIndexReadInconsistency: "index read inconsistency detected", + ErrorTwoPhaseSearchInconsistency: "2-phase search failed, likely due to an overlapping topology change", + ErrorSynonymSearchNotSupported: "synonym search not supported", +} diff --git a/examples_test.go b/examples_test.go new file mode 100644 index 0000000..fdd51ab --- /dev/null +++ b/examples_test.go @@ -0,0 +1,468 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "fmt" + "os" + "testing" + "time" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/highlight/highlighter/ansi" +) + +var indexMapping mapping.IndexMapping +var exampleIndex Index +var err error + +func TestMain(m *testing.M) { + err = os.RemoveAll("path_to_index") + if err != nil { + panic(err) + } + toRun := m.Run() + if exampleIndex != nil { + err = exampleIndex.Close() + if err != nil { + panic(err) + } + } + err = os.RemoveAll("path_to_index") + if err != nil { + panic(err) + } + os.Exit(toRun) +} + +func ExampleNew() { + indexMapping = NewIndexMapping() + exampleIndex, err = New("path_to_index", indexMapping) + if err != nil { + panic(err) + } + count, err := exampleIndex.DocCount() + if err != nil { + panic(err) + } + + fmt.Println(count) + // Output: + // 0 +} + +func ExampleIndex_indexing() { + data := struct { + Name string + Created time.Time + Age int + }{Name: "named one", Created: time.Now(), Age: 50} + data2 := struct { + Name string + Created time.Time + Age int + }{Name: "great nameless one", Created: time.Now(), Age: 25} + + // index some data + err = exampleIndex.Index("document id 1", data) + if err != nil { + panic(err) + } + err = exampleIndex.Index("document id 2", data2) + if err != nil { + panic(err) + } + + // 2 documents have been indexed + count, err := exampleIndex.DocCount() + if err != nil { + panic(err) + } + + fmt.Println(count) + // Output: + // 2 +} + +// Examples for query related functions + +func ExampleNewMatchQuery() { + // finds documents with fields fully matching the given query text + query := NewMatchQuery("named one") + searchRequest := NewSearchRequest(query) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + fmt.Println(searchResults.Hits[0].ID) + // Output: + // document id 1 +} + +func ExampleNewMatchAllQuery() { + // finds all documents in the index + query := NewMatchAllQuery() + searchRequest := NewSearchRequest(query) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + fmt.Println(len(searchResults.Hits)) + // Output: + // 2 +} + +func ExampleNewMatchNoneQuery() { + // matches no documents in the index + query := NewMatchNoneQuery() + searchRequest := NewSearchRequest(query) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + fmt.Println(len(searchResults.Hits)) + // Output: + // 0 +} + +func ExampleNewMatchPhraseQuery() { + // finds all documents with the given phrase in the index + query := NewMatchPhraseQuery("nameless one") + searchRequest := NewSearchRequest(query) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + fmt.Println(searchResults.Hits[0].ID) + // Output: + // document id 2 +} + +func ExampleNewNumericRangeQuery() { + value1 := float64(11) + value2 := float64(100) + data := struct{ Priority float64 }{Priority: float64(15)} + data2 := struct{ Priority float64 }{Priority: float64(10)} + + err = exampleIndex.Index("document id 3", data) + if err != nil { + panic(err) + } + err = exampleIndex.Index("document id 4", data2) + if err != nil { + panic(err) + } + + query := NewNumericRangeQuery(&value1, &value2) + searchRequest := NewSearchRequest(query) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + fmt.Println(searchResults.Hits[0].ID) + // Output: + // document id 3 +} + +func ExampleNewNumericRangeInclusiveQuery() { + value1 := float64(10) + value2 := float64(100) + v1incl := false + v2incl := false + + query := NewNumericRangeInclusiveQuery(&value1, &value2, &v1incl, &v2incl) + searchRequest := NewSearchRequest(query) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + fmt.Println(searchResults.Hits[0].ID) + // Output: + // document id 3 +} + +func ExampleNewPhraseQuery() { + // finds all documents with the given phrases in the given field in the index + query := NewPhraseQuery([]string{"nameless", "one"}, "Name") + searchRequest := NewSearchRequest(query) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + fmt.Println(searchResults.Hits[0].ID) + // Output: + // document id 2 +} + +func ExampleNewPrefixQuery() { + // finds all documents with terms having the given prefix in the index + query := NewPrefixQuery("name") + searchRequest := NewSearchRequest(query) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + fmt.Println(len(searchResults.Hits)) + // Output: + // 2 +} + +func ExampleNewQueryStringQuery() { + query := NewQueryStringQuery("+one -great") + searchRequest := NewSearchRequest(query) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + fmt.Println(searchResults.Hits[0].ID) + // Output: + // document id 1 +} + +func ExampleNewTermQuery() { + query := NewTermQuery("great") + searchRequest := NewSearchRequest(query) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + fmt.Println(searchResults.Hits[0].ID) + // Output: + // document id 2 +} + +func ExampleNewFacetRequest() { + facet := NewFacetRequest("Name", 1) + query := NewMatchAllQuery() + searchRequest := NewSearchRequest(query) + searchRequest.AddFacet("facet name", facet) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + // total number of terms + fmt.Println(searchResults.Facets["facet name"].Total) + // number of docs with no value for this field + fmt.Println(searchResults.Facets["facet name"].Missing) + // term with highest occurrences in field name + fmt.Println(searchResults.Facets["facet name"].Terms.Terms()[0].Term) + // Output: + // 5 + // 2 + // one +} + +func ExampleFacetRequest_AddDateTimeRange() { + facet := NewFacetRequest("Created", 1) + facet.AddDateTimeRange("range name", time.Unix(0, 0), time.Now()) + query := NewMatchAllQuery() + searchRequest := NewSearchRequest(query) + searchRequest.AddFacet("facet name", facet) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + // dates in field Created since starting of unix time till now + fmt.Println(searchResults.Facets["facet name"].DateRanges[0].Count) + // Output: + // 2 +} + +func ExampleFacetRequest_AddNumericRange() { + value1 := float64(11) + + facet := NewFacetRequest("Priority", 1) + facet.AddNumericRange("range name", &value1, nil) + query := NewMatchAllQuery() + searchRequest := NewSearchRequest(query) + searchRequest.AddFacet("facet name", facet) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + // number documents with field Priority in the given range + fmt.Println(searchResults.Facets["facet name"].NumericRanges[0].Count) + // Output: + // 1 +} + +func ExampleNewHighlight() { + query := NewMatchQuery("nameless") + searchRequest := NewSearchRequest(query) + searchRequest.Highlight = NewHighlight() + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + fmt.Println(searchResults.Hits[0].Fragments["Name"][0]) + // Output: + // great nameless one +} + +func ExampleNewHighlightWithStyle() { + query := NewMatchQuery("nameless") + searchRequest := NewSearchRequest(query) + searchRequest.Highlight = NewHighlightWithStyle(ansi.Name) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + fmt.Println(searchResults.Hits[0].Fragments["Name"][0]) + // Output: + // great nameless one +} + +func ExampleSearchRequest_AddFacet() { + facet := NewFacetRequest("Name", 1) + query := NewMatchAllQuery() + searchRequest := NewSearchRequest(query) + searchRequest.AddFacet("facet name", facet) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + // total number of terms + fmt.Println(searchResults.Facets["facet name"].Total) + // number of docs with no value for this field + fmt.Println(searchResults.Facets["facet name"].Missing) + // term with highest occurrences in field name + fmt.Println(searchResults.Facets["facet name"].Terms.Terms()[0].Term) + // Output: + // 5 + // 2 + // one +} + +func ExampleNewSearchRequest() { + // finds documents with fields fully matching the given query text + query := NewMatchQuery("named one") + searchRequest := NewSearchRequest(query) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + fmt.Println(searchResults.Hits[0].ID) + // Output: + // document id 1 +} + +func ExampleNewBooleanQuery() { + must := NewMatchQuery("one") + mustNot := NewMatchQuery("great") + query := NewBooleanQuery() + query.AddMust(must) + query.AddMustNot(mustNot) + searchRequest := NewSearchRequest(query) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + fmt.Println(searchResults.Hits[0].ID) + // Output: + // document id 1 +} + +func ExampleNewConjunctionQuery() { + conjunct1 := NewMatchQuery("great") + conjunct2 := NewMatchQuery("one") + query := NewConjunctionQuery(conjunct1, conjunct2) + searchRequest := NewSearchRequest(query) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + fmt.Println(searchResults.Hits[0].ID) + // Output: + // document id 2 +} + +func ExampleNewDisjunctionQuery() { + disjunct1 := NewMatchQuery("great") + disjunct2 := NewMatchQuery("named") + query := NewDisjunctionQuery(disjunct1, disjunct2) + searchRequest := NewSearchRequest(query) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + fmt.Println(len(searchResults.Hits)) + // Output: + // 2 +} + +func ExampleSearchRequest_SortBy() { + // find docs containing "one", order by Age instead of score + query := NewMatchQuery("one") + searchRequest := NewSearchRequest(query) + searchRequest.SortBy([]string{"Age"}) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + fmt.Println(searchResults.Hits[0].ID) + fmt.Println(searchResults.Hits[1].ID) + // Output: + // document id 2 + // document id 1 +} + +func ExampleSearchRequest_SortByCustom() { + // find all docs, order by Age, with docs missing Age field first + query := NewMatchAllQuery() + searchRequest := NewSearchRequest(query) + searchRequest.SortByCustom(search.SortOrder{ + &search.SortField{ + Field: "Age", + Missing: search.SortFieldMissingFirst, + }, + &search.SortDocID{}, + }) + searchResults, err := exampleIndex.Search(searchRequest) + if err != nil { + panic(err) + } + + fmt.Println(searchResults.Hits[0].ID) + fmt.Println(searchResults.Hits[1].ID) + fmt.Println(searchResults.Hits[2].ID) + fmt.Println(searchResults.Hits[3].ID) + // Output: + // document id 3 + // document id 4 + // document id 2 + // document id 1 +} diff --git a/fusion/fusion.go b/fusion/fusion.go new file mode 100644 index 0000000..ce66865 --- /dev/null +++ b/fusion/fusion.go @@ -0,0 +1,26 @@ +// Copyright (c) 2025 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fusion + +import ( + "github.com/blevesearch/bleve/v2/search" +) + +type FusionResult struct { + Hits search.DocumentMatchCollection + Total uint64 + MaxScore float64 +} diff --git a/fusion/rrf.go b/fusion/rrf.go new file mode 100644 index 0000000..d9ca701 --- /dev/null +++ b/fusion/rrf.go @@ -0,0 +1,143 @@ +// Copyright (c) 2025 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fusion + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/search" +) + +// formatRRFMessage builds the explanation string for a single component of the +// Reciprocal Rank Fusion calculation. +func formatRRFMessage(weight float64, rank int, rankConstant int) string { + return fmt.Sprintf("rrf score (weight=%.3f, rank=%d, rank_constant=%d), normalized score of", weight, rank, rankConstant) +} + +// ReciprocalRankFusion applies Reciprocal Rank Fusion across the primary FTS +// results and each KNN sub-query. Ranks are limited to `windowSize` per source, +// weighted, and combined into a single fused score, with optional explanation +// details. +func ReciprocalRankFusion(hits search.DocumentMatchCollection, weights []float64, rankConstant int, windowSize int, numKNNQueries int, explain bool) *FusionResult { + nHits := len(hits) + if nHits == 0 || windowSize == 0 { + return &FusionResult{ + Hits: search.DocumentMatchCollection{}, + Total: 0, + MaxScore: 0.0, + } + } + + limit := min(nHits, windowSize) + + // precompute rank+scores to prevent additional division ops later + rankReciprocals := make([]float64, limit) + for i := range rankReciprocals { + rankReciprocals[i] = 1.0 / float64(rankConstant+i+1) + } + + // init explanations if required + var fusionExpl map[*search.DocumentMatch][]*search.Explanation + if explain { + fusionExpl = make(map[*search.DocumentMatch][]*search.Explanation, nHits) + } + + // The code here mainly deals with obtaining rank/score for fts hits. + // First sort hits by score + sortDocMatchesByScore(hits) + + // Calculate fts rank+scores + ftsWeight := weights[0] + for i := 0; i < nHits; i++ { + if i < windowSize { + hit := hits[i] + + // No fts scores from this hit onwards, break loop + if hit.Score == 0.0 { + break + } + + contrib := ftsWeight * rankReciprocals[i] + hit.Score = contrib + + if explain { + expl := getFusionExplAt( + hit, + 0, + contrib, + formatRRFMessage(ftsWeight, i+1, rankConstant), + ) + fusionExpl[hit] = append(fusionExpl[hit], expl) + } + } else { + // These FTS hits are not counted in the results, so set to 0 + hits[i].Score = 0.0 + } + } + + // Code from here is to calculate knn ranks and scores + // iterate over each knn query and calculate knn rank+scores + for queryIdx := 0; queryIdx < numKNNQueries; queryIdx++ { + knnWeight := weights[queryIdx+1] + // Sorts hits in decreasing order of hit.ScoreBreakdown[i] + sortDocMatchesByBreakdown(hits, queryIdx) + + for i := 0; i < nHits; i++ { + // break if score breakdown doesn't exist (sort function puts these hits at the end) + // or if we go past the windowSize + _, scoreBreakdownExists := scoreBreakdownForQuery(hits[i], queryIdx) + if i >= windowSize || !scoreBreakdownExists { + break + } + + hit := hits[i] + contrib := knnWeight * rankReciprocals[i] + hit.Score += contrib + + if explain { + expl := getFusionExplAt( + hit, + queryIdx+1, + contrib, + formatRRFMessage(knnWeight, i+1, rankConstant), + ) + fusionExpl[hit] = append(fusionExpl[hit], expl) + } + } + } + + var maxScore float64 + for _, hit := range hits { + if explain { + finalizeFusionExpl(hit, fusionExpl[hit]) + } + hit.ScoreBreakdown = nil + + if hit.Score > maxScore { + maxScore = hit.Score + } + } + + sortDocMatchesByScore(hits) + if nHits > windowSize { + hits = hits[:windowSize] + } + return &FusionResult{ + Hits: hits, + Total: uint64(len(hits)), + MaxScore: maxScore, + } +} diff --git a/fusion/rrf_test.go b/fusion/rrf_test.go new file mode 100644 index 0000000..831fa10 --- /dev/null +++ b/fusion/rrf_test.go @@ -0,0 +1,212 @@ +// Copyright (c) 2025 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fusion + +import ( + "math" + "testing" + + "github.com/blevesearch/bleve/v2/search" +) + +const epsilon float64 = 1e-3 + +func nearlyEqual(a float64, b float64, epsilon float64) bool { + return math.Abs(a-b) < epsilon +} + +func compareFusionResults(a, b FusionResult) bool { + if a.Total != b.Total || !nearlyEqual(a.MaxScore, b.MaxScore, epsilon) || len(a.Hits) != len(b.Hits) { + return false + } + for i := range a.Hits { + if a.Hits[i].ID != b.Hits[i].ID || !nearlyEqual(a.Hits[i].Score, b.Hits[i].Score, epsilon) { + return false + } + + if a.Hits[i].ScoreBreakdown != nil || b.Hits[i].ScoreBreakdown != nil { + return false + } + } + return true +} + +func TestReciprocalRankFusion(t *testing.T) { + tests := []struct { + name string + hits search.DocumentMatchCollection + weights []float64 + rank_constant int + window_size int + numKNNQueries int + want FusionResult + }{ + { + name: "empty hits", + hits: search.DocumentMatchCollection{}, + weights: []float64{0.5, 0.5}, + rank_constant: 60, + window_size: 10, + numKNNQueries: 1, + want: FusionResult{ + Hits: search.DocumentMatchCollection{}, + Total: 0, + MaxScore: 0.0, + }, + }, + { + name: "single knn query", + hits: search.DocumentMatchCollection{ + {ID: "a", Score: 0.9, ScoreBreakdown: map[int]float64{0: 0.8}}, + {ID: "b", Score: 0.8, ScoreBreakdown: map[int]float64{0: 0.9}}, + {ID: "c", Score: 0.7, ScoreBreakdown: map[int]float64{0: 0.7}}, + }, + weights: []float64{0.4, 0.6}, + rank_constant: 1, + window_size: 3, + numKNNQueries: 1, + want: FusionResult{ + Hits: search.DocumentMatchCollection{ + {ID: "b", Score: 0.433}, + {ID: "a", Score: 0.4}, + {ID: "c", Score: 0.25}, + }, + Total: 3, + MaxScore: 0.433, + }, + }, + { + name: "multiple knn queries", + hits: search.DocumentMatchCollection{ + {ID: "a", Score: 0.9, ScoreBreakdown: map[int]float64{0: 0.8, 1: 0.6}}, + {ID: "b", Score: 0.8, ScoreBreakdown: map[int]float64{0: 0.9, 1: 0.5}}, + {ID: "c", Score: 0.7, ScoreBreakdown: map[int]float64{0: 0.7, 1: 0.7}}, + }, + weights: []float64{0.3, 0.4, 0.3}, + rank_constant: 1, + window_size: 3, + numKNNQueries: 2, + want: FusionResult{ + Hits: search.DocumentMatchCollection{ + {ID: "a", Score: 0.383}, + {ID: "b", Score: 0.375}, + {ID: "c", Score: 0.325}, + }, + Total: 3, + MaxScore: 0.383, + }, + }, + { + name: "window size smaller than hits", + hits: search.DocumentMatchCollection{ + {ID: "a", Score: 0.9, ScoreBreakdown: map[int]float64{0: 0.7}}, + {ID: "b", Score: 0.8, ScoreBreakdown: map[int]float64{0: 0.9}}, + {ID: "c", Score: 0.7, ScoreBreakdown: map[int]float64{0: 0.8}}, + }, + weights: []float64{0.4, 0.6}, + rank_constant: 1, + window_size: 2, + numKNNQueries: 1, + want: FusionResult{ + Hits: search.DocumentMatchCollection{ + {ID: "b", Score: 0.433}, + {ID: "a", Score: 0.2}, + }, + Total: 2, + MaxScore: 0.433, + }, + }, + { + name: "documents with partial scores missing KNN scores", + hits: search.DocumentMatchCollection{ + {ID: "a", Score: 0.9, ScoreBreakdown: map[int]float64{0: 0.8}}, // has FTS and KNN query 0, missing KNN query 1 + {ID: "b", Score: 0.8, ScoreBreakdown: map[int]float64{1: 0.7}}, // has FTS and KNN query 1, missing KNN query 0 + {ID: "c", Score: 0.7, ScoreBreakdown: map[int]float64{0: 0.6, 1: 0.9}}, // has all scores + {ID: "d", Score: 0.6, ScoreBreakdown: map[int]float64{}}, // has only FTS, missing all KNN scores + }, + weights: []float64{0.3, 0.4, 0.3}, // FTS, KNN query 0, KNN query 1 + rank_constant: 1, + window_size: 4, + numKNNQueries: 2, + want: FusionResult{ + Hits: search.DocumentMatchCollection{ + {ID: "c", Score: 0.358}, // FTS rank 3, KNN0 rank 2, KNN1 rank 1: 0.3/4 + 0.4/3 + 0.3/2 = 0.075 + 0.133 + 0.15 = 0.358 + {ID: "a", Score: 0.35}, // FTS rank 1, KNN0 rank 1, no KNN1: 0.3/2 + 0.4/2 + 0 = 0.15 + 0.2 + 0 = 0.35 + {ID: "b", Score: 0.2}, // FTS rank 2, no KNN0, KNN1 rank 2: 0.3/3 + 0 + 0.3/3 = 0.1 + 0 + 0.1 = 0.2 + {ID: "d", Score: 0.06}, // FTS rank 4, no KNN0, no KNN1: 0.3/5 + 0 + 0 = 0.06 + }, + Total: 4, + MaxScore: 0.358, + }, + }, + { + name: "documents with only KNN scores", + hits: search.DocumentMatchCollection{ + {ID: "a", Score: 0.0, ScoreBreakdown: map[int]float64{0: 0.9}}, // no FTS rank (Score 0.0), only KNN query 0 + {ID: "b", Score: 0.0, ScoreBreakdown: map[int]float64{1: 0.8}}, // no FTS rank (Score 0.0), only KNN query 1 + {ID: "c", Score: 0.0, ScoreBreakdown: map[int]float64{0: 0.7, 1: 0.6}}, // no FTS rank (Score 0.0), both KNN queries + }, + weights: []float64{0.5, 0.3, 0.2}, // FTS, KNN query 0, KNN query 1 + rank_constant: 1, + window_size: 3, + numKNNQueries: 2, + want: FusionResult{ + Hits: search.DocumentMatchCollection{ + {ID: "c", Score: 0.167}, // no FTS rank, KNN0 rank 2, KNN1 rank 2: 0 + 0.3/3 + 0.2/3 = 0 + 0.1 + 0.067 = 0.167 + {ID: "a", Score: 0.15}, // no FTS rank, KNN0 rank 1, no KNN1: 0 + 0.3/2 + 0 = 0 + 0.15 + 0 = 0.15 + {ID: "b", Score: 0.1}, // no FTS rank, no KNN0, KNN1 rank 1: 0 + 0 + 0.2/2 = 0 + 0 + 0.1 = 0.1 + }, + Total: 3, + MaxScore: 0.167, + }, + }, + { + name: "mixed scenario with gaps in KNN queries", + hits: search.DocumentMatchCollection{ + {ID: "a", Score: 0.8, ScoreBreakdown: map[int]float64{1: 0.9}}, // has FTS and KNN query 1, missing KNN query 0 + {ID: "b", Score: 0.6, ScoreBreakdown: map[int]float64{0: 0.8}}, // has FTS and KNN query 0, missing KNN query 1 + {ID: "c", Score: 0.0, ScoreBreakdown: map[int]float64{0: 0.7}}, // no FTS rank (Score 0.0), only KNN query 0 + {ID: "d", Score: 0.4, ScoreBreakdown: map[int]float64{}}, // only FTS, no KNN scores + }, + weights: []float64{0.4, 0.3, 0.3}, // FTS, KNN query 0, KNN query 1 + rank_constant: 1, + window_size: 4, + numKNNQueries: 2, + want: FusionResult{ + Hits: search.DocumentMatchCollection{ + {ID: "a", Score: 0.35}, // FTS rank 1, no KNN0, KNN1 rank 1: 0.4/2 + 0 + 0.3/2 = 0.2 + 0 + 0.15 = 0.35 + {ID: "b", Score: 0.283}, // FTS rank 2, KNN0 rank 1, no KNN1: 0.4/3 + 0.3/2 + 0 = 0.133 + 0.15 + 0 = 0.283 + {ID: "d", Score: 0.1}, // FTS rank 3, no KNN0, no KNN1: 0.4/4 + 0 + 0 = 0.1 + {ID: "c", Score: 0.1}, // no FTS rank, KNN0 rank 2, no KNN1: 0 + 0.3/3 + 0 = 0 + 0.1 + 0 = 0.1 + }, + Total: 4, + MaxScore: 0.35, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for i, hit := range tt.hits { + hit.HitNumber = uint64(i) + } + + if got := ReciprocalRankFusion(tt.hits, tt.weights, tt.rank_constant, tt.window_size, tt.numKNNQueries, false); !compareFusionResults(*got, tt.want) { + t.Errorf("ReciprocalRankFusion() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/fusion/rsf.go b/fusion/rsf.go new file mode 100644 index 0000000..af6b276 --- /dev/null +++ b/fusion/rsf.go @@ -0,0 +1,164 @@ +// Copyright (c) 2025 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fusion + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/search" +) + +// formatRSFMessage builds the explanation string associated with a single +// component of the Relative Score Fusion calculation. +func formatRSFMessage(weight float64, normalizedScore float64, minScore float64, maxScore float64) string { + return fmt.Sprintf("rsf score (weight=%.3f, normalized=%.6f, min=%.6f, max=%.6f), normalized score of", + weight, normalizedScore, minScore, maxScore) +} + +// RelativeScoreFusion normalizes the best-scoring documents from the primary +// FTS query and each KNN query, scales those normalized values by the supplied +// weights, and combines them into a single fused score. Only the top +// `windowSize` documents per source are considered, and explanations are +// materialized lazily when requested. +func RelativeScoreFusion(hits search.DocumentMatchCollection, weights []float64, windowSize int, numKNNQueries int, explain bool) *FusionResult { + nHits := len(hits) + if nHits == 0 || windowSize == 0 { + return &FusionResult{ + Hits: search.DocumentMatchCollection{}, + Total: 0, + MaxScore: 0.0, + } + } + + // init explanations if required + var fusionExpl map[*search.DocumentMatch][]*search.Explanation + if explain { + fusionExpl = make(map[*search.DocumentMatch][]*search.Explanation, nHits) + } + + // Code here for calculating fts results + // Sort by fts scores + sortDocMatchesByScore(hits) + + // ftsLimit holds the total number of fts hits to consider for rsf + ftsLimit := 0 + for _, hit := range hits { + if hit.Score == 0.0 { + break + } + ftsLimit++ + } + ftsLimit = min(ftsLimit, windowSize) + + // calculate fts scores + if ftsLimit > 0 { + max := hits[0].Score + min := hits[ftsLimit-1].Score + denom := max - min + weight := weights[0] + + for i := 0; i < ftsLimit; i++ { + hit := hits[i] + norm := 1.0 + if denom > 0 { + norm = (hit.Score - min) / denom + } + contrib := weight * norm + if explain { + expl := getFusionExplAt( + hit, + 0, + norm, + formatRSFMessage(weight, norm, min, max), + ) + fusionExpl[hit] = append(fusionExpl[hit], expl) + } + hit.Score = contrib + } + for i := ftsLimit; i < nHits; i++ { + // These FTS hits are not counted in the results, so set to 0 + hits[i].Score = 0.0 + } + } + + // Code from here is for calculating knn scores + for queryIdx := 0; queryIdx < numKNNQueries; queryIdx++ { + sortDocMatchesByBreakdown(hits, queryIdx) + + // knnLimit holds the total number of knn hits retrieved for a specific knn query + knnLimit := 0 + for _, hit := range hits { + if _, ok := scoreBreakdownForQuery(hit, queryIdx); !ok { + break + } + knnLimit++ + } + knnLimit = min(knnLimit, windowSize) + + // if limit is 0, skip calculating + if knnLimit == 0 { + continue + } + + max, _ := scoreBreakdownForQuery(hits[0], queryIdx) + min, _ := scoreBreakdownForQuery(hits[knnLimit-1], queryIdx) + denom := max - min + weight := weights[queryIdx+1] + + for i := 0; i < knnLimit; i++ { + hit := hits[i] + score, _ := scoreBreakdownForQuery(hit, queryIdx) + norm := 1.0 + if denom > 0 { + norm = (score - min) / denom + } + contrib := weight * norm + if explain { + expl := getFusionExplAt( + hit, + queryIdx+1, + norm, + formatRSFMessage(weight, norm, min, max), + ) + fusionExpl[hit] = append(fusionExpl[hit], expl) + } + hit.Score += contrib + } + } + + // Finalize scores + var maxScore float64 + for _, hit := range hits { + if explain { + finalizeFusionExpl(hit, fusionExpl[hit]) + } + if hit.Score > maxScore { + maxScore = hit.Score + } + hit.ScoreBreakdown = nil + } + + sortDocMatchesByScore(hits) + + if nHits > windowSize { + hits = hits[:windowSize] + } + + return &FusionResult{ + Hits: hits, + Total: uint64(len(hits)), + MaxScore: maxScore, + } +} diff --git a/fusion/rsf_test.go b/fusion/rsf_test.go new file mode 100644 index 0000000..4fb67f6 --- /dev/null +++ b/fusion/rsf_test.go @@ -0,0 +1,209 @@ +// Copyright (c) 2025 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fusion + +import ( + "testing" + + "github.com/blevesearch/bleve/v2/search" +) + +func TestRelativeScoreFusion(t *testing.T) { + tests := []struct { + name string + hits search.DocumentMatchCollection + weights []float64 + windowSize int + numKNNQueries int + want FusionResult + }{ + { + name: "empty hits", + hits: search.DocumentMatchCollection{}, + weights: []float64{0.5, 0.5}, + windowSize: 10, + numKNNQueries: 1, + want: FusionResult{ + Hits: search.DocumentMatchCollection{}, + Total: 0, + MaxScore: 0.0, + }, + }, + { + name: "single knn query", + hits: search.DocumentMatchCollection{ + {ID: "a", Score: 0.9, ScoreBreakdown: map[int]float64{0: 0.8}}, + {ID: "b", Score: 0.8, ScoreBreakdown: map[int]float64{0: 0.9}}, + {ID: "c", Score: 0.7, ScoreBreakdown: map[int]float64{0: 0.7}}, + }, + weights: []float64{0.4, 0.6}, + windowSize: 3, + numKNNQueries: 1, + want: FusionResult{ + Hits: search.DocumentMatchCollection{ + {ID: "b", Score: 0.8}, // FTS: (0.8-0.7)/(0.9-0.7) * 0.4 + KNN: (0.9-0.7)/(0.9-0.7) * 0.6 = 0.2 + 0.6 = 0.8 + {ID: "a", Score: 0.7}, // FTS: (0.9-0.7)/(0.9-0.7) * 0.4 + KNN: (0.8-0.7)/(0.9-0.7) * 0.6 = 0.4 + 0.3 = 0.7 + {ID: "c", Score: 0.0}, // FTS: (0.7-0.7)/(0.9-0.7) * 0.4 + KNN: (0.7-0.7)/(0.9-0.7) * 0.6 = 0.0 + 0.0 = 0.0 + }, + Total: 3, + MaxScore: 0.8, + }, + }, + { + name: "multiple knn queries", + hits: search.DocumentMatchCollection{ + {ID: "a", Score: 0.9, ScoreBreakdown: map[int]float64{0: 0.8, 1: 0.6}}, + {ID: "b", Score: 0.8, ScoreBreakdown: map[int]float64{0: 0.9, 1: 0.5}}, + {ID: "c", Score: 0.7, ScoreBreakdown: map[int]float64{0: 0.7, 1: 0.7}}, + }, + weights: []float64{0.3, 0.4, 0.3}, + windowSize: 3, + numKNNQueries: 2, + want: FusionResult{ + Hits: search.DocumentMatchCollection{ + {ID: "a", Score: 0.65}, // FTS: (0.9-0.7)/(0.9-0.7)*0.3 + KNN0: (0.8-0.7)/(0.9-0.7)*0.4 + KNN1: (0.6-0.5)/(0.7-0.5)*0.3 = 1.0*0.3 + 0.5*0.4 + 0.5*0.3 = 0.65 + {ID: "b", Score: 0.55}, // FTS: (0.8-0.7)/(0.9-0.7)*0.3 + KNN0: (0.9-0.7)/(0.9-0.7)*0.4 + KNN1: (0.5-0.5)/(0.7-0.5)*0.3 = 0.5*0.3 + 1.0*0.4 + 0.0*0.3 = 0.55 + {ID: "c", Score: 0.3}, // FTS: (0.7-0.7)/(0.9-0.7)*0.3 + KNN0: (0.7-0.7)/(0.9-0.7)*0.4 + KNN1: (0.7-0.5)/(0.7-0.5)*0.3 = 0.0*0.3 + 0.0*0.4 + 1.0*0.3 = 0.3 + }, + Total: 3, + MaxScore: 0.65, + }, + }, + { + name: "all scores identical should normalize to 1.0", + hits: search.DocumentMatchCollection{ + {ID: "a", Score: 0.8, ScoreBreakdown: map[int]float64{0: 0.9}}, + {ID: "b", Score: 0.8, ScoreBreakdown: map[int]float64{0: 0.9}}, + {ID: "c", Score: 0.8, ScoreBreakdown: map[int]float64{0: 0.9}}, + }, + weights: []float64{0.4, 0.6}, + windowSize: 3, + numKNNQueries: 1, + want: FusionResult{ + Hits: search.DocumentMatchCollection{ + {ID: "a", Score: 1.0}, // All scores identical: 1.0 * 0.4 + 1.0 * 0.6 = 1.0 + {ID: "b", Score: 1.0}, + {ID: "c", Score: 1.0}, + }, + Total: 3, + MaxScore: 1.0, + }, + }, + { + name: "window size smaller than hits", + hits: search.DocumentMatchCollection{ + {ID: "a", Score: 0.9, ScoreBreakdown: map[int]float64{0: 0.7}}, + {ID: "b", Score: 0.8, ScoreBreakdown: map[int]float64{0: 0.9}}, + {ID: "c", Score: 0.7, ScoreBreakdown: map[int]float64{0: 0.8}}, + }, + weights: []float64{0.4, 0.6}, + windowSize: 2, + numKNNQueries: 1, + want: FusionResult{ + Hits: search.DocumentMatchCollection{ + {ID: "b", Score: 0.6}, // Using top 2 for min/max: FTS min/max from [0.9,0.8] = [0.8,0.9], KNN min/max from [0.9,0.7] = [0.7,0.9] + {ID: "a", Score: 0.4}, // FTS: (0.9-0.8)/(0.9-0.8) * 0.4 + KNN: (0.7-0.7)/(0.9-0.7) * 0.6 = 0.4 + 0 = 0.4 + }, + Total: 2, + MaxScore: 0.6, + }, + }, + { + name: "documents with partial scores missing KNN scores", + hits: search.DocumentMatchCollection{ + {ID: "a", Score: 0.9, ScoreBreakdown: map[int]float64{0: 0.8}}, // has FTS and KNN query 0, missing KNN query 1 + {ID: "b", Score: 0.8, ScoreBreakdown: map[int]float64{1: 0.7}}, // has FTS and KNN query 1, missing KNN query 0 + {ID: "c", Score: 0.7, ScoreBreakdown: map[int]float64{0: 0.6, 1: 0.9}}, // has all scores + {ID: "d", Score: 0.6, ScoreBreakdown: map[int]float64{}}, // has only FTS, missing all KNN scores + }, + weights: []float64{0.3, 0.4, 0.3}, // FTS, KNN query 0, KNN query 1 + windowSize: 4, + numKNNQueries: 2, + want: FusionResult{ + Hits: search.DocumentMatchCollection{ + {ID: "a", Score: 0.7}, // FTS: (0.9-0.6)/(0.9-0.6)*0.3 + KNN0: (0.8-0.6)/(0.8-0.6)*0.4 + KNN1: 0 = 1.0*0.3 + 1.0*0.4 + 0 = 0.7 + {ID: "c", Score: 0.4}, // FTS: (0.7-0.6)/(0.9-0.6)*0.3 + KNN0: (0.6-0.6)/(0.8-0.6)*0.4 + KNN1: (0.9-0.7)/(0.9-0.7)*0.3 = 0.33*0.3 + 0.0*0.4 + 1.0*0.3 = 0.1 + 0 + 0.3 = 0.4 + {ID: "b", Score: 0.2}, // FTS: (0.8-0.6)/(0.9-0.6)*0.3 + KNN0: 0 + KNN1: (0.7-0.7)/(0.9-0.7)*0.3 = 0.67*0.3 + 0 + 0.0*0.3 = 0.2 + 0 + 0 = 0.2 + {ID: "d", Score: 0.0}, // FTS: (0.6-0.6)/(0.9-0.6)*0.3 + KNN0: 0 + KNN1: 0 = 0.0*0.3 + 0 + 0 = 0 + }, + Total: 4, + MaxScore: 0.7, + }, + }, + { + name: "documents with only KNN scores", + hits: search.DocumentMatchCollection{ + {ID: "a", Score: 0.0, ScoreBreakdown: map[int]float64{0: 0.9}}, // no FTS rank (Score 0.0), only KNN query 0 + {ID: "b", Score: 0.0, ScoreBreakdown: map[int]float64{1: 0.8}}, // no FTS rank (Score 0.0), only KNN query 1 + {ID: "c", Score: 0.0, ScoreBreakdown: map[int]float64{0: 0.7, 1: 0.6}}, // no FTS rank (Score 0.0), both KNN queries + }, + weights: []float64{0.5, 0.3, 0.2}, // FTS, KNN query 0, KNN query 1 + windowSize: 3, + numKNNQueries: 2, + want: FusionResult{ + Hits: search.DocumentMatchCollection{ + {ID: "a", Score: 0.3}, // FTS: 0 + KNN0: 1.0 * 0.3 + KNN1: 0 = 0.3 + {ID: "b", Score: 0.2}, // FTS: 0 + KNN0: 0 + KNN1: 1.0 * 0.2 = 0.2 + {ID: "c", Score: 0.0}, // FTS: 0 + KNN0: 0 * 0.3 + KNN1: 0 * 0.2 = 0 + }, + Total: 3, + MaxScore: 0.3, + }, + }, + { + name: "mixed scenario with different score ranges", + hits: search.DocumentMatchCollection{ + {ID: "a", Score: 1.0, ScoreBreakdown: map[int]float64{0: 0.1}}, // high FTS, low KNN + {ID: "b", Score: 0.1, ScoreBreakdown: map[int]float64{0: 1.0}}, // low FTS, high KNN + {ID: "c", Score: 0.5, ScoreBreakdown: map[int]float64{0: 0.5}}, // mid FTS, mid KNN + }, + weights: []float64{0.5, 0.5}, // Equal weights + windowSize: 3, + numKNNQueries: 1, + want: FusionResult{ + Hits: search.DocumentMatchCollection{ + {ID: "a", Score: 0.5}, // FTS: (1.0-0.1)/(1.0-0.1)*0.5 + KNN: (0.1-0.1)/(1.0-0.1)*0.5 = 1.0*0.5 + 0.0*0.5 = 0.5 + {ID: "b", Score: 0.5}, // FTS: (0.1-0.1)/(1.0-0.1)*0.5 + KNN: (1.0-0.1)/(1.0-0.1)*0.5 = 0.0*0.5 + 1.0*0.5 = 0.5 + {ID: "c", Score: 0.444}, // FTS: (0.5-0.1)/(1.0-0.1)*0.5 + KNN: (0.5-0.1)/(1.0-0.1)*0.5 = 0.444*0.5 + 0.444*0.5 = 0.444 + }, + Total: 3, + MaxScore: 0.5, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for i, hit := range tt.hits { + hit.HitNumber = uint64(i) + } + + if got := RelativeScoreFusion(tt.hits, tt.weights, tt.windowSize, tt.numKNNQueries, false); !compareFusionResults(*got, tt.want) { + t.Errorf("RelativeScoreFusion() = %v, want %v", got, tt.want) + // Print detailed comparison for debugging + t.Logf("Got hits:") + for i, hit := range got.Hits { + t.Logf(" [%d] ID: %s, Score: %.6f", i, hit.ID, hit.Score) + } + t.Logf("Want hits:") + for i, hit := range tt.want.Hits { + t.Logf(" [%d] ID: %s, Score: %.6f", i, hit.ID, hit.Score) + } + t.Logf("Got Total: %d, MaxScore: %.6f", got.Total, got.MaxScore) + t.Logf("Want Total: %d, MaxScore: %.6f", tt.want.Total, tt.want.MaxScore) + } + }) + } +} diff --git a/fusion/util.go b/fusion/util.go new file mode 100644 index 0000000..8e76487 --- /dev/null +++ b/fusion/util.go @@ -0,0 +1,111 @@ +// Copyright (c) 2025 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fusion + +import ( + "sort" + + "github.com/blevesearch/bleve/v2/search" +) + +// sortDocMatchesByScore orders the provided collection in-place by the primary +// score in descending order, breaking ties with the original `HitNumber` to +// ensure deterministic output. +func sortDocMatchesByScore(hits search.DocumentMatchCollection) { + if len(hits) < 2 { + return + } + + sort.Slice(hits, func(a, b int) bool { + i := hits[a] + j := hits[b] + if i.Score == j.Score { + return i.HitNumber < j.HitNumber + } + return i.Score > j.Score + }) +} + +// scoreBreakdownForQuery fetches the score for a specific KNN query index from +// the provided hit. The boolean return indicates whether the score is present. +func scoreBreakdownForQuery(hit *search.DocumentMatch, idx int) (float64, bool) { + if hit == nil || hit.ScoreBreakdown == nil { + return 0, false + } + + score, ok := hit.ScoreBreakdown[idx] + return score, ok +} + +// sortDocMatchesByBreakdown orders the hits in-place using the KNN score for +// the supplied query index (descending), breaking ties with `HitNumber` and +// placing hits without a score at the end. +func sortDocMatchesByBreakdown(hits search.DocumentMatchCollection, queryIdx int) { + if len(hits) < 2 { + return + } + + sort.SliceStable(hits, func(a, b int) bool { + left := hits[a] + right := hits[b] + + var leftScore float64 + leftOK := false + if left != nil && left.ScoreBreakdown != nil { + leftScore, leftOK = left.ScoreBreakdown[queryIdx] + } + + var rightScore float64 + rightOK := false + if right != nil && right.ScoreBreakdown != nil { + rightScore, rightOK = right.ScoreBreakdown[queryIdx] + } + + if leftOK && rightOK { + if leftScore == rightScore { + return left.HitNumber < right.HitNumber + } + return leftScore > rightScore + } + + if leftOK != rightOK { + return leftOK + } + + return left.HitNumber < right.HitNumber + }) +} + +// getFusionExplAt copies the existing explanation child at the requested index +// and wraps it in a new node describing how the fusion algorithm adjusted the +// score. +func getFusionExplAt(hit *search.DocumentMatch, i int, value float64, message string) *search.Explanation { + return &search.Explanation{ + Value: value, + Message: message, + Children: []*search.Explanation{hit.Expl.Children[i]}, + } +} + +// finalizeFusionExpl installs the collection of fusion explanation children and +// updates the root message so the caller sees the fused score as the sum of its +// parts. +func finalizeFusionExpl(hit *search.DocumentMatch, explChildren []*search.Explanation) { + hit.Expl.Children = explChildren + + hit.Expl.Value = hit.Score + hit.Expl.Message = "sum of" +} diff --git a/geo/README.md b/geo/README.md new file mode 100644 index 0000000..74ff820 --- /dev/null +++ b/geo/README.md @@ -0,0 +1,312 @@ +# Geo spatial search support in bleve + +Latest bleve spatial capabilities are powered by spatial hierarchical tokens generated from s2geometry. +You can find more details about the [s2geometry basics here](http://s2geometry.io/), and explore the extended functionality of our forked golang port of [s2geometry lib here](https://github.com/blevesearch/geo). + +Users can continue to index and query `geopoint` field type and the existing queries like, + +- Point Distance +- Bounded Rectangle +- Bounded Polygon + +as before. + +## New Spatial Field Type - geoshape + +We have introduced a field type (`geoshape`) for representing the new spatial types. + +Using the new `geoshape` field type, users can unblock the spatial capabilities +for the [geojson](https://datatracker.ietf.org/doc/html/rfc7946) shapes like, + +- Point +- LineString +- Polygon +- MultiPoint +- MultiLineString +- MultiPolygon +- GeometryCollection + +In addition to these shapes, bleve will also support additional shapes like, + +- Circle +- Envelope (Bounded box) + +To specify GeoJSON data, use a nested field with: + +- a field named type that specifies the GeoJSON object type and the type value will be case-insensitive. +- a field named coordinates that specifies the object's coordinates. + +```text + "fieldName": { + "type": "GeoJSON Type", + "coordinates": + } +``` + +- If specifying latitude and longitude coordinates, list the longitude first and then latitude. +- Valid longitude values are between -180 and 180, both inclusive. +- Valid latitude values are between -90 and 90, both inclusive. +- Shapes would be internally represented as geodesics. +- The GeoJSON specification strongly suggests splitting geometries so that neither of their parts crosses the antimeridian. + +Examples for the various geojson shapes representations are as below. + +## Point + +The following specifies a [Point](https://tools.ietf.org/html/rfc7946#section-3.1.2) field in a document: + +```json +{ + "type": "point", + "coordinates": [75.05687713623047, 22.53539059204079] +} +``` + +## Linestring + +The following specifies a [Linestring](https://tools.ietf.org/html/rfc7946#section-3.1.4) field in a document: + +```json +{ + "type": "linestring", + "coordinates": [ + [77.01416015625, 23.0797317624497], + [78.134765625, 20.385825381874263] + ] +} +``` + +## Polygon + +The following specifies a [Polygon](https://tools.ietf.org/html/rfc7946#section-3.1.6) field in a document: + +```json +{ + "type": "polygon", + "coordinates": [ + [ + [85.605, 57.207], + [86.396, 55.998], + [87.033, 56.716], + [85.605, 57.207] + ] + ] +} +``` + +The first and last coordinates must match in order to close the polygon. +And the exterior coordinates have to be in Counter Clockwise Order in a polygon. (CCW) + +## MultiPoint + +The following specifies a [Multipoint](https://tools.ietf.org/html/rfc7946#section-3.1.3) field in a document: + +```json +{ + "type": "multipoint", + "coordinates": [ + [-115.8343505859375, 38.45789034424927], + [-115.81237792968749, 38.19502155795575], + [-120.80017089843749, 36.54053616262899], + [-120.67932128906249, 36.33725319397006] + ] +} +``` + +## MultiLineString + +The following specifies a [MultiLineString](https://tools.ietf.org/html/rfc7946#section-3.1.5) field in a document: + +```json +{ + "type": "multilinestring", + "coordinates": [ + [ + [-118.31726074, 35.250105158], + [-117.509765624, 35.3756141] + ], + [ + [-118.696289, 34.624167789], + [-118.317260742, 35.03899204] + ], + [ + [-117.9492187, 35.146862906], + [-117.6745605, 34.41144164] + ] + ] +} +``` + +## MultiPolygon + +The following specifies a [MultiPolygon](https://tools.ietf.org/html/rfc7946#section-3.1.7) field in a document: + +```json +{ + "type": "multipolygon", + "coordinates": [ + [ + [ + [-73.958, 40.8003], + [-73.9498, 40.7968], + [-73.9737, 40.7648], + [-73.9814, 40.7681], + [-73.958, 40.8003] + ] + ], + [ + [ + [-73.958, 40.8003], + [-73.9498, 40.7968], + [-73.9737, 40.7648], + [-73.958, 40.8003] + ] + ] + ] +} +``` + +## GeometryCollection + +The following specifies a [GeometryCollection](https://tools.ietf.org/html/rfc7946#section-3.1.8) field in a document: + +```json +{ + "type": "geometrycollection", + "geometries": [ + { + "type": "multipoint", + "coordinates": [ + [-73.958, 40.8003], + [-73.9498, 40.7968], + [-73.9737, 40.7648], + [-73.9814, 40.7681] + ] + }, + { + "type": "multilinestring", + "coordinates": [ + [ + [-73.96943, 40.78519], + [-73.96082, 40.78095] + ], + [ + [-73.96415, 40.79229], + [-73.95544, 40.78854] + ], + [ + [-73.97162, 40.78205], + [-73.96374, 40.77715] + ], + [ + [-73.9788, 40.77247], + [-73.97036, 40.76811] + ] + ] + }, + { + "type": "polygon", + "coordinates": [ + [ + [0, 0], + [3, 6], + [6, 1], + [0, 0] + ], + [ + [2, 2], + [3, 3], + [4, 2], + [2, 2] + ] + ] + } + ] +} +``` + +## Circle + +If the user wishes to cover a circular region over the earth's surface, then they could use this shape. +A sample circular shape is as below. + +```json +{ + "type": "circle", + "coordinates": [75.05687713623047, 22.53539059204079], + "radius": "1000m" +} +``` + +Circle is specified over the center point coordinates along with the radius. +Example formats supported for radius are: +"5in" , "5inch" , "7yd" , "7yards", "9ft" , "9feet", "11km", "11kilometers", "3nm", "3nauticalmiles", "13mm" , "13millimeters", "15cm", "15centimeters", "17mi", "17miles", "19m" or "19meters". + +If the unit cannot be determined, the entire string is parsed and the unit of meters is assumed. + +## Envelope + +Envelope type, which consists of coordinates for upper left and lower right points of the shape to represent a bounding rectangle in the format [[minLon, maxLat], [maxLon, minLat]]. + +```json +{ + "type": "envelope", + "coordinates": [ + [72.83, 18.979], + [78.508, 17.4555] + ] +} +``` + +## GeoShape Query + +Geoshape query support three types/filters of spatial querying capability across those heterogeneous types of documents indexed. + +### Query Structure + +```json +{ + "query": { + "geometry": { + "shape": { + "type": "", + "coordinates": [ + [[]] + ] + }, + "relation": "<>" + } + } +} +``` + +*shapeType* => can be any of the aforementioned types like Point, LineString, Polygon, MultiPoint, +Geometrycollection, MultiLineString, MultiPolygon, Circle and Envelope. + +*filterName* => can be any of the 3 types like *intersects*, *contains* and *within*. + +### Relation + +| FilterName | Description | +| :-----------:| :-----------------------------------------------------------------: | +| `intersects` | Return all documents whose shape field intersects the query geometry. | +| `contains` | Return all documents whose shape field contains the query geometry | +| `within` | Return all documents whose shape field is within the query geometry. | + +------------------------------------------------------------------------------------------------------------------------ + +### Older Implementation + +First, all of this geo code is a Go adaptation of the [Lucene 5.3.2 sandbox geo support](https://lucene.apache.org/core/5_3_2/sandbox/org/apache/lucene/util/package-summary.html). + +## Notes + +- All of the APIs will use float64 for lon/lat values. +- When describing a point in function arguments or return values, we always use the order lon, lat. +- High level APIs will use TopLeft and BottomRight to describe bounding boxes. This may not map cleanly to min/max lon/lat when crossing the dateline. The lower level APIs will use min/max lon/lat and require the higher-level code to split boxes accordingly. +- Points and MultiPoints may only contain Points and MultiPoints. +- LineStrings and MultiLineStrings may only contain Points and MultiPoints. +- Polygons or MultiPolygons intersecting Polygons and MultiPolygons may return arbitrary results when the overlap is only an edge or a vertex. +- Circles containing polygon will return a false positive result if all of the vertices of the polygon are within the circle, but the orientation of those points are clock-wise. +- The edges of an Envelope follows the latitude and longitude lines instead of the shortest path on a globe. +- Envelope intersecting queries with LineStrings, MultiLineStrings, Polygons and MultiPolygons implicitly converts the Envelope into a Polygon which changes the curvature of the edges causing inaccurate results for few edge cases. diff --git a/geo/benchmark_geohash_test.go b/geo/benchmark_geohash_test.go new file mode 100644 index 0000000..2cdeb1d --- /dev/null +++ b/geo/benchmark_geohash_test.go @@ -0,0 +1,43 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geo + +import ( + "testing" +) + +func BenchmarkGeoHashLen5NewDecode(b *testing.B) { + b.ResetTimer() + hash := "d3hn3" + for i := 0; i < b.N; i++ { + _, _ = DecodeGeoHash(hash) + } +} + +func BenchmarkGeoHashLen6NewDecode(b *testing.B) { + b.ResetTimer() + hash := "u4pruy" + for i := 0; i < b.N; i++ { + _, _ = DecodeGeoHash(hash) + } +} + +func BenchmarkGeoHashLen7NewDecode(b *testing.B) { + b.ResetTimer() + hash := "u4pruyd" + for i := 0; i < b.N; i++ { + _, _ = DecodeGeoHash(hash) + } +} diff --git a/geo/geo.go b/geo/geo.go new file mode 100644 index 0000000..e9cc0f5 --- /dev/null +++ b/geo/geo.go @@ -0,0 +1,210 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geo + +import ( + "fmt" + "math" + + "github.com/blevesearch/bleve/v2/numeric" +) + +// GeoBits is the number of bits used for a single geo point +// Currently this is 32bits for lon and 32bits for lat +var GeoBits uint = 32 + +var minLon = -180.0 +var minLat = -90.0 +var maxLon = 180.0 +var maxLat = 90.0 +var minLonRad = minLon * degreesToRadian +var minLatRad = minLat * degreesToRadian +var maxLonRad = maxLon * degreesToRadian +var maxLatRad = maxLat * degreesToRadian +var geoTolerance = 1e-6 +var lonScale = float64((uint64(0x1)<> 1)) +} + +func unscaleLon(lon uint64) float64 { + return (float64(lon) / lonScale) + minLon +} + +func unscaleLat(lat uint64) float64 { + return (float64(lat) / latScale) + minLat +} + +// compareGeo will compare two float values and see if they are the same +// taking into consideration a known geo tolerance. +func compareGeo(a, b float64) float64 { + compare := a - b + if math.Abs(compare) <= geoTolerance { + return 0 + } + return compare +} + +// RectIntersects checks whether rectangles a and b intersect +func RectIntersects(aMinX, aMinY, aMaxX, aMaxY, bMinX, bMinY, bMaxX, bMaxY float64) bool { + return !(aMaxX < bMinX || aMinX > bMaxX || aMaxY < bMinY || aMinY > bMaxY) +} + +// RectWithin checks whether box a is within box b +func RectWithin(aMinX, aMinY, aMaxX, aMaxY, bMinX, bMinY, bMaxX, bMaxY float64) bool { + rv := !(aMinX < bMinX || aMinY < bMinY || aMaxX > bMaxX || aMaxY > bMaxY) + return rv +} + +// BoundingBoxContains checks whether the lon/lat point is within the box +func BoundingBoxContains(lon, lat, minLon, minLat, maxLon, maxLat float64) bool { + return compareGeo(lon, minLon) >= 0 && compareGeo(lon, maxLon) <= 0 && + compareGeo(lat, minLat) >= 0 && compareGeo(lat, maxLat) <= 0 +} + +const degreesToRadian = math.Pi / 180 +const radiansToDegrees = 180 / math.Pi + +// DegreesToRadians converts an angle in degrees to radians +func DegreesToRadians(d float64) float64 { + return d * degreesToRadian +} + +// RadiansToDegrees converts an angle in radians to degrees +func RadiansToDegrees(r float64) float64 { + return r * radiansToDegrees +} + +var earthMeanRadiusMeters = 6371008.7714 + +func RectFromPointDistance(lon, lat, dist float64) (float64, float64, float64, float64, error) { + err := checkLongitude(lon) + if err != nil { + return 0, 0, 0, 0, err + } + err = checkLatitude(lat) + if err != nil { + return 0, 0, 0, 0, err + } + radLon := DegreesToRadians(lon) + radLat := DegreesToRadians(lat) + radDistance := (dist + 7e-2) / earthMeanRadiusMeters + + minLatL := radLat - radDistance + maxLatL := radLat + radDistance + + var minLonL, maxLonL float64 + if minLatL > minLatRad && maxLatL < maxLatRad { + deltaLon := math.Asin(math.Sin(radDistance) / math.Cos(radLat)) + minLonL = radLon - deltaLon + if minLonL < minLonRad { + minLonL += 2 * math.Pi + } + maxLonL = radLon + deltaLon + if maxLonL > maxLonRad { + maxLonL -= 2 * math.Pi + } + } else { + // pole is inside distance + minLatL = math.Max(minLatL, minLatRad) + maxLatL = math.Min(maxLatL, maxLatRad) + minLonL = minLonRad + maxLonL = maxLonRad + } + + return RadiansToDegrees(minLonL), + RadiansToDegrees(maxLatL), + RadiansToDegrees(maxLonL), + RadiansToDegrees(minLatL), + nil +} + +func checkLatitude(latitude float64) error { + if math.IsNaN(latitude) || latitude < minLat || latitude > maxLat { + return fmt.Errorf("invalid latitude %f; must be between %f and %f", latitude, minLat, maxLat) + } + return nil +} + +func checkLongitude(longitude float64) error { + if math.IsNaN(longitude) || longitude < minLon || longitude > maxLon { + return fmt.Errorf("invalid longitude %f; must be between %f and %f", longitude, minLon, maxLon) + } + return nil +} + +func BoundingRectangleForPolygon(polygon []Point) ( + float64, float64, float64, float64, error) { + err := checkLongitude(polygon[0].Lon) + if err != nil { + return 0, 0, 0, 0, err + } + err = checkLatitude(polygon[0].Lat) + if err != nil { + return 0, 0, 0, 0, err + } + maxY, minY := polygon[0].Lat, polygon[0].Lat + maxX, minX := polygon[0].Lon, polygon[0].Lon + for i := 1; i < len(polygon); i++ { + err := checkLongitude(polygon[i].Lon) + if err != nil { + return 0, 0, 0, 0, err + } + err = checkLatitude(polygon[i].Lat) + if err != nil { + return 0, 0, 0, 0, err + } + + maxY = math.Max(maxY, polygon[i].Lat) + minY = math.Min(minY, polygon[i].Lat) + + maxX = math.Max(maxX, polygon[i].Lon) + minX = math.Min(minX, polygon[i].Lon) + } + + return minX, maxY, maxX, minY, nil +} diff --git a/geo/geo_dist.go b/geo/geo_dist.go new file mode 100644 index 0000000..a621787 --- /dev/null +++ b/geo/geo_dist.go @@ -0,0 +1,98 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geo + +import ( + "fmt" + "math" + "strconv" + "strings" +) + +type distanceUnit struct { + conv float64 + suffixes []string +} + +var inch = distanceUnit{0.0254, []string{"in", "inch"}} +var yard = distanceUnit{0.9144, []string{"yd", "yards"}} +var feet = distanceUnit{0.3048, []string{"ft", "feet"}} +var kilom = distanceUnit{1000, []string{"km", "kilometers"}} +var nauticalm = distanceUnit{1852.0, []string{"nm", "nauticalmiles"}} +var millim = distanceUnit{0.001, []string{"mm", "millimeters"}} +var centim = distanceUnit{0.01, []string{"cm", "centimeters"}} +var miles = distanceUnit{1609.344, []string{"mi", "miles"}} +var meters = distanceUnit{1, []string{"m", "meters"}} + +var distanceUnits = []*distanceUnit{ + &inch, &yard, &feet, &kilom, &nauticalm, &millim, ¢im, &miles, &meters, +} + +// ParseDistance attempts to parse a distance string and return distance in +// meters. Example formats supported: +// "5in" "5inch" "7yd" "7yards" "9ft" "9feet" "11km" "11kilometers" +// "3nm" "3nauticalmiles" "13mm" "13millimeters" "15cm" "15centimeters" +// "17mi" "17miles" "19m" "19meters" +// If the unit cannot be determined, the entire string is parsed and the +// unit of meters is assumed. +// If the number portion cannot be parsed, 0 and the parse error are returned. +func ParseDistance(d string) (float64, error) { + for _, unit := range distanceUnits { + for _, unitSuffix := range unit.suffixes { + if strings.HasSuffix(d, unitSuffix) { + parsedNum, err := strconv.ParseFloat(d[0:len(d)-len(unitSuffix)], 64) + if err != nil { + return 0, err + } + return parsedNum * unit.conv, nil + } + } + } + // no unit matched, try assuming meters? + parsedNum, err := strconv.ParseFloat(d, 64) + if err != nil { + return 0, err + } + return parsedNum, nil +} + +// ParseDistanceUnit attempts to parse a distance unit and return the +// multiplier for converting this to meters. If the unit cannot be parsed +// then 0 and the error message is returned. +func ParseDistanceUnit(u string) (float64, error) { + for _, unit := range distanceUnits { + for _, unitSuffix := range unit.suffixes { + if u == unitSuffix { + return unit.conv, nil + } + } + } + return 0, fmt.Errorf("unknown distance unit: %s", u) +} + +// Haversin computes the distance between two points. +// This implementation uses the sloppy math implementations which trade off +// accuracy for performance. The distance returned is in kilometers. +func Haversin(lon1, lat1, lon2, lat2 float64) float64 { + x1 := lat1 * degreesToRadian + x2 := lat2 * degreesToRadian + h1 := 1 - math.Cos(x1-x2) + h2 := 1 - math.Cos((lon1-lon2)*degreesToRadian) + h := (h1 + math.Cos(x1)*math.Cos(x2)*h2) / 2 + avgLat := (x1 + x2) / 2 + diameter := earthDiameter(avgLat) + + return diameter * math.Asin(math.Min(1, math.Sqrt(h))) +} diff --git a/geo/geo_dist_test.go b/geo/geo_dist_test.go new file mode 100644 index 0000000..a7d67b2 --- /dev/null +++ b/geo/geo_dist_test.go @@ -0,0 +1,120 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geo + +import ( + "fmt" + "math" + "reflect" + "strconv" + "testing" +) + +func TestParseDistance(t *testing.T) { + tests := []struct { + dist string + want float64 + wantErr error + }{ + {"5mi", 5 * 1609.344, nil}, + {"3", 3, nil}, + {"3m", 3, nil}, + {"5km", 5000, nil}, + {"km", 0, &strconv.NumError{Func: "ParseFloat", Num: "", Err: strconv.ErrSyntax}}, + {"", 0, &strconv.NumError{Func: "ParseFloat", Num: "", Err: strconv.ErrSyntax}}, + } + + for _, test := range tests { + got, err := ParseDistance(test.dist) + if !reflect.DeepEqual(err, test.wantErr) { + t.Errorf("expected err: %v, got %v for %s", test.wantErr, err, test.dist) + } + if got != test.want { + t.Errorf("expected distance %f got %f for %s", test.want, got, test.dist) + } + } +} + +func TestParseDistanceUnit(t *testing.T) { + tests := []struct { + dist string + want float64 + wantErr error + }{ + {"mi", 1609.344, nil}, + {"m", 1, nil}, + {"km", 1000, nil}, + {"", 0, fmt.Errorf("unknown distance unit: ")}, + {"kam", 0, fmt.Errorf("unknown distance unit: kam")}, + } + + for _, test := range tests { + got, err := ParseDistanceUnit(test.dist) + if !reflect.DeepEqual(err, test.wantErr) { + t.Errorf("expected err: %v, got %v for %s", test.wantErr, err, test.dist) + } + if got != test.want { + t.Errorf("expected distance %f got %f for %s", test.want, got, test.dist) + } + } +} + +func TestHaversinDistance(t *testing.T) { + earthRadiusKMs := 6378.137 + halfCircle := earthRadiusKMs * math.Pi + + tests := []struct { + lon1 float64 + lat1 float64 + lon2 float64 + lat2 float64 + want float64 + }{ + {1, 1, math.NaN(), 1, math.NaN()}, + {1, 1, 1, math.NaN(), math.NaN()}, + {1, math.NaN(), 1, 1, math.NaN()}, + {math.NaN(), 1, 1, 1, math.NaN()}, + + {0, 0, 0, 0, 0}, + {-180, 0, -180, 0, 0}, + {-180, 0, 180, 0, 0}, + {180, 0, 180, 0, 0}, + + {0, 90, 0, 90, 0}, + {-180, 90, -180, 90, 0}, + {-180, 90, 180, 90, 0}, + {180, 90, 180, 90, 0}, + + {0, 0, 180, 0, halfCircle}, + + {-74.0059731, 40.7143528, -74.0059731, 40.7143528, 0}, + {-74.0059731, 40.7143528, -73.9844722, 40.759011, 5.286}, + {-74.0059731, 40.7143528, -74.007819, 40.718266, 0.4621}, + {-74.0059731, 40.7143528, -74.0088305, 40.7051157, 1.055}, + {-74.0059731, 40.7143528, -74, 40.7247222, 1.258}, + {-74.0059731, 40.7143528, -73.9962255, 40.731033, 2.029}, + {-74.0059731, 40.7143528, -73.95, 40.65, 8.572}, + } + + for _, test := range tests { + got := Haversin(test.lon1, test.lat1, test.lon2, test.lat2) + if math.IsNaN(test.want) && !math.IsNaN(got) { + t.Errorf("expected NaN, got %f", got) + } + if !math.IsNaN(test.want) && math.Abs(got-test.want) > 1e-2 { + t.Errorf("expected %f got %f", test.want, got) + } + } +} diff --git a/geo/geo_s2plugin_impl.go b/geo/geo_s2plugin_impl.go new file mode 100644 index 0000000..95434db --- /dev/null +++ b/geo/geo_s2plugin_impl.go @@ -0,0 +1,523 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geo + +import ( + "encoding/json" + "sync" + + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/geo/geojson" + "github.com/blevesearch/geo/s2" +) + +const ( + PointType = "point" + MultiPointType = "multipoint" + LineStringType = "linestring" + MultiLineStringType = "multilinestring" + PolygonType = "polygon" + MultiPolygonType = "multipolygon" + GeometryCollectionType = "geometrycollection" + CircleType = "circle" + EnvelopeType = "envelope" +) + +// spatialPluginsMap is spatial plugin cache. +var ( + spatialPluginsMap = make(map[string]index.SpatialAnalyzerPlugin) + pluginsMapLock = sync.RWMutex{} +) + +func init() { + registerS2RegionTermIndexer() +} + +func registerS2RegionTermIndexer() { + spatialPlugin := S2SpatialAnalyzerPlugin{ + s2Indexer: s2.NewRegionTermIndexerWithOptions(initS2IndexerOptions()), + s2Searcher: s2.NewRegionTermIndexerWithOptions(initS2SearcherOptions()), + s2GeoPointsRegionTermIndexer: s2.NewRegionTermIndexerWithOptions(initS2OptionsForGeoPoints()), + } + + RegisterSpatialAnalyzerPlugin(&spatialPlugin) +} + +// RegisterSpatialAnalyzerPlugin registers the given plugin implementation. +func RegisterSpatialAnalyzerPlugin(plugin index.SpatialAnalyzerPlugin) { + pluginsMapLock.Lock() + spatialPluginsMap[plugin.Type()] = plugin + pluginsMapLock.Unlock() +} + +// GetSpatialAnalyzerPlugin retrieves the given implementation type. +func GetSpatialAnalyzerPlugin(typ string) index.SpatialAnalyzerPlugin { + pluginsMapLock.RLock() + rv := spatialPluginsMap[typ] + pluginsMapLock.RUnlock() + return rv +} + +// initS2IndexerOptions returns the options for s2's region +// term indexer for the index time tokens of geojson shapes. +func initS2IndexerOptions() s2.Options { + options := s2.Options{} + // maxLevel control the maximum size of the + // S2Cells used to approximate regions. + options.SetMaxLevel(16) + + // minLevel control the minimum size of the + // S2Cells used to approximate regions. + options.SetMinLevel(2) + + // levelMod value greater than 1 increases the effective branching + // factor of the S2Cell hierarchy by skipping some levels. + options.SetLevelMod(1) + + // maxCells controls the maximum number of cells + // when approximating each s2 region. + options.SetMaxCells(20) + + return options +} + +// initS2SearcherOptions returns the options for s2's region +// term indexer for the query time tokens of geojson shapes. +func initS2SearcherOptions() s2.Options { + options := s2.Options{} + // maxLevel control the maximum size of the + // S2Cells used to approximate regions. + options.SetMaxLevel(16) + + // minLevel control the minimum size of the + // S2Cells used to approximate regions. + options.SetMinLevel(2) + + // levelMod value greater than 1 increases the effective branching + // factor of the S2Cell hierarchy by skipping some levels. + options.SetLevelMod(1) + + // maxCells controls the maximum number of cells + // when approximating each s2 region. + options.SetMaxCells(8) + + return options +} + +// initS2OptionsForGeoPoints returns the options for +// s2's region term indexer for the original geopoints. +func initS2OptionsForGeoPoints() s2.Options { + options := s2.Options{} + // maxLevel control the maximum size of the + // S2Cells used to approximate regions. + options.SetMaxLevel(16) + + // minLevel control the minimum size of the + // S2Cells used to approximate regions. + options.SetMinLevel(4) + + // levelMod value greater than 1 increases the effective branching + // factor of the S2Cell hierarchy by skipping some levels. + options.SetLevelMod(2) + + // maxCells controls the maximum number of cells + // when approximating each s2 region. + options.SetMaxCells(8) + + // explicit for geo points. + options.SetPointsOnly(true) + + return options +} + +// S2SpatialAnalyzerPlugin is an implementation of +// the index.SpatialAnalyzerPlugin interface. +type S2SpatialAnalyzerPlugin struct { + s2Indexer *s2.RegionTermIndexer + s2Searcher *s2.RegionTermIndexer + s2GeoPointsRegionTermIndexer *s2.RegionTermIndexer +} + +func (s *S2SpatialAnalyzerPlugin) Type() string { + return "s2" +} + +func (s *S2SpatialAnalyzerPlugin) GetIndexTokens(queryShape index.GeoJSON) []string { + var rv []string + shapes := []index.GeoJSON{queryShape} + if gc, ok := queryShape.(*geojson.GeometryCollection); ok { + shapes = gc.Shapes + } + + for _, shape := range shapes { + if s2t, ok := shape.(s2Tokenizable); ok { + rv = append(rv, s2t.IndexTokens(s.s2Indexer)...) + } else if s2t, ok := shape.(s2TokenizableEx); ok { + rv = append(rv, s2t.IndexTokens(s)...) + } + } + + return geojson.DeduplicateTerms(rv) +} + +func (s *S2SpatialAnalyzerPlugin) GetQueryTokens(queryShape index.GeoJSON) []string { + var rv []string + shapes := []index.GeoJSON{queryShape} + if gc, ok := queryShape.(*geojson.GeometryCollection); ok { + shapes = gc.Shapes + } + + for _, shape := range shapes { + if s2t, ok := shape.(s2Tokenizable); ok { + rv = append(rv, s2t.QueryTokens(s.s2Searcher)...) + } else if s2t, ok := shape.(s2TokenizableEx); ok { + rv = append(rv, s2t.QueryTokens(s)...) + } + } + + return geojson.DeduplicateTerms(rv) +} + +// ------------------------------------------------------------------------ +// s2Tokenizable is an optional interface for shapes that support +// the generation of s2 based tokens that can be used for both +// indexing and querying. + +type s2Tokenizable interface { + // IndexTokens returns the tokens for indexing. + IndexTokens(*s2.RegionTermIndexer) []string + + // QueryTokens returns the tokens for searching. + QueryTokens(*s2.RegionTermIndexer) []string +} + +// ------------------------------------------------------------------------ +// s2TokenizableEx is an optional interface for shapes that support +// the generation of s2 based tokens that can be used for both +// indexing and querying. This is intended for the older geopoint +// indexing and querying. +type s2TokenizableEx interface { + // IndexTokens returns the tokens for indexing. + IndexTokens(*S2SpatialAnalyzerPlugin) []string + + // QueryTokens returns the tokens for searching. + QueryTokens(*S2SpatialAnalyzerPlugin) []string +} + +//---------------------------------------------------------------------------------- + +func (p *Point) Type() string { + return PointType +} + +func (p *Point) Value() ([]byte, error) { + return util.MarshalJSON(p) +} + +func (p *Point) Intersects(s index.GeoJSON) (bool, error) { + // placeholder implementation + return false, nil +} + +func (p *Point) Contains(s index.GeoJSON) (bool, error) { + // placeholder implementation + return false, nil +} + +func (p *Point) IndexTokens(s *S2SpatialAnalyzerPlugin) []string { + return s.s2GeoPointsRegionTermIndexer.GetIndexTermsForPoint(s2.PointFromLatLng( + s2.LatLngFromDegrees(p.Lat, p.Lon)), "") +} + +func (p *Point) QueryTokens(s *S2SpatialAnalyzerPlugin) []string { + return nil +} + +func (p *Point) IndexCells() ([]uint64, []uint64) { + // placeholder implementation + return nil, nil +} + +func (p *Point) QueryCells() ([]uint64, []uint64) { + // placeholder implementation + return nil, nil +} + +func (p *Point) BoundingBox() index.GeoJSON { + // placeholder implementation + return nil +} + +//---------------------------------------------------------------------------------- + +type boundedRectangle struct { + minLat float64 + maxLat float64 + minLon float64 + maxLon float64 +} + +func NewBoundedRectangle(minLat, minLon, maxLat, + maxLon float64) *boundedRectangle { + return &boundedRectangle{minLat: minLat, + maxLat: maxLat, minLon: minLon, maxLon: maxLon} +} + +func (br *boundedRectangle) Type() string { + // placeholder implementation + return "boundedRectangle" +} + +func (br *boundedRectangle) Value() ([]byte, error) { + return util.MarshalJSON(br) +} + +func (p *boundedRectangle) Intersects(s index.GeoJSON) (bool, error) { + // placeholder implementation + return false, nil +} + +func (p *boundedRectangle) Contains(s index.GeoJSON) (bool, error) { + // placeholder implementation + return false, nil +} + +func (br *boundedRectangle) IndexTokens(s *S2SpatialAnalyzerPlugin) []string { + return nil +} + +func (br *boundedRectangle) QueryTokens(s *S2SpatialAnalyzerPlugin) []string { + rect := s2.RectFromDegrees(br.minLat, br.minLon, br.maxLat, br.maxLon) + + // obtain the terms to be searched for the given bounding box. + terms := s.s2GeoPointsRegionTermIndexer.GetQueryTermsForRegion(rect, "") + + return geojson.StripCoveringTerms(terms) +} + +func (br *boundedRectangle) IndexCells() ([]uint64, []uint64) { + // placeholder implementation + return nil, nil +} + +func (br *boundedRectangle) QueryCells() ([]uint64, []uint64) { + // placeholder implementation + return nil, nil +} + +func (br *boundedRectangle) BoundingBox() index.GeoJSON { + // placeholder implementation + return nil +} + +//---------------------------------------------------------------------------------- + +type boundedPolygon struct { + coordinates []Point +} + +func NewBoundedPolygon(coordinates []Point) *boundedPolygon { + return &boundedPolygon{coordinates: coordinates} +} + +func (bp *boundedPolygon) Type() string { + // placeholder implementation + return "boundedPolygon" +} + +func (bp *boundedPolygon) Value() ([]byte, error) { + return util.MarshalJSON(bp) +} + +func (p *boundedPolygon) Intersects(s index.GeoJSON) (bool, error) { + // placeholder implementation + return false, nil +} + +func (p *boundedPolygon) Contains(s index.GeoJSON) (bool, error) { + // placeholder implementation + return false, nil +} + +func (bp *boundedPolygon) IndexTokens(s *S2SpatialAnalyzerPlugin) []string { + return nil +} + +func (bp *boundedPolygon) QueryTokens(s *S2SpatialAnalyzerPlugin) []string { + vertices := make([]s2.Point, len(bp.coordinates)) + for i, point := range bp.coordinates { + vertices[i] = s2.PointFromLatLng( + s2.LatLngFromDegrees(point.Lat, point.Lon)) + } + s2polygon := s2.PolygonFromOrientedLoops([]*s2.Loop{s2.LoopFromPoints(vertices)}) + + // obtain the terms to be searched for the given polygon. + terms := s.s2GeoPointsRegionTermIndexer.GetQueryTermsForRegion( + s2polygon.CapBound(), "") + + return geojson.StripCoveringTerms(terms) +} + +func (bp *boundedPolygon) IndexCells() ([]uint64, []uint64) { + // placeholder implementation + return nil, nil +} + +func (bp *boundedPolygon) QueryCells() ([]uint64, []uint64) { + // placeholder implementation + return nil, nil +} + +func (bp *boundedPolygon) BoundingBox() index.GeoJSON { + // placeholder implementation + return nil +} + +//---------------------------------------------------------------------------------- + +type pointDistance struct { + dist float64 + centerLat float64 + centerLon float64 +} + +func (p *pointDistance) Type() string { + // placeholder implementation + return "pointDistance" +} + +func (p *pointDistance) Value() ([]byte, error) { + return util.MarshalJSON(p) +} + +func NewPointDistance(centerLat, centerLon, + dist float64) *pointDistance { + return &pointDistance{centerLat: centerLat, + centerLon: centerLon, dist: dist} +} + +func (p *pointDistance) Intersects(s index.GeoJSON) (bool, error) { + // placeholder implementation + return false, nil +} + +func (p *pointDistance) Contains(s index.GeoJSON) (bool, error) { + // placeholder implementation + return false, nil +} + +func (pd *pointDistance) IndexTokens(s *S2SpatialAnalyzerPlugin) []string { + return nil +} + +func (pd *pointDistance) QueryTokens(s *S2SpatialAnalyzerPlugin) []string { + // obtain the covering query region from the given points. + queryRegion := s2.CapFromCenterAndRadius(pd.centerLat, + pd.centerLon, pd.dist) + + // obtain the query terms for the query region. + terms := s.s2GeoPointsRegionTermIndexer.GetQueryTermsForRegion(queryRegion, "") + + return geojson.StripCoveringTerms(terms) +} + +func (pd *pointDistance) IndexCells() ([]uint64, []uint64) { + // placeholder implementation + return nil, nil +} + +func (pd *pointDistance) QueryCells() ([]uint64, []uint64) { + // placeholder implementation + return nil, nil +} + +func (pd *pointDistance) BoundingBox() index.GeoJSON { + // placeholder implementation + return nil +} + +// ------------------------------------------------------------------------ + +// NewGeometryCollection instantiate a geometrycollection +// and prefix the byte contents with certain glue bytes that +// can be used later while filering the doc values. +func NewGeometryCollection(coordinates [][][][][]float64, + typs []string) (index.GeoJSON, []byte, error) { + shapes := make([]*geojson.GeoShape, len(coordinates)) + for i := range coordinates { + shapes[i] = &geojson.GeoShape{ + Coordinates: coordinates[i], + Type: typs[i], + } + } + + return geojson.NewGeometryCollection(shapes) +} + +func NewGeometryCollectionFromShapes(shapes []*geojson.GeoShape) ( + index.GeoJSON, []byte, error) { + + return geojson.NewGeometryCollection(shapes) +} + +// NewGeoCircleShape instantiate a circle shape and +// prefix the byte contents with certain glue bytes that +// can be used later while filering the doc values. +func NewGeoCircleShape(cp []float64, + radius string) (index.GeoJSON, []byte, error) { + return geojson.NewGeoCircleShape(cp, radius) +} + +func NewGeoJsonShape(coordinates [][][][]float64, typ string) ( + index.GeoJSON, []byte, error) { + return geojson.NewGeoJsonShape(coordinates, typ) +} + +func NewGeoJsonPoint(points []float64) index.GeoJSON { + return geojson.NewGeoJsonPoint(points) +} + +func NewGeoJsonMultiPoint(points [][]float64) index.GeoJSON { + return geojson.NewGeoJsonMultiPoint(points) +} + +func NewGeoJsonLinestring(points [][]float64) index.GeoJSON { + return geojson.NewGeoJsonLinestring(points) +} + +func NewGeoJsonMultilinestring(points [][][]float64) index.GeoJSON { + return geojson.NewGeoJsonMultilinestring(points) +} + +func NewGeoJsonPolygon(points [][][]float64) index.GeoJSON { + return geojson.NewGeoJsonPolygon(points) +} + +func NewGeoJsonMultiPolygon(points [][][][]float64) index.GeoJSON { + return geojson.NewGeoJsonMultiPolygon(points) +} + +func NewGeoCircle(points []float64, radius string) index.GeoJSON { + return geojson.NewGeoCircle(points, radius) +} + +func NewGeoEnvelope(points [][]float64) index.GeoJSON { + return geojson.NewGeoEnvelope(points) +} + +func ParseGeoJSONShape(input json.RawMessage) (index.GeoJSON, error) { + return geojson.ParseGeoJSONShape(input) +} diff --git a/geo/geo_test.go b/geo/geo_test.go new file mode 100644 index 0000000..52a38e2 --- /dev/null +++ b/geo/geo_test.go @@ -0,0 +1,183 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geo + +import ( + "math" + "testing" +) + +func TestMortonHashMortonUnhash(t *testing.T) { + tests := []struct { + lon float64 + lat float64 + }{ + {-180.0, -90.0}, + {-5, 27.3}, + {0, 0}, + {1.0, 1.0}, + {24.7, -80.4}, + {180.0, 90.0}, + } + + for _, test := range tests { + hash := MortonHash(test.lon, test.lat) + lon := MortonUnhashLon(hash) + lat := MortonUnhashLat(hash) + if compareGeo(test.lon, lon) != 0 { + t.Errorf("expected lon %f, got %f, hash %x", test.lon, lon, hash) + } + if compareGeo(test.lat, lat) != 0 { + t.Errorf("expected lat %f, got %f, hash %x", test.lat, lat, hash) + } + } +} + +func TestScaleLonUnscaleLon(t *testing.T) { + tests := []struct { + lon float64 + }{ + {-180.0}, + {0.0}, + {1.0}, + {180.0}, + } + + for _, test := range tests { + s := scaleLon(test.lon) + lon := unscaleLon(s) + if compareGeo(test.lon, lon) != 0 { + t.Errorf("expected %f, got %f, scaled was %d", test.lon, lon, s) + } + } +} + +func TestScaleLatUnscaleLat(t *testing.T) { + tests := []struct { + lat float64 + }{ + {-90.0}, + {0.0}, + {1.0}, + {90.0}, + } + + for _, test := range tests { + s := scaleLat(test.lat) + lat := unscaleLat(s) + if compareGeo(test.lat, lat) != 0 { + t.Errorf("expected %.16f, got %.16f, scaled was %d", test.lat, lat, s) + } + } +} + +func TestRectFromPointDistance(t *testing.T) { + // at the equator 1 degree of latitude is about 110567 meters + _, upperLeftLat, _, lowerRightLat, err := RectFromPointDistance(0, 0, 110567) + if err != nil { + t.Fatal(err) + } + if math.Abs(upperLeftLat-1) > 1e-2 { + t.Errorf("expected bounding box upper left lat to be almost 1, got %f", upperLeftLat) + } + if math.Abs(lowerRightLat+1) > 1e-2 { + t.Errorf("expected bounding box lower right lat to be almost -1, got %f", lowerRightLat) + } +} + +func TestRectIntersects(t *testing.T) { + tests := []struct { + aMinX float64 + aMinY float64 + aMaxX float64 + aMaxY float64 + bMinX float64 + bMinY float64 + bMaxX float64 + bMaxY float64 + want bool + }{ + // clearly overlap + {0, 0, 2, 2, 1, 1, 3, 3, true}, + // clearly do not overalp + {0, 0, 1, 1, 2, 2, 3, 3, false}, + // share common point + {0, 0, 1, 1, 1, 1, 2, 2, true}, + } + + for _, test := range tests { + got := RectIntersects(test.aMinX, test.aMinY, test.aMaxX, test.aMaxY, test.bMinX, test.bMinY, test.bMaxX, test.bMaxY) + if test.want != got { + t.Errorf("expected intersects %t, got %t for %f %f %f %f %f %f %f %f", test.want, got, test.aMinX, test.aMinY, test.aMaxX, test.aMaxY, test.bMinX, test.bMinY, test.bMaxX, test.bMaxY) + } + } +} + +func TestRectWithin(t *testing.T) { + tests := []struct { + aMinX float64 + aMinY float64 + aMaxX float64 + aMaxY float64 + bMinX float64 + bMinY float64 + bMaxX float64 + bMaxY float64 + want bool + }{ + // clearly within + {1, 1, 2, 2, 0, 0, 3, 3, true}, + // clearly not within + {0, 0, 1, 1, 2, 2, 3, 3, false}, + // overlapping + {0, 0, 2, 2, 1, 1, 3, 3, false}, + // share common point + {0, 0, 1, 1, 1, 1, 2, 2, false}, + // within, but boxes reversed (b is within a, but not a within b) + {0, 0, 3, 3, 1, 1, 2, 2, false}, + } + + for _, test := range tests { + got := RectWithin(test.aMinX, test.aMinY, test.aMaxX, test.aMaxY, test.bMinX, test.bMinY, test.bMaxX, test.bMaxY) + if test.want != got { + t.Errorf("expected within %t, got %t for %f %f %f %f %f %f %f %f", test.want, got, test.aMinX, test.aMinY, test.aMaxX, test.aMaxY, test.bMinX, test.bMinY, test.bMaxX, test.bMaxY) + } + } +} + +func TestBoundingBoxContains(t *testing.T) { + tests := []struct { + lon float64 + lat float64 + minX float64 + minY float64 + maxX float64 + maxY float64 + want bool + }{ + // clearly contains + {1, 1, 0, 0, 2, 2, true}, + // clearly does not contain + {0, 0, 1, 1, 2, 2, false}, + // on corner + {0, 0, 0, 0, 2, 2, true}, + } + for _, test := range tests { + got := BoundingBoxContains(test.lon, test.lat, test.minX, test.minY, test.maxX, test.maxY) + if test.want != got { + t.Errorf("expected box contains %t, got %t for %f,%f in %f %f %f %f ", test.want, got, test.lon, test.lat, test.minX, test.minY, test.maxX, test.maxY) + } + } +} diff --git a/geo/geohash.go b/geo/geohash.go new file mode 100644 index 0000000..d3d4dfa --- /dev/null +++ b/geo/geohash.go @@ -0,0 +1,111 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// This implementation is inspired from the geohash-js +// ref: https://github.com/davetroy/geohash-js + +package geo + +// encoding encapsulates an encoding defined by a given base32 alphabet. +type encoding struct { + enc string + dec [256]byte +} + +// newEncoding constructs a new encoding defined by the given alphabet, +// which must be a 32-byte string. +func newEncoding(encoder string) *encoding { + e := new(encoding) + e.enc = encoder + for i := 0; i < len(e.dec); i++ { + e.dec[i] = 0xff + } + for i := 0; i < len(encoder); i++ { + e.dec[encoder[i]] = byte(i) + } + return e +} + +// base32encoding with the Geohash alphabet. +var base32encoding = newEncoding("0123456789bcdefghjkmnpqrstuvwxyz") + +var masks = []uint64{16, 8, 4, 2, 1} + +// DecodeGeoHash decodes the string geohash faster with +// higher precision. This api is in experimental phase. +func DecodeGeoHash(geoHash string) (float64, float64) { + even := true + lat := []float64{-90.0, 90.0} + lon := []float64{-180.0, 180.0} + + for i := 0; i < len(geoHash); i++ { + cd := uint64(base32encoding.dec[geoHash[i]]) + for j := 0; j < 5; j++ { + if even { + if cd&masks[j] > 0 { + lon[0] = (lon[0] + lon[1]) / 2 + } else { + lon[1] = (lon[0] + lon[1]) / 2 + } + } else { + if cd&masks[j] > 0 { + lat[0] = (lat[0] + lat[1]) / 2 + } else { + lat[1] = (lat[0] + lat[1]) / 2 + } + } + even = !even + } + } + + return (lat[0] + lat[1]) / 2, (lon[0] + lon[1]) / 2 +} + +func EncodeGeoHash(lat, lon float64) string { + even := true + lats := []float64{-90.0, 90.0} + lons := []float64{-180.0, 180.0} + precision := 12 + var ch, bit uint64 + var geoHash string + + for len(geoHash) < precision { + if even { + mid := (lons[0] + lons[1]) / 2 + if lon > mid { + ch |= masks[bit] + lons[0] = mid + } else { + lons[1] = mid + } + } else { + mid := (lats[0] + lats[1]) / 2 + if lat > mid { + ch |= masks[bit] + lats[0] = mid + } else { + lats[1] = mid + } + } + even = !even + if bit < 4 { + bit++ + } else { + geoHash += string(base32encoding.enc[ch]) + ch = 0 + bit = 0 + } + } + + return geoHash +} diff --git a/geo/geohash_test.go b/geo/geohash_test.go new file mode 100644 index 0000000..11b920d --- /dev/null +++ b/geo/geohash_test.go @@ -0,0 +1,64 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geo + +import ( + "strings" + "testing" +) + +func TestDecodeGeoHash(t *testing.T) { + tests := []struct { + hash string + lon float64 + lat float64 + }{ + {"d3hn3", -73.059082, 6.745605}, // -73.05908203, 6.74560547 as per http://geohash.co/ + {"u4pru", 10.393066, 57.634277}, // 10.39306641, 57.63427734 + {"u4pruy", 10.409546, 57.648010}, // 10.40954590, 57.64801025 + {"u4pruyd", 10.407486, 57.648697}, // 10.40748596, 57.64869690 + {"u4pruydqqvj", 10.40744, 57.64911}, // 10.40743969, 57.64911063 + } + + for _, test := range tests { + lat, lon := DecodeGeoHash(test.hash) + + if compareGeo(test.lon, lon) != 0 { + t.Errorf("expected lon %f, got %f, hash %s", test.lon, lon, test.hash) + } + if compareGeo(test.lat, lat) != 0 { + t.Errorf("expected lat %f, got %f, hash %s", test.lat, lat, test.hash) + } + } +} + +func TestEncodeGeoHash(t *testing.T) { + tests := []struct { + lon float64 + lat float64 + hash string + }{ + {2.29449034, 48.85841131, "u09tunquc"}, + {76.491540, 10.060349, "t9y3hx7my0fp"}, + } + + for _, test := range tests { + hash := EncodeGeoHash(test.lat, test.lon) + + if !strings.HasPrefix(hash, test.hash) { + t.Errorf("expected hash %s, got %s", test.hash, hash) + } + } +} diff --git a/geo/parse.go b/geo/parse.go new file mode 100644 index 0000000..7be0a06 --- /dev/null +++ b/geo/parse.go @@ -0,0 +1,465 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geo + +import ( + "reflect" + "strconv" + "strings" + + "github.com/blevesearch/bleve/v2/util" + "github.com/blevesearch/geo/geojson" +) + +// ExtractGeoPoint takes an arbitrary interface{} and tries it's best to +// interpret it is as geo point. Supported formats: +// Container: +// slice length 2 (GeoJSON) +// +// first element lon, second element lat +// +// string (coordinates separated by comma, or a geohash) +// +// first element lat, second element lon +// +// map[string]interface{} +// +// exact keys lat and lon or lng +// +// struct +// +// w/exported fields case-insensitive match on lat and lon or lng +// +// struct +// +// satisfying Later and Loner or Lnger interfaces +// +// in all cases values must be some sort of numeric-like thing: int/uint/float +func ExtractGeoPoint(thing interface{}) (lon, lat float64, success bool) { + var foundLon, foundLat bool + + thingVal := reflect.ValueOf(thing) + if !thingVal.IsValid() { + return lon, lat, false + } + + thingTyp := thingVal.Type() + + // is it a slice + if thingVal.Kind() == reflect.Slice { + // must be length 2 + if thingVal.Len() == 2 { + first := thingVal.Index(0) + if first.CanInterface() { + firstVal := first.Interface() + lon, foundLon = util.ExtractNumericValFloat64(firstVal) + } + second := thingVal.Index(1) + if second.CanInterface() { + secondVal := second.Interface() + lat, foundLat = util.ExtractNumericValFloat64(secondVal) + } + } + } + + // is it a string + if thingVal.Kind() == reflect.String { + geoStr := thingVal.Interface().(string) + if strings.Contains(geoStr, ",") { + // geo point with coordinates split by comma + points := strings.Split(geoStr, ",") + for i, point := range points { + // trim any leading or trailing white spaces + points[i] = strings.TrimSpace(point) + } + if len(points) == 2 { + var err error + lat, err = strconv.ParseFloat(points[0], 64) + if err == nil { + foundLat = true + } + lon, err = strconv.ParseFloat(points[1], 64) + if err == nil { + foundLon = true + } + } + } else { + // geohash + if len(geoStr) <= geoHashMaxLength { + lat, lon = DecodeGeoHash(geoStr) + foundLat = true + foundLon = true + } + } + } + + // is it a map + if l, ok := thing.(map[string]interface{}); ok { + if lval, ok := l["lon"]; ok { + lon, foundLon = util.ExtractNumericValFloat64(lval) + } else if lval, ok := l["lng"]; ok { + lon, foundLon = util.ExtractNumericValFloat64(lval) + } + if lval, ok := l["lat"]; ok { + lat, foundLat = util.ExtractNumericValFloat64(lval) + } + } + + // now try reflection on struct fields + if thingVal.Kind() == reflect.Struct { + for i := 0; i < thingVal.NumField(); i++ { + fieldName := thingTyp.Field(i).Name + if strings.HasPrefix(strings.ToLower(fieldName), "lon") { + if thingVal.Field(i).CanInterface() { + fieldVal := thingVal.Field(i).Interface() + lon, foundLon = util.ExtractNumericValFloat64(fieldVal) + } + } + if strings.HasPrefix(strings.ToLower(fieldName), "lng") { + if thingVal.Field(i).CanInterface() { + fieldVal := thingVal.Field(i).Interface() + lon, foundLon = util.ExtractNumericValFloat64(fieldVal) + } + } + if strings.HasPrefix(strings.ToLower(fieldName), "lat") { + if thingVal.Field(i).CanInterface() { + fieldVal := thingVal.Field(i).Interface() + lat, foundLat = util.ExtractNumericValFloat64(fieldVal) + } + } + } + } + + // last hope, some interfaces + // lon + if l, ok := thing.(loner); ok { + lon = l.Lon() + foundLon = true + } else if l, ok := thing.(lnger); ok { + lon = l.Lng() + foundLon = true + } + // lat + if l, ok := thing.(later); ok { + lat = l.Lat() + foundLat = true + } + + return lon, lat, foundLon && foundLat +} + +// various support interfaces which can be used to find lat/lon +type loner interface { + Lon() float64 +} + +type later interface { + Lat() float64 +} + +type lnger interface { + Lng() float64 +} + +// GlueBytes primarily for quicker filtering of docvalues +// during the filtering phase. +var GlueBytes = []byte("##") + +var GlueBytesOffset = len(GlueBytes) + +func extractCoordinates(thing interface{}) []float64 { + thingVal := reflect.ValueOf(thing) + if !thingVal.IsValid() { + return nil + } + + if thingVal.Kind() == reflect.Slice { + // must be length 2 + if thingVal.Len() == 2 { + var foundLon, foundLat bool + var lon, lat float64 + first := thingVal.Index(0) + if first.CanInterface() { + firstVal := first.Interface() + lon, foundLon = util.ExtractNumericValFloat64(firstVal) + } + second := thingVal.Index(1) + if second.CanInterface() { + secondVal := second.Interface() + lat, foundLat = util.ExtractNumericValFloat64(secondVal) + } + + if !foundLon || !foundLat { + return nil + } + + return []float64{lon, lat} + } + } + return nil +} + +func extract2DCoordinates(thing interface{}) [][]float64 { + thingVal := reflect.ValueOf(thing) + if !thingVal.IsValid() { + return nil + } + + rv := make([][]float64, 0, 8) + if thingVal.Kind() == reflect.Slice { + for j := 0; j < thingVal.Len(); j++ { + edges := thingVal.Index(j).Interface() + if es, ok := edges.([]interface{}); ok { + v := extractCoordinates(es) + if len(v) == 2 { + rv = append(rv, v) + } + } + } + + return rv + } + + return nil +} + +func extract3DCoordinates(thing interface{}) (c [][][]float64) { + coords := reflect.ValueOf(thing) + if !coords.IsValid() { + return nil + } + + if coords.Kind() == reflect.Slice { + for i := 0; i < coords.Len(); i++ { + vals := coords.Index(i) + edges := vals.Interface() + if es, ok := edges.([]interface{}); ok { + loop := extract2DCoordinates(es) + if len(loop) > 0 { + c = append(c, loop) + } + } + } + } + + return c +} + +func extract4DCoordinates(thing interface{}) (rv [][][][]float64) { + thingVal := reflect.ValueOf(thing) + if !thingVal.IsValid() { + return nil + } + + if thingVal.Kind() == reflect.Slice { + for j := 0; j < thingVal.Len(); j++ { + c := extract3DCoordinates(thingVal.Index(j).Interface()) + rv = append(rv, c) + } + } + + return rv +} + +func ParseGeoShapeField(thing interface{}) (interface{}, string, error) { + thingVal := reflect.ValueOf(thing) + if !thingVal.IsValid() { + return nil, "", nil + } + + var shape string + var coordValue interface{} + + if thingVal.Kind() == reflect.Map { + iter := thingVal.MapRange() + for iter.Next() { + if iter.Key().String() == "type" { + shape = iter.Value().Interface().(string) + continue + } + + if iter.Key().String() == "coordinates" { + coordValue = iter.Value().Interface() + } + } + } + + return coordValue, strings.ToLower(shape), nil +} + +func extractGeoShape(thing interface{}) (*geojson.GeoShape, bool) { + + coordValue, typ, err := ParseGeoShapeField(thing) + if err != nil { + return nil, false + } + + if typ == CircleType { + return ExtractCircle(thing) + } + + return ExtractGeoShapeCoordinates(coordValue, typ) +} + +// ExtractGeometryCollection takes an interface{} and tries it's best to +// interpret all the member geojson shapes within it. +func ExtractGeometryCollection(thing interface{}) ([]*geojson.GeoShape, bool) { + thingVal := reflect.ValueOf(thing) + if !thingVal.IsValid() { + return nil, false + } + var rv []*geojson.GeoShape + var f bool + + if thingVal.Kind() == reflect.Map { + iter := thingVal.MapRange() + for iter.Next() { + + if iter.Key().String() == "type" { + continue + } + + if iter.Key().String() == "geometries" { + collection := iter.Value().Interface() + items := reflect.ValueOf(collection) + + for j := 0; j < items.Len(); j++ { + shape, found := extractGeoShape(items.Index(j).Interface()) + if found { + f = found + rv = append(rv, shape) + } + } + } + } + } + + return rv, f +} + +// ExtractCircle takes an interface{} and tries it's best to +// interpret the center point coordinates and the radius for a +// given circle shape. +func ExtractCircle(thing interface{}) (*geojson.GeoShape, bool) { + thingVal := reflect.ValueOf(thing) + if !thingVal.IsValid() { + return nil, false + } + + rv := &geojson.GeoShape{ + Type: CircleType, + Center: make([]float64, 0, 2), + } + + if thingVal.Kind() == reflect.Map { + iter := thingVal.MapRange() + for iter.Next() { + + if iter.Key().String() == "radius" { + rv.Radius = iter.Value().Interface().(string) + continue + } + + if iter.Key().String() == "coordinates" { + lng, lat, found := ExtractGeoPoint(iter.Value().Interface()) + if !found { + return nil, false + } + rv.Center = append(rv.Center, lng, lat) + } + } + } + + return rv, true +} + +// ExtractGeoShapeCoordinates takes an interface{} and tries it's best to +// interpret the coordinates for any of the given geoshape typ like +// a point, multipoint, linestring, multilinestring, polygon, multipolygon, +func ExtractGeoShapeCoordinates(coordValue interface{}, + typ string) (*geojson.GeoShape, bool) { + rv := &geojson.GeoShape{ + Type: typ, + } + + if typ == PointType { + point := extractCoordinates(coordValue) + + // ignore the contents with invalid entry. + if len(point) < 2 { + return nil, false + } + + rv.Coordinates = [][][][]float64{{{point}}} + return rv, true + } + + if typ == MultiPointType || typ == LineStringType || + typ == EnvelopeType { + coords := extract2DCoordinates(coordValue) + + // ignore the contents with invalid entry. + if len(coords) == 0 { + return nil, false + } + + if typ == EnvelopeType && len(coords) != 2 { + return nil, false + } + + if typ == LineStringType && len(coords) < 2 { + return nil, false + } + + rv.Coordinates = [][][][]float64{{coords}} + return rv, true + } + + if typ == PolygonType || typ == MultiLineStringType { + coords := extract3DCoordinates(coordValue) + + // ignore the contents with invalid entry. + if len(coords) == 0 { + return nil, false + } + + if typ == PolygonType && len(coords[0]) < 3 || + typ == MultiLineStringType && len(coords[0]) < 2 { + return nil, false + } + + rv.Coordinates = [][][][]float64{coords} + return rv, true + } + + if typ == MultiPolygonType { + coords := extract4DCoordinates(coordValue) + + // ignore the contents with invalid entry. + if len(coords) == 0 || len(coords[0]) == 0 { + return nil, false + + } + + if len(coords[0][0]) < 3 { + return nil, false + } + + rv.Coordinates = coords + return rv, true + } + + return rv, false +} diff --git a/geo/parse_test.go b/geo/parse_test.go new file mode 100644 index 0000000..c55623e --- /dev/null +++ b/geo/parse_test.go @@ -0,0 +1,478 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geo + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestExtractGeoPoint(t *testing.T) { + tests := []struct { + in interface{} + lon float64 + lat float64 + success bool + }{ + // values are ints + { + in: map[string]interface{}{ + "lat": 5, + "lon": 5, + }, + lon: 5, + lat: 5, + success: true, + }, + // values are uints + { + in: map[string]interface{}{ + "lat": uint(5), + "lon": uint(5), + }, + lon: 5, + lat: 5, + success: true, + }, + // values float64 as with parsed JSON + { + in: map[string]interface{}{ + "lat": 5.0, + "lon": 5.0, + }, + lon: 5, + lat: 5, + success: true, + }, + // values are bool (not supported) + { + in: map[string]interface{}{ + "lat": true, + "lon": false, + }, + lon: 0, + lat: 0, + success: false, + }, + // using lng variant of lon + { + in: map[string]interface{}{ + "lat": 5.0, + "lng": 5.0, + }, + lon: 5, + lat: 5, + success: true, + }, + // using struct + { + in: struct { + Lon float64 + Lat float64 + }{ + Lon: 3.0, + Lat: 7.5, + }, + lon: 3.0, + lat: 7.5, + success: true, + }, + // struct with lng alternate + { + in: struct { + Lng float64 + Lat float64 + }{ + Lng: 3.0, + Lat: 7.5, + }, + lon: 3.0, + lat: 7.5, + success: true, + }, + // test going through interface + { + in: &s11{ + lon: 4.0, + lat: 6.9, + }, + lon: 4.0, + lat: 6.9, + success: true, + }, + // test going through interface with lng variant + { + in: &s12{ + lng: 4.0, + lat: 6.9, + }, + lon: 4.0, + lat: 6.9, + success: true, + }, + // try GeoJSON slice + { + in: []interface{}{3.4, 5.9}, + lon: 3.4, + lat: 5.9, + success: true, + }, + // try GeoJSON slice too long + { + in: []interface{}{3.4, 5.9, 9.4}, + lon: 0, + lat: 0, + success: false, + }, + // slice of floats + { + in: []float64{3.4, 5.9}, + lon: 3.4, + lat: 5.9, + success: true, + }, + // values are nil (not supported) + { + in: map[string]interface{}{ + "lat": nil, + "lon": nil, + }, + lon: 0, + lat: 0, + success: false, + }, + // input is nil + { + in: nil, + lon: 0, + lat: 0, + success: false, + }, + } + + for _, test := range tests { + lon, lat, success := ExtractGeoPoint(test.in) + if success != test.success { + t.Errorf("expected extract geo point %t, got %t for %v", test.success, success, test.in) + } + if lon != test.lon { + t.Errorf("expected lon %f, got %f for %v", test.lon, lon, test.in) + } + if lat != test.lat { + t.Errorf("expected lat %f, got %f for %v", test.lat, lat, test.in) + } + } +} + +type s11 struct { + lon float64 + lat float64 +} + +func (s *s11) Lon() float64 { + return s.lon +} + +func (s *s11) Lat() float64 { + return s.lat +} + +type s12 struct { + lng float64 + lat float64 +} + +func (s *s12) Lng() float64 { + return s.lng +} + +func (s *s12) Lat() float64 { + return s.lat +} + +func TestExtractGeoShape(t *testing.T) { + tests := []struct { + in interface{} + resTyp string + coordinates [][][][]float64 + center []float64 + success bool + }{ + // valid point slice + { + in: map[string]interface{}{ + "coordinates": []interface{}{3.4, 5.9}, + "type": "Point", + }, + resTyp: "point", + coordinates: [][][][]float64{{{{3.4, 5.9}}}}, + success: true, + }, + // invalid point slice + { + in: map[string]interface{}{ + "coordinates": []interface{}{3.4}, + "type": "point"}, + + resTyp: "point", + coordinates: nil, + success: false, + }, + // valid multipoint slice containing single point + { + in: map[string]interface{}{ + "coordinates": [][]interface{}{{3.4, 5.9}}, + "type": "multipoint"}, + resTyp: "multipoint", + coordinates: [][][][]float64{{{{3.4, 5.9}}}}, + success: true, + }, + // valid multipoint slice + { + in: map[string]interface{}{ + "coordinates": [][]interface{}{{3.4, 5.9}, {6.7, 9.8}}, + "type": "multipoint"}, + resTyp: "multipoint", + coordinates: [][][][]float64{{{{3.4, 5.9}, {6.7, 9.8}}}}, + success: true, + }, + // valid multipoint slice containing one invalid entry + { + in: map[string]interface{}{ + "coordinates": [][]interface{}{{3.4, 5.9}, {6.7}}, + "type": "multipoint"}, + resTyp: "multipoint", + coordinates: [][][][]float64{{{{3.4, 5.9}}}}, + success: true, + }, + // invalid multipoint slice + { + in: map[string]interface{}{ + "coordinates": [][]interface{}{{3.4}}, + "type": "multipoint"}, + resTyp: "multipoint", + coordinates: nil, + success: false, + }, + // valid linestring slice + { + in: map[string]interface{}{ + "coordinates": [][]interface{}{{3.4, 4.4}, {8.4, 9.4}}, + "type": "linestring"}, + resTyp: "linestring", + coordinates: [][][][]float64{{{{3.4, 4.4}, {8.4, 9.4}}}}, + success: true, + }, + // valid linestring slice + { + in: map[string]interface{}{ + "coordinates": [][]interface{}{{3.4, 4.4}, {8.4, 9.4}, {10.1, 12.3}}, + "type": "linestring"}, + resTyp: "linestring", + coordinates: [][][][]float64{{{{3.4, 4.4}, {8.4, 9.4}, {10.1, 12.3}}}}, + success: true, + }, + // invalid linestring slice with single entry + { + in: map[string]interface{}{ + "coordinates": [][]interface{}{{3.4, 4.4}}, + "type": "linestring"}, + resTyp: "linestring", + coordinates: nil, + success: false, + }, + // invalid linestring slice with wrong parenthesis + { + in: map[string]interface{}{ + "coordinates": [][][]interface{}{{{3.4, 4.4}, {8.4, 9.4}}}, + "type": "linestring"}, + resTyp: "linestring", + coordinates: nil, + success: false, + }, + // valid envelope + { + in: map[string]interface{}{ + "coordinates": [][]interface{}{{3.4, 4.4}, {8.4, 9.4}}, + "type": "envelope"}, + resTyp: "envelope", + coordinates: [][][][]float64{{{{3.4, 4.4}, {8.4, 9.4}}}}, + success: true, + }, + // invalid envelope + { + in: map[string]interface{}{ + "coordinates": [][]interface{}{{3.4, 4.4}}, + "type": "envelope"}, + resTyp: "envelope", + coordinates: nil, + success: false, + }, + // invalid envelope + { + in: map[string]interface{}{ + "coordinates": [][][]interface{}{{{3.4, 4.4}, {8.4, 9.4}}}, + "type": "envelope"}, + resTyp: "envelope", + coordinates: nil, + success: false, + }, + // invalid envelope with >2 vertices + { + in: map[string]interface{}{ + "coordinates": [][]interface{}{{3.4, 4.4}, {5.6, 6.4}, {7.4, 7.4}}, + "type": "envelope"}, + resTyp: "envelope", + coordinates: nil, + success: false, + }, + // valid circle + { + in: map[string]interface{}{ + "coordinates": []interface{}{4.4, 5.0}, + "radius": "200m", + "type": "circle"}, + resTyp: "circle", + center: []float64{4.4, 5.0}, + success: true, + }, + // invalid circle + { + in: map[string]interface{}{ + "coordinates": []interface{}{4.4}, + "radius": "200m", + "type": "circle"}, + resTyp: "circle", + success: false, + }, + } + + for _, test := range tests { + res, success := extractGeoShape(test.in) + if success != test.success { + t.Errorf("expected extract geo point: %t, got: %t for: %v", test.success, success, test.in) + } + if success && res.Type != test.resTyp { + t.Errorf("expected shape type: %v, got: %v for input: %v", test.resTyp, res.Type, test.in) + } + if success && !reflect.DeepEqual(test.coordinates, res.Coordinates) { + t.Errorf("expected result %+v, got %+v for %v", test.coordinates, res.Coordinates, test.in) + } + if success && test.center != nil && !reflect.DeepEqual(test.center, res.Center) { + t.Errorf("expected center %+v, got %+v for %v", test.center, res.Center, test.in) + } + } +} + +func TestExtractGeoShapeCoordinates(t *testing.T) { + tests := []struct { + x []byte + typ string + expectOK bool + }{ + { + x: []byte(`[ + [ + [77.58894681930542,12.976498523818783], + [77.58677959442139,12.974533005048169], + [77.58894681930542,12.976498523818783] + ] + ]`), + typ: PolygonType, + expectOK: true, + }, + { // Invalid construct, but handled + x: []byte(`[ + [ + {"lon":77.58894681930542,"lat":12.976498523818783}, + {"lon":77.58677959442139,"lat":12.974533005048169}, + {"lon":77.58894681930542,"lat":12.976498523818783} + ] + ]`), + typ: PolygonType, + expectOK: false, + }, + { // Invalid construct causes panic (within extract3DCoordinates), fix MB-65807 + x: []byte(`{ + "coordinates": [ + [77.58894681930542,12.976498523818783], + [77.58677959442139,12.974533005048169], + [77.58894681930542,12.976498523818783] + ] + }`), + typ: PolygonType, + expectOK: false, + }, + { + x: []byte(`[ + [ + [ + [-0.163421630859375,51.531600743186644], + [-0.15277862548828125,51.52455221546295], + [-0.15895843505859375,51.53693981046689], + [-0.163421630859375,51.531600743186644] + ] + ], + [ + [ + [-0.1902008056640625,51.5091698216777], + [-0.1599884033203125,51.51322956905176], + [-0.1902008056640625,51.5091698216777] + ] + ] + ]`), + typ: MultiPolygonType, + expectOK: true, + }, + { // Invalid construct causes panic (within extract3DCoordinates), fix MB-65807 + x: []byte(`[ + { + "coordinates": [ + [-0.163421630859375,51.531600743186644], + [-0.15277862548828125,51.52455221546295], + [-0.15895843505859375,51.53693981046689], + [-0.163421630859375,51.531600743186644] + ] + }, + { + "coordinates": [ + [-0.1902008056640625,51.5091698216777], + [-0.1599884033203125,51.51322956905176], + [-0.1902008056640625,51.5091698216777] + ] + } + ]`), + typ: MultiPolygonType, + expectOK: false, + }, + } + + for i := range tests { + var x interface{} + if err := json.Unmarshal(tests[i].x, &x); err != nil { + t.Fatalf("[%d] JSON err: %v", i+1, err) + } + + res, ok := ExtractGeoShapeCoordinates(x, tests[i].typ) + if ok != tests[i].expectOK { + t.Errorf("[%d] expected ok %t, got %t", i+1, tests[i].expectOK, ok) + } + + if ok && res.Type != tests[i].typ { + t.Errorf("[%d] expected type %s, got %s", i+1, tests[i].typ, res.Type) + } + } +} diff --git a/geo/sloppy.go b/geo/sloppy.go new file mode 100644 index 0000000..e9de06d --- /dev/null +++ b/geo/sloppy.go @@ -0,0 +1,59 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geo + +import ( + "math" +) + +var earthDiameterPerLatitude []float64 + +const ( + radiusTabsSize = (1 << 10) + 1 + radiusDelta = (math.Pi / 2) / (radiusTabsSize - 1) + radiusIndexer = 1 / radiusDelta +) + +func init() { + // initializes the tables used for the sloppy math functions + + // earth radius + a := 6378137.0 + b := 6356752.31420 + a2 := a * a + b2 := b * b + earthDiameterPerLatitude = make([]float64, radiusTabsSize) + earthDiameterPerLatitude[0] = 2.0 * a / 1000 + earthDiameterPerLatitude[radiusTabsSize-1] = 2.0 * b / 1000 + for i := 1; i < radiusTabsSize-1; i++ { + lat := math.Pi * float64(i) / (2*radiusTabsSize - 1) + one := math.Pow(a2*math.Cos(lat), 2) + two := math.Pow(b2*math.Sin(lat), 2) + three := math.Pow(float64(a)*math.Cos(lat), 2) + four := math.Pow(b*math.Sin(lat), 2) + radius := math.Sqrt((one + two) / (three + four)) + earthDiameterPerLatitude[i] = 2 * radius / 1000 + } +} + +// earthDiameter returns an estimation of the earth's diameter at the specified +// latitude in kilometers +func earthDiameter(lat float64) float64 { + index := math.Mod(math.Abs(lat)*radiusIndexer+0.5, float64(len(earthDiameterPerLatitude))) + if math.IsNaN(index) { + return 0 + } + return earthDiameterPerLatitude[int(index)] +} diff --git a/geov2/cells.go b/geov2/cells.go new file mode 100644 index 0000000..3fac3d9 --- /dev/null +++ b/geov2/cells.go @@ -0,0 +1,49 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geov2 + +import "github.com/blevesearch/geo/s2" + +// getCellSearchBounds takes a raw uint64 S2 cell ID and returns the +// minimum and maximum uint64 values defining its absolute spatial range. +func getCellSearchBounds(cellUint uint64) (min uint64, max uint64) { + cellID := s2.CellID(cellUint) + + rangeMin := cellID.RangeMin() + rangeMax := cellID.RangeMax() + + return uint64(rangeMin), uint64(rangeMax) +} + +// Returns the level of the given S2 cell ID +func getCellLevel(cell uint64) uint64 { + return uint64(s2.CellID(cell).Level()) +} + +// Returns the parent cell ID of the given S2 cell ID at the specified level +func getParentCell(cell uint64, level int) uint64 { + return uint64(s2.CellID(cell).Parent(level)) +} + +// CalcCellsScore returns the total area of the given S2 cells +// in level 16 cell units (maxCellLevel in the geo repo's region coverer +// configuration - see geo/geojson/geojson_v2.go) +func CalcCellsScore(cells []uint64) uint64 { + var score uint64 + for _, cell := range cells { + score += calcScore(0, getCellLevel(cell)) + } + return score +} diff --git a/geov2/evaluator.go b/geov2/evaluator.go new file mode 100644 index 0000000..40c9917 --- /dev/null +++ b/geov2/evaluator.go @@ -0,0 +1,122 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geov2 + +import segment "github.com/blevesearch/scorch_segment_api/v2" + +type queryEvaluator struct { + queryInnerCells []uint64 + queryCrossCells []uint64 + + innerCells []uint64 + innerDocIDs []uint32 + + crossCells []uint64 + crossDocIDs []uint32 +} + +func NewQueryEvaluator(query Query, geoData segment.GeoShapeV2Data) *queryEvaluator { + return &queryEvaluator{ + queryInnerCells: query.InnerCells(), + queryCrossCells: query.CrossCells(), + innerCells: geoData.InnerCells(), + innerDocIDs: geoData.InnerDocIDs(), + crossCells: geoData.CrossCells(), + crossDocIDs: geoData.CrossDocIDs(), + } +} + +// find the leftmost index in arr where arr[i] >= target, or +// len(arr) if no such index exists +func binarySearchLeftmostGreaterOrEqual(arr []uint64, target uint64) int { + lo, hi := 0, len(arr) + for lo < hi { + mid := int(uint(lo+hi) >> 1) // bitshift avoids the addition overflow risk + if arr[mid] < target { + lo = mid + 1 + } else { + hi = mid + } + } + return lo +} + +// forEachScoredDoc invokes fn exactly once for every docID present in either +// score map, passing that doc's inner and cross scores (0 when the doc is +// absent from the corresponding map). +func forEachScoredDoc(innerScores, crossScores map[uint32]uint64, + fn func(id uint32, inner, cross uint64)) { + for id, inner := range innerScores { + fn(id, inner, crossScores[id]) + } + for id, cross := range crossScores { + if _, ok := innerScores[id]; !ok { + fn(id, 0, cross) + } + } +} + +// scan and score the overlap of query inner cells with all index cells +func (qe *queryEvaluator) rangeScanInner(innerScores, crossScores map[uint32]uint64) { + for _, cell := range qe.queryInnerCells { + minVal, maxVal := getCellSearchBounds(cell) + cellLevel := getCellLevel(cell) + rangeScanOne(cell, minVal, maxVal, cellLevel, qe.innerCells, qe.innerDocIDs, innerScores) + rangeScanOne(cell, minVal, maxVal, cellLevel, qe.crossCells, qe.crossDocIDs, crossScores) + } +} + +// scan and score the overlap of query cross cells with all index cells +func (qe *queryEvaluator) rangeScanCross(innerScores, crossScores map[uint32]uint64) { + for _, cell := range qe.queryCrossCells { + minVal, maxVal := getCellSearchBounds(cell) + cellLevel := getCellLevel(cell) + rangeScanOne(cell, minVal, maxVal, cellLevel, qe.innerCells, qe.innerDocIDs, innerScores) + rangeScanOne(cell, minVal, maxVal, cellLevel, qe.crossCells, qe.crossDocIDs, crossScores) + } +} + +// scan and score the overlap of a single query cell with the given index cells +func rangeScanOne(queryCell uint64, minVal, maxVal, cellLevel uint64, + indexCells []uint64, docIds []uint32, scores map[uint32]uint64) { + // find the range of index cells within the min/max bounds of the query cell + // end will be < start if there are no index cells within the bounds + start := binarySearchLeftmostGreaterOrEqual(indexCells, minVal) + end := binarySearchLeftmostGreaterOrEqual(indexCells, maxVal+1) - 1 + + // score all index cells within the bounds of the query cell + for i := start; i <= end; i++ { + id := docIds[i] + val := indexCells[i] + + valLevel := getCellLevel(val) + scores[id] += calcScore(cellLevel, valLevel) + } + + // score all parent cells of the query cell that are present in the index + // since parent cells are not within the min/max bounds + for level := int(cellLevel) - 1; level >= 0; level-- { + // get the parent cell of the query cell at this level + parentCell := getParentCell(queryCell, level) + // search for the leftmost index of this parent cell in the index cells + parentStart := binarySearchLeftmostGreaterOrEqual(indexCells, parentCell) + + // score all index cells that match this parent cell exactly + for i := parentStart; i < len(indexCells) && indexCells[i] == parentCell; i++ { + id := docIds[i] + scores[id] += calcScore(cellLevel, uint64(level)) + } + } +} diff --git a/geov2/evaluator_test.go b/geov2/evaluator_test.go new file mode 100644 index 0000000..0afed49 --- /dev/null +++ b/geov2/evaluator_test.go @@ -0,0 +1,128 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geov2 + +import ( + "sort" + "testing" + + "github.com/blevesearch/geo/s2" +) + +func TestBinarySearchLeftmostGreaterOrEqual(t *testing.T) { + tests := []struct { + name string + arr []uint64 + target uint64 + want int + }{ + {"empty", nil, 5, 0}, + {"all less than target", []uint64{1, 2, 3}, 5, 3}, + {"all greater than target", []uint64{5, 6, 7}, 1, 0}, + {"exact match", []uint64{1, 3, 5, 7}, 5, 2}, + {"target between elements", []uint64{1, 3, 5, 7}, 4, 2}, + {"leftmost of duplicates", []uint64{1, 3, 3, 3, 5}, 3, 1}, + {"target before all", []uint64{2, 4, 6}, 0, 0}, + {"target equals last", []uint64{2, 4, 6}, 6, 2}, + {"single element hit", []uint64{9}, 9, 0}, + {"single element miss high", []uint64{9}, 10, 1}, + } + for _, test := range tests { + if got := binarySearchLeftmostGreaterOrEqual(test.arr, test.target); got != test.want { + t.Errorf("%s: binarySearchLeftmostGreaterOrEqual(%v, %d) = %d, want %d", + test.name, test.arr, test.target, got, test.want) + } + } +} + +// TestRangeScanOne verifies the two scoring paths of rangeScanOne: index +// cells that fall within the query cell's range (the query cell itself and +// its descendants), and ancestor cells found by walking the query cell's +// parents (which lie outside the range). +func TestRangeScanOne(t *testing.T) { + const queryLevel = 10 + + queryCell := uint64(s2.CellIDFromFace(2).ChildBeginAtLevel(queryLevel)) + childCell := uint64(s2.CellID(queryCell).ChildBeginAtLevel(12)) // descendant, in range + parentCell := uint64(s2.CellID(queryCell).Parent(5)) // ancestor, out of range + unrelated := uint64(s2.CellIDFromFace(4).ChildBeginAtLevel(queryLevel)) + + // assign each cell a distinct doc ID + type entry struct { + cell uint64 + docID uint32 + } + entries := []entry{ + {queryCell, 0}, + {childCell, 1}, + {parentCell, 2}, + {unrelated, 3}, + } + + // index cells must be sorted ascending, with docIds kept parallel + sort.Slice(entries, func(i, j int) bool { + return entries[i].cell < entries[j].cell + }) + indexCells := make([]uint64, len(entries)) + docIds := make([]uint32, len(entries)) + for i, e := range entries { + indexCells[i] = e.cell + docIds[i] = e.docID + } + + scores := make(map[uint32]uint64) + minVal, maxVal := getCellSearchBounds(queryCell) + rangeScanOne(queryCell, minVal, maxVal, queryLevel, indexCells, docIds, scores) + + // docID 0 is the query cell itself: in range, equal levels + if want := calcScore(queryLevel, queryLevel); scores[0] != want { + t.Errorf("query cell score = %d, want %d", scores[0], want) + } + // docID 1 is a descendant at level 12: in range, index cell deeper + if want := calcScore(queryLevel, 12); scores[1] != want { + t.Errorf("descendant cell score = %d, want %d", scores[1], want) + } + // docID 2 is an ancestor at level 5: found via the parent walk + if want := calcScore(queryLevel, 5); scores[2] != want { + t.Errorf("ancestor cell score = %d, want %d", scores[2], want) + } + // docID 3 is unrelated (different face): must not be scored + if scores[3] != 0 { + t.Errorf("unrelated cell score = %d, want 0", scores[3]) + } +} + +// TestRangeScanOneNoMatches confirms that a query cell with no overlapping +// index cells produces no scores, exercising the empty-range case where +// end < start. +func TestRangeScanOneNoMatches(t *testing.T) { + const queryLevel = 10 + queryCell := uint64(s2.CellIDFromFace(0).ChildBeginAtLevel(queryLevel)) + + // index cells entirely on a different, non-ancestor part of the tree + indexCells := []uint64{ + uint64(s2.CellIDFromFace(5).ChildBeginAtLevel(queryLevel)), + uint64(s2.CellIDFromFace(5).ChildBeginAtLevel(queryLevel).Next()), + } + docIds := []uint32{0, 1} + scores := make(map[uint32]uint64) + + minVal, maxVal := getCellSearchBounds(queryCell) + rangeScanOne(queryCell, minVal, maxVal, queryLevel, indexCells, docIds, scores) + + if len(scores) != 0 { + t.Errorf("expected no scores, got %d entries: %v", len(scores), scores) + } +} diff --git a/geov2/query.go b/geov2/query.go new file mode 100644 index 0000000..8222b4e --- /dev/null +++ b/geov2/query.go @@ -0,0 +1,42 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geov2 + +import ( + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +type Query interface { + Evaluate(geoData segment.GeoShapeV2Data) *util.Bitset + InnerCells() []uint64 + CrossCells() []uint64 +} + +func NewQuery(shape index.GeoJSON, relation string) Query { + switch relation { + case "contains": + return NewContainsQuery(shape) + case "intersects": + return NewIntersectsQuery(shape) + case "within": + return NewWithinQuery(shape) + case "disjoint": + return NewDisjointQuery(shape) + default: + return nil + } +} diff --git a/geov2/query_contains.go b/geov2/query_contains.go new file mode 100644 index 0000000..08b5f2a --- /dev/null +++ b/geov2/query_contains.go @@ -0,0 +1,141 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geov2 + +import ( + "bytes" + + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/geo/geojson" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +type containsQuery struct { + innerCells []uint64 + crossCells []uint64 + + shape index.GeoJSON + bBox index.GeoJSON + + score uint64 +} + +func NewContainsQuery(shape index.GeoJSON) Query { + inner, cross := shape.QueryCells() + + score := CalcCellsScore(inner) + CalcCellsScore(cross) + + return &containsQuery{ + innerCells: inner, + crossCells: cross, + shape: shape, + bBox: shape.BoundingBox(), + score: score, + } +} + +func (cq *containsQuery) Evaluate(geoData segment.GeoShapeV2Data) *util.Bitset { + numDocs := int(geoData.NumDocs()) + exclude := geoData.Excluded() + + // create bitsets for hits and maybeHits providing exclude to the bitset + // which will make it impossible to set those bits + hits := util.NewBitset(numDocs, exclude) + maybeHits := util.NewBitset(numDocs, exclude) + + // failsafe for a degenerate query shape that produced no cells + if cq.score == 0 { + return hits + } + + // obtain zeroed score arrays from the segment-level pool and return + // them once the evaluation is done + innerScores := geoData.GetScoreMap() + crossScores := geoData.GetScoreMap() + defer geoData.PutScoreMap(innerScores) + defer geoData.PutScoreMap(crossScores) + + // create an evaluator instance to scan the query cells against the index cells + evaluator := NewQueryEvaluator(cq, geoData) + + // scan and score the overlap of all query cells with all index cells + evaluator.rangeScanInner(innerScores, crossScores) + evaluator.rangeScanCross(innerScores, crossScores) + + // if all of the query cells are contained within the inner index cells + // then we have a guaranteed hit, if they are contained within both the inner + // and cross index cells then we have a maybe hit, otherwise we have no hit + forEachScoredDoc(innerScores, crossScores, func(id uint32, inner, cross uint64) { + docNum := int(id) + if inner == cq.score { + hits.Add(docNum) + } else if inner+cross == cq.score { + maybeHits.Add(docNum) + } + }) + + var reader *bytes.Reader + + // filter out any maybeHits that do not have a bounding box that + // contains the query bounding box + boxFilter := func(docNum int) { + docBBoxBytes, err := geoData.BoundingBox(uint32(docNum)) + if docBBoxBytes == nil || err != nil { + return + } + + docBBox, err := geojson.ExtractShapesFromBytes(docBBoxBytes, &reader, nil) + if err != nil { + return + } + + if ok, err := docBBox.Contains(cq.bBox); err == nil && !ok { + maybeHits.Remove(docNum) + } + } + + maybeHits.Iterate(boxFilter) + + // filter out any maybeHits that do not have a shape that + // contains the query shape + shapeFilter := func(docNum int) { + docShapeBytes, err := geoData.Shape(uint32(docNum)) + if docShapeBytes == nil || err != nil { + return + } + + docShape, err := geojson.ExtractShapesFromBytes(docShapeBytes, &reader, nil) + if err != nil { + return + } + + if ok, err := docShape.Contains(cq.shape); err == nil && ok { + hits.Add(docNum) + } + } + + maybeHits.Iterate(shapeFilter) + + return hits +} + +func (cq *containsQuery) InnerCells() []uint64 { + return cq.innerCells +} + +func (cq *containsQuery) CrossCells() []uint64 { + return cq.crossCells +} diff --git a/geov2/query_disjoint.go b/geov2/query_disjoint.go new file mode 100644 index 0000000..f45d80c --- /dev/null +++ b/geov2/query_disjoint.go @@ -0,0 +1,66 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geov2 + +import ( + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +type disjointQuery struct { + innerCells []uint64 + crossCells []uint64 + + shape index.GeoJSON + bBox index.GeoJSON +} + +func NewDisjointQuery(shape index.GeoJSON) Query { + inner, cross := shape.QueryCells() + + return &disjointQuery{ + innerCells: inner, + crossCells: cross, + shape: shape, + bBox: shape.BoundingBox(), + } +} + +func (dq *disjointQuery) Evaluate(geoData segment.GeoShapeV2Data) *util.Bitset { + // evaluate the disjoint query by creating an intersects query and negating the results + intersectsQuery := &intersectsQuery{ + innerCells: dq.innerCells, + crossCells: dq.crossCells, + shape: dq.shape, + bBox: dq.bBox, + } + + // evaluate the intersects query to get the hits + hits := intersectsQuery.Evaluate(geoData) + + // invert the hits to get the disjoint results + hits.Invert() + + return hits +} + +func (dq *disjointQuery) InnerCells() []uint64 { + return dq.innerCells +} + +func (dq *disjointQuery) CrossCells() []uint64 { + return dq.crossCells +} diff --git a/geov2/query_intersects.go b/geov2/query_intersects.go new file mode 100644 index 0000000..fc3756a --- /dev/null +++ b/geov2/query_intersects.go @@ -0,0 +1,145 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geov2 + +import ( + "bytes" + + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/geo/geojson" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +type intersectsQuery struct { + innerCells []uint64 + crossCells []uint64 + + shape index.GeoJSON + bBox index.GeoJSON +} + +func NewIntersectsQuery(shape index.GeoJSON) Query { + inner, cross := shape.QueryCells() + + return &intersectsQuery{ + innerCells: inner, + crossCells: cross, + shape: shape, + bBox: shape.BoundingBox(), + } +} + +func (iq *intersectsQuery) Evaluate(geoData segment.GeoShapeV2Data) *util.Bitset { + numDocs := int(geoData.NumDocs()) + exclude := geoData.Excluded() + + // create bitsets for hits and maybeHits providing exclude to the bitset + // which will make it impossible to set those bits + hits := util.NewBitset(numDocs, exclude) + maybeHits := util.NewBitset(numDocs, exclude) + + // obtain zeroed score arrays from the segment-level pool and return + // them once the evaluation is done + innerScores := geoData.GetScoreMap() + crossScores := geoData.GetScoreMap() + defer geoData.PutScoreMap(innerScores) + defer geoData.PutScoreMap(crossScores) + + // create an evaluator instance to scan the query cells against the index cells + evaluator := NewQueryEvaluator(iq, geoData) + + // scan and score the overlap of query inner cells with all index cells + evaluator.rangeScanInner(innerScores, crossScores) + + // if there is any overlap of query inner cells with any of the index cells + // then we have a guaranteed hit. Reset scores to reuse score maps for the + // next step + forEachScoredDoc(innerScores, crossScores, func(id uint32, inner, cross uint64) { + if inner > 0 || cross > 0 { + hits.Add(int(id)) + } + }) + clear(innerScores) + clear(crossScores) + + // scan and score the overlap of query cross cells with all index cells + evaluator.rangeScanCross(innerScores, crossScores) + + // if there is any overlap of query cross cells with any of the index inner + // cells then we have a guaranteed hit, if there is any overlap of query cross + // cells with any of the index cross cells then we have a maybe hit, otherwise + // we have no hit. + forEachScoredDoc(innerScores, crossScores, func(id uint32, inner, cross uint64) { + docNum := int(id) + if inner > 0 && !hits.Contains(docNum) { + hits.Add(docNum) + } else if cross > 0 && !hits.Contains(docNum) { + maybeHits.Add(docNum) + } + }) + + var reader *bytes.Reader + + // filter out any maybeHits that do not have a bounding box that + // intersects the query bounding box + boxFilter := func(docNum int) { + docBBoxBytes, err := geoData.BoundingBox(uint32(docNum)) + if docBBoxBytes == nil || err != nil { + return + } + + docBBox, err := geojson.ExtractShapesFromBytes(docBBoxBytes, &reader, nil) + if err != nil { + return + } + + if ok, err := docBBox.Intersects(iq.bBox); err == nil && !ok { + maybeHits.Remove(docNum) + } + } + + maybeHits.Iterate(boxFilter) + + // filter out any maybeHits that do not have a shape that + // intersects the query shape + shapeFilter := func(docNum int) { + docShapeBytes, err := geoData.Shape(uint32(docNum)) + if docShapeBytes == nil || err != nil { + return + } + + docShape, err := geojson.ExtractShapesFromBytes(docShapeBytes, &reader, nil) + if err != nil { + return + } + + if ok, err := docShape.Intersects(iq.shape); err == nil && ok { + hits.Add(docNum) + } + } + + maybeHits.Iterate(shapeFilter) + + return hits +} + +func (iq *intersectsQuery) InnerCells() []uint64 { + return iq.innerCells +} + +func (iq *intersectsQuery) CrossCells() []uint64 { + return iq.crossCells +} diff --git a/geov2/query_within.go b/geov2/query_within.go new file mode 100644 index 0000000..913a167 --- /dev/null +++ b/geov2/query_within.go @@ -0,0 +1,146 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geov2 + +import ( + "bytes" + + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/geo/geojson" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +type withinQuery struct { + innerCells []uint64 + crossCells []uint64 + + shape index.GeoJSON + bBox index.GeoJSON +} + +func NewWithinQuery(shape index.GeoJSON) Query { + inner, cross := shape.QueryCells() + + return &withinQuery{ + innerCells: inner, + crossCells: cross, + shape: shape, + bBox: shape.BoundingBox(), + } +} + +func (wq *withinQuery) Evaluate(geoData segment.GeoShapeV2Data) *util.Bitset { + numDocs := int(geoData.NumDocs()) + exclude := geoData.Excluded() + + // create bitsets for hits and maybeHits providing exclude to the bitset + // which will make it impossible to set those bits + hits := util.NewBitset(numDocs, exclude) + maybeHits := util.NewBitset(numDocs, exclude) + + // obtain zeroed score arrays from the segment-level pool and return + // them once the evaluation is done + innerScores := geoData.GetScoreMap() + crossScores := geoData.GetScoreMap() + defer geoData.PutScoreMap(innerScores) + defer geoData.PutScoreMap(crossScores) + + docScoresInner, docScoresCross := geoData.DocScores() + + // create an evaluator instance to scan the query cells against the index cells + evaluator := NewQueryEvaluator(wq, geoData) + + // scan and score the overlap of query inner cells with all index cells + evaluator.rangeScanInner(innerScores, crossScores) + + // if all of the index cells are contained within the query inner cells, + // then we have a guaranteed hit. Only consider documents with non zero + // total scores as hits + forEachScoredDoc(innerScores, crossScores, func(id uint32, inner, cross uint64) { + total := inner + cross + if total == docScoresInner[id]+docScoresCross[id] && total != 0 { + hits.Add(int(id)) + } + }) + + // scan and score the overlap of query cross cells with all index cells + evaluator.rangeScanCross(innerScores, crossScores) + + // A document is a maybe-hit once the accumulated score reaches at least + // its own inner-cell score. Complete overlap of index and cross scores + // may not always be possible due to the nature of region coverer being + // non exhaustive in the sense that the boundaries are not always only + // covered by the smallest possible cells. + forEachScoredDoc(innerScores, crossScores, func(id uint32, inner, cross uint64) { + docNum := int(id) + total := inner + cross + if !hits.Contains(docNum) && total >= docScoresInner[id] && total != 0 { + maybeHits.Add(docNum) + } + }) + + var reader *bytes.Reader + + // filter out any maybeHits that do not have a bounding box that + // is within the query bounding box + boxFilter := func(docNum int) { + docBBoxBytes, err := geoData.BoundingBox(uint32(docNum)) + if docBBoxBytes == nil || err != nil { + return + } + + docBBox, err := geojson.ExtractShapesFromBytes(docBBoxBytes, &reader, nil) + if err != nil { + return + } + + if ok, err := wq.bBox.Contains(docBBox); err == nil && !ok { + maybeHits.Remove(docNum) + } + } + + maybeHits.Iterate(boxFilter) + + // filter out any maybeHits that do not have a shape that + // is within the query shape + shapeFilter := func(docNum int) { + docShapeBytes, err := geoData.Shape(uint32(docNum)) + if docShapeBytes == nil || err != nil { + return + } + + docShape, err := geojson.ExtractShapesFromBytes(docShapeBytes, &reader, nil) + if err != nil { + return + } + + if ok, err := wq.shape.Contains(docShape); err == nil && ok { + hits.Add(docNum) + } + } + + maybeHits.Iterate(shapeFilter) + + return hits +} + +func (wq *withinQuery) InnerCells() []uint64 { + return wq.innerCells +} + +func (wq *withinQuery) CrossCells() []uint64 { + return wq.crossCells +} diff --git a/geov2/score.go b/geov2/score.go new file mode 100644 index 0000000..920fe5c --- /dev/null +++ b/geov2/score.go @@ -0,0 +1,74 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geov2 + +// pow4Table precomputes all valid powers of 4 for a uint64. +// Array size 32 covers exp 0 through 31. +var pow4Table = [32]uint64{ + 1, // 4^0 + 4, // 4^1 + 16, // 4^2 + 64, // 4^3 + 256, // 4^4 + 1024, // 4^5 + 4096, // 4^6 + 16384, // 4^7 + 65536, // 4^8 + 262144, // 4^9 + 1048576, // 4^10 + 4194304, // 4^11 + 16777216, // 4^12 + 67108864, // 4^13 + 268435456, // 4^14 + 1073741824, // 4^15 + 4294967296, // 4^16 + 17179869184, // 4^17 + 68719476736, // 4^18 + 274877906944, // 4^19 + 1099511627776, // 4^20 + 4398046511104, // 4^21 + 17592186044416, // 4^22 + 70368744177664, // 4^23 + 281474976710656, // 4^24 + 1125899906842624, // 4^25 + 4503599627370496, // 4^26 + 18014398509481984, // 4^27 + 72057594037927936, // 4^28 + 288230376151711744, // 4^29 + 1152921504606846976, // 4^30 + 4611686018427387904, // 4^31 +} + +// pow4 returns 4^exp quickly using the lookup table. +func pow4(exp uint64) uint64 { + if exp >= 32 { + // Handle overflow safely. + return 0 + } + return pow4Table[exp] +} + +// returns the overlap of query and index cells based on their levels. +// Both levels are assumed to be at most 16 (maxCellLevel in the geo repo's +// region coverer configuration - see geo/geojson/geojson_v2.go), the deepest +// level used across the geoshape_v2 indexing and query cell coverings; +// cells deeper than level 16 are outside this function's contract by design. +func calcScore(queryCellLevel, indexCellLevel uint64) uint64 { + if indexCellLevel > queryCellLevel { + return pow4(16 - indexCellLevel) + } else { + return pow4(16 - queryCellLevel) + } +} diff --git a/geov2/score_test.go b/geov2/score_test.go new file mode 100644 index 0000000..650f0fe --- /dev/null +++ b/geov2/score_test.go @@ -0,0 +1,117 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geov2 + +import ( + "testing" + + "github.com/blevesearch/geo/s2" +) + +func TestPow4(t *testing.T) { + tests := []struct { + exp uint64 + want uint64 + }{ + {0, 1}, + {1, 4}, + {10, 1048576}, + {15, 1073741824}, + {31, 4611686018427387904}, // last representable power of 4 in a uint64 + {32, 0}, // out of range, guarded to 0 + {100, 0}, // well out of range, guarded to 0 + } + for _, test := range tests { + if got := pow4(test.exp); got != test.want { + t.Errorf("pow4(%d) = %d, want %d", test.exp, got, test.want) + } + } +} + +func TestCalcScore(t *testing.T) { + tests := []struct { + name string + queryCellLevel uint64 + indexCellLevel uint64 + want uint64 + }{ + { + // equal levels: the overlap is a single cell at that level + name: "equal levels", queryCellLevel: 16, indexCellLevel: 16, want: pow4(0), + }, + { + name: "equal levels shallow", queryCellLevel: 6, indexCellLevel: 6, want: pow4(10), + }, + { + // index cell is deeper (smaller): the overlap is the index cell + name: "index deeper", queryCellLevel: 6, indexCellLevel: 10, want: pow4(6), + }, + { + // query cell is deeper (smaller): the overlap is the query cell + name: "query deeper", queryCellLevel: 11, indexCellLevel: 8, want: pow4(5), + }, + { + name: "both at level 0", queryCellLevel: 0, indexCellLevel: 0, want: pow4(16), + }, + } + for _, test := range tests { + if got := calcScore(test.queryCellLevel, test.indexCellLevel); got != test.want { + t.Errorf("%s: calcScore(%d, %d) = %d, want %d", + test.name, test.queryCellLevel, test.indexCellLevel, got, test.want) + } + } +} + +// cellAtLevel returns a valid S2 cell ID at the requested level, derived +// deterministically from a fixed face so tests do not depend on any RNG. +func cellAtLevel(level int) uint64 { + return uint64(s2.CellIDFromFace(1).ChildBeginAtLevel(level)) +} + +func TestCalcCellsScore(t *testing.T) { + // build cells at known levels; each contributes calcScore(0, level) + // which equals pow4(16 - level) + cells := []uint64{ + cellAtLevel(16), // contributes pow4(0) = 1 + cellAtLevel(15), // contributes pow4(1) = 4 + cellAtLevel(14), // contributes pow4(2) = 16 + } + want := pow4(0) + pow4(1) + pow4(2) + if got := CalcCellsScore(cells); got != want { + t.Fatalf("CalcCellsScore = %d, want %d", got, want) + } + + // an empty slice scores zero + if got := CalcCellsScore(nil); got != 0 { + t.Fatalf("CalcCellsScore(nil) = %d, want 0", got) + } +} + +func TestCellLevelAndParent(t *testing.T) { + cell := cellAtLevel(12) + if got := getCellLevel(cell); got != 12 { + t.Fatalf("getCellLevel = %d, want 12", got) + } + parent := getParentCell(cell, 8) + if got := getCellLevel(parent); got != 8 { + t.Fatalf("getCellLevel(parent) = %d, want 8", got) + } + // the parent's range must contain the child cell + minVal, maxVal := getCellSearchBounds(parent) + if cell < minVal || cell > maxVal { + t.Fatalf("expected child cell %d to fall within parent bounds [%d, %d]", + cell, minVal, maxVal) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..bce5da2 --- /dev/null +++ b/go.mod @@ -0,0 +1,46 @@ +module github.com/blevesearch/bleve/v2 + +go 1.25.0 + +require ( + github.com/RoaringBitmap/roaring/v2 v2.14.5 + github.com/bits-and-blooms/bitset v1.24.2 + github.com/blevesearch/bleve_index_api v1.4.1 + github.com/blevesearch/geo v0.2.7-0.20260831141219-9c3e3a4d84bc + github.com/blevesearch/go-faiss v1.1.5 + github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475 + github.com/blevesearch/go-porterstemmer v1.0.3 + github.com/blevesearch/goleveldb v1.0.1 + github.com/blevesearch/gtreap v0.1.1 + github.com/blevesearch/scorch_segment_api/v2 v2.4.10 + github.com/blevesearch/segment v0.9.1 + github.com/blevesearch/snowball v0.6.1 + github.com/blevesearch/snowballstem v0.9.0 + github.com/blevesearch/stempel v0.2.0 + github.com/blevesearch/upsidedown_store_api v1.0.2 + github.com/blevesearch/vellum v1.2.0 + github.com/blevesearch/zapx/v11 v11.4.3 + github.com/blevesearch/zapx/v12 v12.4.3 + github.com/blevesearch/zapx/v13 v13.4.3 + github.com/blevesearch/zapx/v14 v14.4.3 + github.com/blevesearch/zapx/v15 v15.4.3 + github.com/blevesearch/zapx/v16 v16.3.4 + github.com/blevesearch/zapx/v17 v17.2.3 + github.com/couchbase/moss v0.2.0 + github.com/spf13/cobra v1.10.2 + go.etcd.io/bbolt v1.4.0 + golang.org/x/text v0.37.0 + google.golang.org/protobuf v1.36.6 +) + +require ( + github.com/blevesearch/mmap-go v1.2.0 // indirect + github.com/couchbase/ghistogram v0.1.0 // indirect + github.com/golang/snappy v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede // indirect + github.com/mschoch/smat v0.2.0 // indirect + github.com/spf13/pflag v1.0.9 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/sys v0.45.0 // indirect +) diff --git a/index.go b/index.go new file mode 100644 index 0000000..889da1f --- /dev/null +++ b/index.go @@ -0,0 +1,413 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "context" + "fmt" + + "github.com/blevesearch/bleve/v2/index/upsidedown" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +// A Batch groups together multiple Index and Delete +// operations you would like performed at the same +// time. The Batch structure is NOT thread-safe. +// You should only perform operations on a batch +// from a single thread at a time. Once batch +// execution has started, you may not modify it. +type Batch struct { + index Index + internal *index.Batch + + lastDocSize uint64 + totalSize uint64 +} + +// Index adds the specified index operation to the +// batch. NOTE: the bleve Index is not updated +// until the batch is executed. +func (b *Batch) Index(id string, data interface{}) error { + if id == "" { + return ErrorEmptyID + } + if eventIndex, ok := b.index.(index.EventIndex); ok { + eventIndex.FireIndexEvent() + } + doc := document.NewDocument(id) + err := b.index.Mapping().MapDocument(doc, data) + if err != nil { + return err + } + b.internal.Update(doc) + + b.lastDocSize = uint64(doc.Size() + + len(id) + size.SizeOfString) // overhead from internal + b.totalSize += b.lastDocSize + + return nil +} + +func (b *Batch) IndexSynonym(id string, collection string, definition *SynonymDefinition) error { + if id == "" { + return ErrorEmptyID + } + if eventIndex, ok := b.index.(index.EventIndex); ok { + eventIndex.FireIndexEvent() + } + synMap, ok := b.index.Mapping().(mapping.SynonymMapping) + if !ok { + return ErrorSynonymSearchNotSupported + } + + if err := definition.Validate(); err != nil { + return err + } + + doc := document.NewSynonymDocument(id) + err := synMap.MapSynonymDocument(doc, collection, definition.Input, definition.Synonyms) + if err != nil { + return err + } + b.internal.Update(doc) + + b.lastDocSize = uint64(doc.Size() + + len(id) + size.SizeOfString) // overhead from internal + b.totalSize += b.lastDocSize + + return nil +} + +func (b *Batch) LastDocSize() uint64 { + return b.lastDocSize +} + +func (b *Batch) TotalDocsSize() uint64 { + return b.totalSize +} + +// IndexAdvanced adds the specified index operation to the +// batch which skips the mapping. NOTE: the bleve Index is not updated +// until the batch is executed. +func (b *Batch) IndexAdvanced(doc *document.Document) (err error) { + if doc.ID() == "" { + return ErrorEmptyID + } + b.internal.Update(doc) + return nil +} + +// Delete adds the specified delete operation to the +// batch. NOTE: the bleve Index is not updated until +// the batch is executed. +func (b *Batch) Delete(id string) { + if id != "" { + b.internal.Delete(id) + } +} + +// SetInternal adds the specified set internal +// operation to the batch. NOTE: the bleve Index is +// not updated until the batch is executed. +func (b *Batch) SetInternal(key, val []byte) { + b.internal.SetInternal(key, val) +} + +// DeleteInternal adds the specified delete internal +// operation to the batch. NOTE: the bleve Index is +// not updated until the batch is executed. +func (b *Batch) DeleteInternal(key []byte) { + b.internal.DeleteInternal(key) +} + +// Size returns the total number of operations inside the batch +// including normal index operations and internal operations. +func (b *Batch) Size() int { + return len(b.internal.IndexOps) + len(b.internal.InternalOps) +} + +// String prints a user friendly string representation of what +// is inside this batch. +func (b *Batch) String() string { + return b.internal.String() +} + +// Reset returns a Batch to the empty state so that it can +// be reused in the future. +func (b *Batch) Reset() { + b.internal.Reset() + b.lastDocSize = 0 + b.totalSize = 0 +} + +func (b *Batch) Merge(o *Batch) { + if o != nil && o.internal != nil { + b.internal.Merge(o.internal) + if o.LastDocSize() > 0 { + b.lastDocSize = o.LastDocSize() + } + b.totalSize = uint64(b.internal.TotalDocSize()) + } +} + +func (b *Batch) SetPersistedCallback(f index.BatchCallback) { + b.internal.SetPersistedCallback(f) +} + +func (b *Batch) PersistedCallback() index.BatchCallback { + return b.internal.PersistedCallback() +} + +// An Index implements all the indexing and searching +// capabilities of bleve. An Index can be created +// using the New() and Open() methods. +// +// Index() takes an input value, deduces a DocumentMapping for its type, +// assigns string paths to its fields or values then applies field mappings on +// them. +// +// The DocumentMapping used to index a value is deduced by the following rules: +// 1. If value implements mapping.bleveClassifier interface, resolve the mapping +// from BleveType(). +// 2. If value implements mapping.Classifier interface, resolve the mapping +// from Type(). +// 3. If value has a string field or value at IndexMapping.TypeField. +// +// (defaulting to "_type"), use it to resolve the mapping. Fields addressing +// is described below. +// 4) If IndexMapping.DefaultType is registered, return it. +// 5) Return IndexMapping.DefaultMapping. +// +// Each field or nested field of the value is identified by a string path, then +// mapped to one or several FieldMappings which extract the result for analysis. +// +// Struct values fields are identified by their "json:" tag, or by their name. +// Nested fields are identified by prefixing with their parent identifier, +// separated by a dot. +// +// Map values entries are identified by their string key. Entries not indexed +// by strings are ignored. Entry values are identified recursively like struct +// fields. +// +// Slice and array values are identified by their field name. Their elements +// are processed sequentially with the same FieldMapping. +// +// String, float64 and time.Time values are identified by their field name. +// Other types are ignored. +// +// Each value identifier is decomposed in its parts and recursively address +// SubDocumentMappings in the tree starting at the root DocumentMapping. If a +// mapping is found, all its FieldMappings are applied to the value. If no +// mapping is found and the root DocumentMapping is dynamic, default mappings +// are used based on value type and IndexMapping default configurations. +// +// Finally, mapped values are analyzed, indexed or stored. See +// FieldMapping.Analyzer to know how an analyzer is resolved for a given field. +// +// Examples: +// +// type Date struct { +// Day string `json:"day"` +// Month string +// Year string +// } +// +// type Person struct { +// FirstName string `json:"first_name"` +// LastName string +// BirthDate Date `json:"birth_date"` +// } +// +// A Person value FirstName is mapped by the SubDocumentMapping at +// "first_name". Its LastName is mapped by the one at "LastName". The day of +// BirthDate is mapped to the SubDocumentMapping "day" of the root +// SubDocumentMapping "birth_date". It will appear as the "birth_date.day" +// field in the index. The month is mapped to "birth_date.Month". +type Index interface { + // Index analyzes, indexes or stores mapped data fields. Supplied + // identifier is bound to analyzed data and will be retrieved by search + // requests. See Index interface documentation for details about mapping + // rules. + Index(id string, data interface{}) error + Delete(id string) error + + NewBatch() *Batch + Batch(b *Batch) error + + // Document returns specified document or nil if the document is not + // indexed or stored. + Document(id string) (index.Document, error) + // DocCount returns the number of documents in the index. + DocCount() (uint64, error) + + Search(req *SearchRequest) (*SearchResult, error) + SearchInContext(ctx context.Context, req *SearchRequest) (*SearchResult, error) + + Fields() ([]string, error) + + FieldDict(field string) (index.FieldDict, error) + FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) + FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) + + Close() error + + Mapping() mapping.IndexMapping + + Stats() *IndexStat + StatsMap() map[string]interface{} + + GetInternal(key []byte) ([]byte, error) + SetInternal(key, val []byte) error + DeleteInternal(key []byte) error + + // Name returns the name of the index (by default this is the path) + Name() string + // SetName lets you assign your own logical name to this index + SetName(string) + + // Advanced returns the internal index implementation + Advanced() (index.Index, error) +} + +// New index at the specified path, must not exist. +// The provided mapping will be used for all +// Index/Search operations. +func New(path string, mapping mapping.IndexMapping) (Index, error) { + return newIndexUsing(path, mapping, Config.DefaultIndexType, Config.DefaultKVStore, nil) +} + +// NewMemOnly creates a memory-only index. +// The contents of the index is NOT persisted, +// and will be lost once closed. +// The provided mapping will be used for all +// Index/Search operations. +func NewMemOnly(mapping mapping.IndexMapping) (Index, error) { + return newIndexUsing("", mapping, upsidedown.Name, Config.DefaultMemKVStore, nil) +} + +// NewUsing creates index at the specified path, +// which must not already exist. +// The provided mapping will be used for all +// Index/Search operations. +// The specified index type will be used. +// The specified kvstore implementation will be used +// and the provided kvconfig will be passed to its +// constructor. Note that currently the values of kvconfig must +// be able to be marshaled and unmarshaled using the encoding/json library (used +// when reading/writing the index metadata file). +func NewUsing(path string, mapping mapping.IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (Index, error) { + return newIndexUsing(path, mapping, indexType, kvstore, kvconfig) +} + +// Open index at the specified path, must exist. +// The mapping used when it was created will be used for all Index/Search operations. +func Open(path string) (Index, error) { + return openIndexUsing(path, nil) +} + +// OpenUsing opens index at the specified path, must exist. +// The mapping used when it was created will be used for all Index/Search operations. +// The provided runtimeConfig can override settings +// persisted when the kvstore was created. +// If runtimeConfig has updated mapping, then an index update is attempted +// Throws an error without any changes to the index if an unupdatable mapping is provided +func OpenUsing(path string, runtimeConfig map[string]interface{}) (Index, error) { + return openIndexUsing(path, runtimeConfig) +} + +// Builder is a limited interface, used to build indexes in an offline mode. +// Items cannot be updated or deleted, and the caller MUST ensure a document is +// indexed only once. +type Builder interface { + Index(id string, data interface{}) error + Close() error +} + +// NewBuilder creates a builder, which will build an index at the specified path, +// using the specified mapping and options. +func NewBuilder(path string, mapping mapping.IndexMapping, config map[string]interface{}) (Builder, error) { + return newBuilder(path, mapping, config) +} + +// IndexCopyable is an index which supports an online copy operation +// of the index. +type IndexCopyable interface { + // CopyTo creates a fully functional copy of the index at the + // specified destination directory implementation. + CopyTo(d index.Directory) error +} + +// FileSystemDirectory is the default implementation for the +// index.Directory interface. +type FileSystemDirectory string + +// SynonymDefinition represents a synonym mapping in Bleve. +// Each instance associates one or more input terms with a list of synonyms, +// defining how terms are treated as equivalent in searches. +type SynonymDefinition struct { + // Input is an optional list of terms for unidirectional synonym mapping. + // When terms are specified in Input, they will map to the terms in Synonyms, + // making the relationship unidirectional (each Input maps to all Synonyms). + // If Input is omitted, the relationship is bidirectional among all Synonyms. + Input []string `json:"input,omitempty"` + + // Synonyms is a list of terms that are considered equivalent. + // If Input is specified, each term in Input will map to each term in Synonyms. + // If Input is not specified, the Synonyms list will be treated bidirectionally, + // meaning each term in Synonyms is treated as synonymous with all others. + Synonyms []string `json:"synonyms"` +} + +func (sd *SynonymDefinition) Validate() error { + if len(sd.Synonyms) == 0 { + return fmt.Errorf("synonym definition must have at least one synonym") + } + return nil +} + +// SynonymIndex supports indexing synonym definitions alongside regular documents. +// Synonyms, grouped by collection name, define term relationships for query expansion in searches. +type SynonymIndex interface { + Index + // IndexSynonym indexes a synonym definition, with the specified id and belonging to the specified collection. + IndexSynonym(id string, collection string, definition *SynonymDefinition) error +} + +type IndexWithCallbacks interface { + FileWriterIDsInUse() (map[string]struct{}, error) + DropFileWriterIDs(ids map[string]struct{}) error +} + +type InsightsIndex interface { + Index + // TermFrequencies returns the tokens ordered by frequencies for the field index. + TermFrequencies(field string, limit int, descending bool) ([]index.TermFreq, error) + // CentroidCardinalities returns the centroids (clusters) from IVF indexes ordered by data density. + CentroidCardinalities(field string, limit int, desceding bool) ([]index.CentroidCardinality, error) +} + +type TrainableIndex interface { + Index + Train(*Batch) error +} + +type IndexFileCopyable interface { + SetPathInBolt(key []byte, value []byte) error //dest index + CopyFile(file string, d index.IndexDirectory) error // source index +} diff --git a/index/scorch/README.md b/index/scorch/README.md new file mode 100644 index 0000000..e385676 --- /dev/null +++ b/index/scorch/README.md @@ -0,0 +1,369 @@ +# scorch + +## Definitions + +Batch + +- A collection of Documents to mutate in the index. + +Document + +- Has a unique identifier (arbitrary bytes). +- Is comprised of a list of fields. + +Field + +- Has a name (string). +- Has a type (text, number, date, geopoint). +- Has a value (depending on type). +- Can be indexed, stored, or both. +- If indexed, can be analyzed. +-m If indexed, can optionally store term vectors. + +## Scope + +Scorch *MUST* implement the bleve.index API without requiring any changes to this API. + +Scorch *MAY* introduce new interfaces, which can be discovered to allow use of new capabilities not in the current API. + +## Implementation + +The scorch implementation starts with the concept of a segmented index. + +A segment is simply a slice, subset, or portion of the entire index. A segmented index is one which is composed of one or more segments. Although segments are created in a particular order, knowing this ordering is not required to achieve correct semantics when querying. Because there is no ordering, this means that when searching an index, you can (and should) search all the segments concurrently. + +### Internal Wrapper + +In order to accommodate the existing APIs while also improving the implementation, the scorch implementation includes some wrapper functionality that must be described. + +#### \_id field + +In scorch, field 0 is prearranged to be named \_id. All documents have a value for this field, which is the documents external identifier. In this version the field *MUST* be both indexed AND stored. The scorch wrapper adds this field, as it will not be present in the Document from the calling bleve code. + +NOTE: If a document already contains a field \_id, it will be replaced. If this is problematic, the caller must ensure such a scenario does not happen. + +### Proposed Structures + +```go +type Segment interface { + + Dictionary(field string) TermDictionary + +} + +type TermDictionary interface { + + PostingsList(term string, excluding PostingsList) PostingsList + +} + +type PostingsList interface { + + Next() Posting + + And(other PostingsList) PostingsList + Or(other PostingsList) PostingsList + +} + +type Posting interface { + Number() uint64 + + Frequency() uint64 + Norm() float64 + + Locations() Locations +} + +type Locations interface { + Start() uint64 + End() uint64 + Pos() uint64 + ArrayPositions() ... +} + +type DeletedDocs { + +} + +type SegmentSnapshot struct { + segment Segment + deleted PostingsList +} + +type IndexSnapshot struct { + segment []SegmentSnapshot +} +``` + +**What about errors?** +**What about memory mgmnt or context?** +**Postings List separate iterator to separate stateful from stateless** + +### Mutating the Index + +The bleve.index API has methods for directly making individual mutations (Update/Delete/SetInternal/DeleteInternal), however for this first implementation, we assume that all of these calls can simply be turned into a Batch of size 1. This may be highly inefficient, but it will be correct. This decision is made based on the fact that Couchbase FTS always uses Batches. + +NOTE: As a side-effect of this decision, it should be clear that performance tuning may depend on the batch size, which may in-turn require changes in FTS. + +From this point forward, only Batch mutations will be discussed. + +Sequence of Operations: + +1. For each document in the batch, search through all existing segments. The goal is to build up a per-segment bitset which tells us which documents in that segment are obsoleted by the addition of the new segment we're currently building. NOTE: we're not ready for this change to take effect yet, so rather than this operation mutating anything, they simply return bitsets, which we can apply later. Logically, this is something like: + + ```go + foreach segment { + dict := segment.Dictionary("\_id") + postings := empty postings list + foreach docID { + postings = postings.Or(dict.PostingsList(docID, nil)) + } + } + ``` + + NOTE: it is illustrated above as nested for loops, but some or all of these could be concurrently. The end result is that for each segment, we have (possibly empty) bitset. + +2. Also concurrent with 1, the documents in the batch are analyzed. This analysis proceeds using the existing analyzer pool. + +3. (after 2 completes) Analyzed documents are fed into a function which builds a new Segment representing this information. + +4. We now have everything we need to update the state of the system to include this new snapshot. + - Acquire a lock + - Create a new IndexSnapshot + - For each SegmentSnapshot in the IndexSnapshot, take the deleted PostingsList and OR it with the new postings list for this Segment. Construct a new SegmentSnapshot for the segment using this new deleted PostingsList. Append this SegmentSnapshot to the IndexSnapshot. + - Create a new SegmentSnapshot wrapping our new segment with nil deleted docs. + - Append the new SegmentSnapshot to the IndexSnapshot + - Release the lock + +An ASCII art example: + +```text + 0 - Empty Index + + No segments + + IndexSnapshot + segments [] + deleted [] + + + 1 - Index Batch [ A B C ] + + segment 0 + numbers [ 1 2 3 ] + \_id [ A B C ] + + IndexSnapshot + segments [ 0 ] + deleted [ nil ] + + + 2 - Index Batch [ B' ] + + segment 0 1 + numbers [ 1 2 3 ] [ 1 ] + \_id [ A B C ] [ B ] + + Compute bitset segment-0-deleted-by-1: + [ 0 1 0 ] + + OR it with previous (nil) (call it 0-1) + [ 0 1 0 ] + + IndexSnapshot + segments [ 0 1 ] + deleted [ 0-1 nil ] + + 3 - Index Batch [ C' ] + + segment 0 1 2 + numbers [ 1 2 3 ] [ 1 ] [ 1 ] + \_id [ A B C ] [ B ] [ C ] + + Compute bitset segment-0-deleted-by-2: + [ 0 0 1 ] + + OR it with previous ([ 0 1 0 ]) (call it 0-12) + [ 0 1 1 ] + + Compute bitset segment-1-deleted-by-2: + [ 0 ] + + OR it with previous (nil) + still just nil + + + IndexSnapshot + segments [ 0 1 2 ] + deleted [ 0-12 nil nil ] + ``` + +**is there opportunity to stop early when doc is found in one segment** +**also, more efficient way to find bits for long lists of ids?** + +### Searching + +In the bleve.index API all searching starts by getting an IndexReader, which represents a snapshot of the index at a point in time. + +As described in the section above, our index implementation maintains a pointer to the current IndexSnapshot. When a caller gets an IndexReader, they get a copy of this pointer, and can use it as long as they like. The IndexSnapshot contains SegmentSnapshots, which only contain pointers to immutable segments. The deleted posting lists associated with a segment change over time, but the particular deleted posting list in YOUR snapshot is immutable. This gives a stable view of the data. + +#### Term Search + +Term search is the only searching primitive exposed in today's bleve.index API. This ultimately could limit our ability to take advantage of the indexing improvements, but it also means it will be easier to get a first version of this working. + +A term search for term T in field F will look something like this: + +```go + searchResultPostings = empty + foreach segment { + dict := segment.Dictionary(F) + segmentResultPostings = dict.PostingsList(T, segmentSnapshotDeleted) + // make segmentLocal numbers into global numbers, and flip bits in searchResultPostings + } +``` + +The searchResultPostings will be a new implementation of the TermFieldReader interface. + +As a reminder this interface is: + +```go +// TermFieldReader is the interface exposing the enumeration of documents +// containing a given term in a given field. Documents are returned in byte +// lexicographic order over their identifiers. +type TermFieldReader interface { + // Next returns the next document containing the term in this field, or nil + // when it reaches the end of the enumeration. The preAlloced TermFieldDoc + // is optional, and when non-nil, will be used instead of allocating memory. + Next(preAlloced *TermFieldDoc) (*TermFieldDoc, error) + + // Advance resets the enumeration at specified document or its immediate + // follower. + Advance(ID IndexInternalID, preAlloced *TermFieldDoc) (*TermFieldDoc, error) + + // Count returns the number of documents contains the term in this field. + Count() uint64 + Close() error +} +``` + +At first glance this appears problematic, we have no way to return documents in order of their identifiers. But it turns out the wording of this perhaps too strong, or a bit ambiguous. Originally, this referred to the external identifiers, but with the introduction of a distinction between internal/external identifiers, returning them in order of their internal identifiers is also acceptable. **ASIDE**: the reason for this is that most callers just use Next() and literally don't care what the order is, they could be in any order and it would be fine. There is only one search that cares and that is the ConjunctionSearcher, which relies on Next/Advance having very specific semantics. Later in this document we will have a proposal to split into multiple interfaces: + +- The weakest interface, only supports Next() no ordering at all. +- Ordered, supporting Advance() +- And/Or'able capable of internally efficiently doing these ops with like interfaces (if not capable then can always fall back to external walking) + +But, the good news is that we don't even have to do that for our first implementation. As long as the global numbers we use for internal identifiers are consistent within this IndexSnapshot, then Next() will be ordered by ascending document number, and Advance() will still work correctly. + +NOTE: there is another place where we rely on the ordering of these hits, and that is in the "\_id" sort order. Previously this was the natural order, and a NOOP for the collector, now it must be implemented by actually sorting on the "\_id" field. We probably should introduce at least a marker interface to detect this. + +An ASCII art example: + +```text +Let's start with the IndexSnapshot we ended with earlier: + +3 - Index Batch [ C' ] + + segment 0 1 2 + numbers [ 1 2 3 ] [ 1 ] [ 1 ] + \_id [ A B C ] [ B ] [ C ] + + Compute bitset segment-0-deleted-by-2: + [ 0 0 1 ] + + OR it with previous ([ 0 1 0 ]) (call it 0-12) + [ 0 1 1 ] + +Compute bitset segment-1-deleted-by-2: + [ 0 0 0 ] + +OR it with previous (nil) + still just nil + + + IndexSnapshot + segments [ 0 1 2 ] + deleted [ 0-12 nil nil ] + +Now let's search for the term 'cat' in the field 'desc' and let's assume that Document C (both versions) would match it. + +Concurrently: + + - Segment 0 + - Get Term Dictionary For Field 'desc' + - From it get Postings List for term 'cat' EXCLUDING 0-12 + - raw segment matches [ 0 0 1 ] but excluding [ 0 1 1 ] gives [ 0 0 0 ] + - Segment 1 + - Get Term Dictionary For Field 'desc' + - From it get Postings List for term 'cat' excluding nil + - [ 0 ] + - Segment 2 + - Get Term Dictionary For Field 'desc' + - From it get Postings List for term 'cat' excluding nil + - [ 1 ] + +Map local bitsets into global number space (global meaning cross-segment but still unique to this snapshot) + +IndexSnapshot already should have mapping something like: +0 - Offset 0 +1 - Offset 3 (because segment 0 had 3 docs) +2 - Offset 4 (because segment 1 had 1 doc) + +This maps to search result bitset: + +[ 0 0 0 0 1] + +Caller would call Next() and get doc number 5 (assuming 1 based indexing for now) + +Caller could then ask to get term locations, stored fields, external doc ID for document number 5. Internally in the IndexSnapshot, we can now convert that back, and realize doc number 5 comes from segment 2, 5-4=1 so we're looking for doc number 1 in segment 2. That happens to be C... + +``` + +#### Future improvements + +In the future, interfaces to detect these non-serially operating TermFieldReaders could expose their own And() and Or() up to the higher level Conjunction/Disjunction searchers. Doing this alone offers some win, but also means there would be greater burden on the Searcher code rewriting logical expressions for maximum performance. + +Another related topic is that of peak memory usage. With serially operating TermFieldReaders it was necessary to start them all at the same time and operate in unison. However, with these non-serially operating TermFieldReaders we have the option of doing a few at a time, consolidating them, dispoting the intermediaries, and then doing a few more. For very complex queries with many clauses this could reduce peak memory usage. + +### Memory Tracking + +All segments must be able to produce two statistics, an estimate of their explicit memory usage, and their actual size on disk (if any). For in-memory segments, disk usage could be zero, and the memory usage represents the entire information content. For mmap-based disk segments, the memory could be as low as the size of tracking structure itself (say just a few pointers). + +This would allow the implementation to throttle or block incoming mutations when a threshold memory usage has (or would be) exceeded. + +### Persistence + +Obviously, we want to support (but maybe not require) asynchronous persistence of segments. My expectation is that segments are initially built in memory. At some point they are persisted to disk. This poses some interesting challenges. + +At runtime, the state of an index (it's IndexSnapshot) is not only the contents of the segments, but also the bitmasks of deleted documents. These bitmasks indirectly encode an ordering in which the segments were added. The reason is that the bitmasks encode which items have been obsoleted by other (subsequent or more future) segments. In the runtime implementation we compute bitmask deltas and then merge them at the same time we bring the new segment in. One idea is that we could take a similar approach on disk. When we persist a segment, we persist the bitmask deltas of segments known to exist at that time, and eventually these can get merged up into a base segment deleted bitmask. + +This also relates to the topic rollback, addressed next... + +### Rollback + +One desirable property in the Couchbase ecosystem is the ability to rollback to some previous (though typically not long ago) state. One idea for keeping this property in this design is to protect some of the most recent segments from merging. Then, if necessary, they could be "undone" to reveal previous states of the system. In these scenarios "undone" has to properly undo the deleted bitmasks on the other segments. Again, the current thinking is that rather than "undo" anything, it could be work that was deferred in the first place, thus making it easier to logically undo. + +Another possibly related approach would be to tie this into our existing snapshot mechanism. Perhaps simulating a slow reader (holding onto index snapshots) for some period of time, can be the mechanism to achieve the desired end goal. + +### Internal Storage + +The bleve.index API has support for "internal storage". The ability to store information under a separate name space. + +This is not used for high volume storage, so it is tempting to think we could just put a small k/v store alongside the rest of the index. But, the reality is that this storage is used to maintain key information related to the rollback scenario. Because of this, its crucial that ordering and overwriting of key/value pairs correspond with actual segment persistence in the index. Based on this, I believe its important to put the internal key/value pairs inside the segments themselves. But, this also means that they must follow a similar "deleted" bitmask approach to obsolete values in older segments. But, this also seems to substantially increase the complexity of the solution because of the separate name space, it would appear to require its own bitmask. Further keys aren't numeric, which then implies yet another mapping from internal key to number, etc. + +More thought is required here. + +### Merging + +The segmented index approach requires merging to prevent the number of segments from growing too large. + +Recent experience with LSMs has taught us that having the correct merge strategy can make a huge difference in the overall performance of the system. In particular, a simple merge strategy which merges segments too aggressively can lead to high write amplification and unnecessarily rendering cached data useless. + +A few simple principles have been identified. + +- Roughly we merge multiple smaller segments into a single larger one. +- The larger a segment gets the less likely we should be to ever merge it. +- Segments with large numbers of deleted/obsoleted items are good candidates as the merge will result in a space savings. +- Segments with all items deleted/obsoleted can be dropped. + +Merging of a segment should be able to proceed even if that segment is held by an ongoing snapshot, it should only delay the removal of it. diff --git a/index/scorch/builder.go b/index/scorch/builder.go new file mode 100644 index 0000000..92e3f68 --- /dev/null +++ b/index/scorch/builder.go @@ -0,0 +1,332 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + "os" + "sync" + + "github.com/RoaringBitmap/roaring/v2" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +const DefaultBuilderBatchSize = 1000 +const DefaultBuilderMergeMax = 10 + +type Builder struct { + m sync.Mutex + segCount uint64 + path string + buildPath string + segPaths []string + batchSize int + mergeMax int + batch *index.Batch + internal map[string][]byte + segPlugin SegmentPlugin +} + +func NewBuilder(config map[string]interface{}) (*Builder, error) { + path, ok := config["path"].(string) + if !ok { + return nil, fmt.Errorf("must specify path") + } + + buildPathPrefix, _ := config["buildPathPrefix"].(string) + buildPath, err := os.MkdirTemp(buildPathPrefix, "scorch-offline-build") + if err != nil { + return nil, err + } + + rv := &Builder{ + path: path, + buildPath: buildPath, + mergeMax: DefaultBuilderMergeMax, + batchSize: DefaultBuilderBatchSize, + batch: index.NewBatch(), + segPlugin: defaultSegmentPlugin, + } + + err = rv.parseConfig(config) + if err != nil { + return nil, fmt.Errorf("error parsing builder config: %v", err) + } + + return rv, nil +} + +func (o *Builder) parseConfig(config map[string]interface{}) (err error) { + if v, ok := config["mergeMax"]; ok { + var t int + if t, err = parseToInteger(v); err != nil { + return fmt.Errorf("mergeMax parse err: %v", err) + } + if t > 0 { + o.mergeMax = t + } + } + + if v, ok := config["batchSize"]; ok { + var t int + if t, err = parseToInteger(v); err != nil { + return fmt.Errorf("batchSize parse err: %v", err) + } + if t > 0 { + o.batchSize = t + } + } + + if v, ok := config["internal"]; ok { + if vinternal, ok := v.(map[string][]byte); ok { + o.internal = vinternal + } + } + + forcedSegmentType, forcedSegmentVersion, err := configForceSegmentTypeVersion(config) + if err != nil { + return err + } + if forcedSegmentType != "" && forcedSegmentVersion != 0 { + segPlugin, err := chooseSegmentPlugin(forcedSegmentType, + uint32(forcedSegmentVersion)) + if err != nil { + return err + } + o.segPlugin = segPlugin + } + + return nil +} + +// Index will place the document into the index. +// It is invalid to index the same document multiple times. +func (o *Builder) Index(doc index.Document) error { + o.m.Lock() + defer o.m.Unlock() + + o.batch.Update(doc) + + return o.maybeFlushBatchLOCKED(o.batchSize) +} + +func (o *Builder) maybeFlushBatchLOCKED(moreThan int) error { + if len(o.batch.IndexOps) >= moreThan { + defer o.batch.Reset() + return o.executeBatchLOCKED(o.batch) + } + return nil +} + +func (o *Builder) executeBatchLOCKED(batch *index.Batch) (err error) { + analysisResults := make([]index.Document, 0, len(batch.IndexOps)) + for _, doc := range batch.IndexOps { + if doc != nil { + // insert _id field + doc.AddIDField() + // perform analysis directly + analyze(doc, nil) + analysisResults = append(analysisResults, doc) + } + } + + seg, _, err := o.segPlugin.New(analysisResults) + if err != nil { + return fmt.Errorf("error building segment base: %v", err) + } + + filename := zapFileName(o.segCount) + o.segCount++ + path := o.buildPath + string(os.PathSeparator) + filename + + if segUnpersisted, ok := seg.(segment.UnpersistedSegment); ok { + err = segUnpersisted.Persist(path) + if err != nil { + return fmt.Errorf("error persisting segment base to %s: %v", path, err) + } + + o.segPaths = append(o.segPaths, path) + return nil + } + + return fmt.Errorf("new segment does not implement unpersisted: %T", seg) +} + +func (o *Builder) doMerge() error { + // as long as we have more than 1 segment, keep merging + for len(o.segPaths) > 1 { + + // merge the next number of segments into one new one + // or, if there are fewer than remaining, merge them all + mergeCount := o.mergeMax + if mergeCount > len(o.segPaths) { + mergeCount = len(o.segPaths) + } + + mergePaths := o.segPaths[0:mergeCount] + o.segPaths = o.segPaths[mergeCount:] + + // open each of the segments to be merged + mergeSegs := make([]segment.Segment, 0, mergeCount) + + // closeOpenedSegs attempts to close all opened + // segments even if an error occurs, in which case + // the first error is returned + closeOpenedSegs := func() error { + var err error + for _, seg := range mergeSegs { + clErr := seg.Close() + if clErr != nil && err == nil { + err = clErr + } + } + return err + } + + for _, mergePath := range mergePaths { + seg, err := o.segPlugin.Open(mergePath) + if err != nil { + _ = closeOpenedSegs() + return fmt.Errorf("error opening segment (%s) for merge: %v", mergePath, err) + } + mergeSegs = append(mergeSegs, seg) + } + + // do the merge + mergedSegPath := o.buildPath + string(os.PathSeparator) + zapFileName(o.segCount) + drops := make([]*roaring.Bitmap, mergeCount) + _, _, err := o.segPlugin.Merge(mergeSegs, drops, mergedSegPath, nil, nil) + if err != nil { + _ = closeOpenedSegs() + return fmt.Errorf("error merging segments (%v): %v", mergePaths, err) + } + o.segCount++ + o.segPaths = append(o.segPaths, mergedSegPath) + + // close segments opened for merge + err = closeOpenedSegs() + if err != nil { + return fmt.Errorf("error closing opened segments: %v", err) + } + + // remove merged segments + for _, mergePath := range mergePaths { + err = os.RemoveAll(mergePath) + if err != nil { + return fmt.Errorf("error removing segment %s after merge: %v", mergePath, err) + } + } + } + + return nil +} + +func (o *Builder) Close() error { + o.m.Lock() + defer o.m.Unlock() + + // see if there is a partial batch + err := o.maybeFlushBatchLOCKED(1) + if err != nil { + return fmt.Errorf("error flushing batch before close: %v", err) + } + + // perform all the merging + err = o.doMerge() + if err != nil { + return fmt.Errorf("error while merging: %v", err) + } + + // ensure the store path exists + err = os.MkdirAll(o.path, 0700) + if err != nil { + return err + } + + // move final segment into place + // segment id 2 is chosen to match the behavior of a scorch + // index which indexes a single batch of data + finalSegPath := o.path + string(os.PathSeparator) + zapFileName(2) + err = os.Rename(o.segPaths[0], finalSegPath) + if err != nil { + return fmt.Errorf("error moving final segment into place: %v", err) + } + + // remove the buildPath, as it is no longer needed + err = os.RemoveAll(o.buildPath) + if err != nil { + return fmt.Errorf("error removing build path: %v", err) + } + + // prepare wrapping + seg, err := o.segPlugin.Open(finalSegPath) + if err != nil { + return fmt.Errorf("error opening final segment") + } + + // create a segment snapshot for this segment + ss := &SegmentSnapshot{ + segment: seg, + } + is := &IndexSnapshot{ + epoch: 3, // chosen to match scorch behavior when indexing a single batch + segment: []*SegmentSnapshot{ss}, + creator: "scorch-builder", + internal: o.internal, + } + + // create the root bolt + rootBoltPath := o.path + string(os.PathSeparator) + "root.bolt" + rootBolt, err := util.OpenBolt(rootBoltPath, 0600, nil) + if err != nil { + return err + } + + // start a write transaction + tx, err := rootBolt.Begin(true) + if err != nil { + return err + } + + // fill the root bolt with this fake index snapshot + _, _, err = prepareBoltSnapshot(is, tx, o.path, o.segPlugin, nil) + if err != nil { + _ = tx.Rollback() + _ = rootBolt.Close() + return fmt.Errorf("error preparing bolt snapshot in root.bolt: %v", err) + } + + // commit bolt data + err = tx.Commit() + if err != nil { + _ = rootBolt.Close() + return fmt.Errorf("error committing bolt tx in root.bolt: %v", err) + } + + // close bolt + err = rootBolt.Close() + if err != nil { + return fmt.Errorf("error closing root.bolt: %v", err) + } + + // close final segment + err = seg.Close() + if err != nil { + return fmt.Errorf("error closing final segment: %v", err) + } + return nil +} diff --git a/index/scorch/builder_test.go b/index/scorch/builder_test.go new file mode 100644 index 0000000..3fac77c --- /dev/null +++ b/index/scorch/builder_test.go @@ -0,0 +1,159 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/blevesearch/bleve/v2/document" + index "github.com/blevesearch/bleve_index_api" +) + +func TestBuilder(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "scorch-builder-test") + if err != nil { + t.Fatal(err) + } + defer func() { + err = os.RemoveAll(tmpDir) + if err != nil { + t.Fatalf("error cleaning up test index: %v", err) + } + }() + options := map[string]interface{}{ + "path": tmpDir, + "batchSize": 2, + "mergeMax": 2, + } + b, err := NewBuilder(options) + if err != nil { + t.Fatal(err) + } + + for i := 0; i < 10; i++ { + doc := document.NewDocument(fmt.Sprintf("%d", i)) + doc.AddField(document.NewTextField("name", nil, []byte("hello"))) + err = b.Index(doc) + if err != nil { + t.Fatal(err) + } + } + + err = b.Close() + if err != nil { + t.Fatal(err) + } + + checkIndex(t, tmpDir, []byte("hello"), "name", 10) +} + +func checkIndex(t *testing.T, path string, term []byte, field string, expectCount int) { + cfg := make(map[string]interface{}) + cfg["path"] = path + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err = idx.Close() + if err != nil { + t.Fatalf("error closing index: %v", err) + } + }() + + r, err := idx.Reader() + if err != nil { + t.Fatalf("error accessing index reader: %v", err) + } + defer func() { + err = r.Close() + if err != nil { + t.Fatalf("error closing reader: %v", err) + } + }() + + // check the count, expect 10 docs + count, err := r.DocCount() + if err != nil { + t.Errorf("error accessing index doc count: %v", err) + } else if count != uint64(expectCount) { + t.Errorf("expected %d docs, got %d", expectCount, count) + } + + // run a search for hello + tfr, err := r.TermFieldReader(context.TODO(), term, field, false, false, false) + if err != nil { + t.Errorf("error accessing term field reader: %v", err) + } else { + var rows int + tfd, err := tfr.Next(nil) + for err == nil && tfd != nil { + rows++ + tfd, err = tfr.Next(nil) + } + if err != nil { + t.Errorf("error calling next on term field reader: %v", err) + } + if rows != expectCount { + t.Errorf("expected %d rows for term hello, field name, got %d", expectCount, rows) + } + } +} + +func TestBuilderFlushFinalBatch(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "scorch-builder-test") + if err != nil { + t.Fatal(err) + } + defer func() { + err = os.RemoveAll(tmpDir) + if err != nil { + t.Fatalf("error cleaning up test index: %v", err) + } + }() + options := map[string]interface{}{ + "path": tmpDir, + "batchSize": 2, + "mergeMax": 2, + } + b, err := NewBuilder(options) + if err != nil { + t.Fatal(err) + } + + for i := 0; i < 9; i++ { + doc := document.NewDocument(fmt.Sprintf("%d", i)) + doc.AddField(document.NewTextField("name", nil, []byte("hello"))) + err = b.Index(doc) + if err != nil { + t.Fatal(err) + } + } + + err = b.Close() + if err != nil { + t.Fatal(err) + } + + checkIndex(t, tmpDir, []byte("hello"), "name", 9) +} diff --git a/index/scorch/empty.go b/index/scorch/empty.go new file mode 100644 index 0000000..eabe29f --- /dev/null +++ b/index/scorch/empty.go @@ -0,0 +1,50 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "github.com/RoaringBitmap/roaring/v2" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +type emptyPostingsIterator struct{} + +func (e *emptyPostingsIterator) Next() (segment.Posting, error) { + return nil, nil +} + +func (e *emptyPostingsIterator) Advance(uint64) (segment.Posting, error) { + return nil, nil +} + +func (e *emptyPostingsIterator) Size() int { + return 0 +} + +func (e *emptyPostingsIterator) BytesRead() uint64 { + return 0 +} + +func (e *emptyPostingsIterator) ResetBytesRead(uint64) {} + +func (e *emptyPostingsIterator) BytesWritten() uint64 { return 0 } + +func (e *emptyPostingsIterator) ActualBitmap() *roaring.Bitmap { return nil } + +func (e *emptyPostingsIterator) DocNum1Hit() (uint64, bool) { return 0, false } + +func (e *emptyPostingsIterator) ReplaceActual(*roaring.Bitmap) {} + +var anEmptyPostingsIterator = &emptyPostingsIterator{} diff --git a/index/scorch/event.go b/index/scorch/event.go new file mode 100644 index 0000000..0acdf10 --- /dev/null +++ b/index/scorch/event.go @@ -0,0 +1,79 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import "time" + +// RegistryAsyncErrorCallbacks should be treated as read-only after +// process init()'ialization. +var RegistryAsyncErrorCallbacks = map[string]func(error, string){} + +// RegistryEventCallbacks should be treated as read-only after +// process init()'ialization. +// In the event of not having a callback, these return true. +var RegistryEventCallbacks = map[string]func(Event) bool{} + +// Event represents the information provided in an OnEvent() callback. +type Event struct { + Kind EventKind + Scorch *Scorch + Duration time.Duration +} + +// EventKind represents an event code for OnEvent() callbacks. +type EventKind int + +const ( + // EventKindCloseStart is fired when a Scorch.Close() has begun. + EventKindCloseStart EventKind = iota + + // EventKindClose is fired when a scorch index has been fully closed. + EventKindClose + + // EventKindMergerProgress is fired when the merger has completed a + // round of merge processing. + EventKindMergerProgress + + // EventKindPersisterProgress is fired when the persister has completed + // a round of persistence processing. + EventKindPersisterProgress + + // EventKindBatchIntroductionStart is fired when Batch() is invoked which + // introduces a new segment. + EventKindBatchIntroductionStart + + // EventKindBatchIntroduction is fired when Batch() completes. + EventKindBatchIntroduction + + // EventKindMergeTaskIntroductionStart is fired when the merger is about to + // start the introduction of merged segment from a single merge task. + EventKindMergeTaskIntroductionStart + + // EventKindMergeTaskIntroduction is fired when the merger has completed + // the introduction of merged segment from a single merge task. + EventKindMergeTaskIntroduction + + // EventKindPreMergeCheck is fired before the merge begins to check if + // the caller should proceed with the merge. + EventKindPreMergeCheck + + // EventKindIndexStart is fired when Index() is invoked which + // creates a new Document object from an interface using the index mapping. + EventKindIndexStart + + // EventKindPurgerCheck is fired before the purge code is invoked and decides + // whether to execute or not. For unit test purposes + EventKindPurgerCheck +) diff --git a/index/scorch/event_test.go b/index/scorch/event_test.go new file mode 100644 index 0000000..be163a9 --- /dev/null +++ b/index/scorch/event_test.go @@ -0,0 +1,79 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "testing" + + "github.com/blevesearch/bleve/v2/document" + index "github.com/blevesearch/bleve_index_api" +) + +func TestEventBatchIntroductionStart(t *testing.T) { + testConfig := CreateConfig("TestEventBatchIntroductionStart") + err := InitTest(testConfig) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(testConfig) + if err != nil { + t.Fatal(err) + } + }() + + var count int + RegistryEventCallbacks["test"] = func(e Event) bool { + if e.Kind == EventKindBatchIntroductionStart { + count++ + } + return true + } + + ourConfig := make(map[string]interface{}, len(testConfig)) + for k, v := range testConfig { + ourConfig[k] = v + } + ourConfig["eventCallbackName"] = "test" + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, ourConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + if count != 1 { + t.Fatalf("expected to see 1 batch introduction event event, saw %d", count) + } +} diff --git a/index/scorch/field_dict_test.go b/index/scorch/field_dict_test.go new file mode 100644 index 0000000..44afe26 --- /dev/null +++ b/index/scorch/field_dict_test.go @@ -0,0 +1,186 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/document" + index "github.com/blevesearch/bleve_index_api" +) + +func TestIndexFieldDict(t *testing.T) { + cfg := CreateConfig("TestIndexFieldDict") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + cerr := idx.Close() + if cerr != nil { + t.Fatal(cerr) + } + }() + + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + doc = document.NewDocument("2") + doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test test test"), testAnalyzer)) + doc.AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("eat more rice"), index.IndexField|index.IncludeTermVectors, testAnalyzer)) + doc.AddField(document.NewTextFieldCustom("prefix", []uint64{}, []byte("bob cat cats catting dog doggy zoo"), index.IndexField|index.IncludeTermVectors, testAnalyzer)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + dict, err := indexReader.FieldDict("name") + if err != nil { + t.Errorf("error creating reader: %v", err) + } + defer func() { + err := dict.Close() + if err != nil { + t.Fatal(err) + } + }() + + termCount := 0 + curr, err := dict.Next() + for err == nil && curr != nil { + termCount++ + if curr.Term != "test" { + t.Errorf("expected term to be 'test', got '%s'", curr.Term) + } + curr, err = dict.Next() + } + if termCount != 1 { + t.Errorf("expected 1 term for this field, got %d", termCount) + } + + dict2, err := indexReader.FieldDict("desc") + if err != nil { + t.Fatalf("error creating reader: %v", err) + } + defer func() { + err := dict2.Close() + if err != nil { + t.Fatal(err) + } + }() + + termCount = 0 + terms := make([]string, 0) + curr, err = dict2.Next() + for err == nil && curr != nil { + termCount++ + terms = append(terms, curr.Term) + curr, err = dict2.Next() + } + if termCount != 3 { + t.Errorf("expected 3 term for this field, got %d", termCount) + } + expectedTerms := []string{"eat", "more", "rice"} + if !reflect.DeepEqual(expectedTerms, terms) { + t.Errorf("expected %#v, got %#v", expectedTerms, terms) + } + // test start and end range + dict3, err := indexReader.FieldDictRange("desc", []byte("fun"), []byte("nice")) + if err != nil { + t.Errorf("error creating reader: %v", err) + } + defer func() { + err := dict3.Close() + if err != nil { + t.Fatal(err) + } + }() + + termCount = 0 + terms = make([]string, 0) + curr, err = dict3.Next() + for err == nil && curr != nil { + termCount++ + terms = append(terms, curr.Term) + curr, err = dict3.Next() + } + if termCount != 1 { + t.Errorf("expected 1 term for this field, got %d", termCount) + } + expectedTerms = []string{"more"} + if !reflect.DeepEqual(expectedTerms, terms) { + t.Errorf("expected %#v, got %#v", expectedTerms, terms) + } + + // test use case for prefix + dict4, err := indexReader.FieldDictPrefix("prefix", []byte("cat")) + if err != nil { + t.Errorf("error creating reader: %v", err) + } + defer func() { + err := dict4.Close() + if err != nil { + t.Fatal(err) + } + }() + + termCount = 0 + terms = make([]string, 0) + curr, err = dict4.Next() + for err == nil && curr != nil { + termCount++ + terms = append(terms, curr.Term) + curr, err = dict4.Next() + } + if termCount != 3 { + t.Errorf("expected 3 term for this field, got %d", termCount) + } + expectedTerms = []string{"cat", "cats", "catting"} + if !reflect.DeepEqual(expectedTerms, terms) { + t.Errorf("expected %#v, got %#v", expectedTerms, terms) + } +} diff --git a/index/scorch/fuzzy_omitcount_test.go b/index/scorch/fuzzy_omitcount_test.go new file mode 100644 index 0000000..8a2dfdf --- /dev/null +++ b/index/scorch/fuzzy_omitcount_test.go @@ -0,0 +1,98 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package scorch + +import ( + "fmt" + "testing" + + "github.com/blevesearch/bleve/v2/document" + index "github.com/blevesearch/bleve_index_api" +) + +// TestFieldDictFuzzyAutomatonOmitsCount verifies that the fuzzy automaton field +// dictionary is wired to the count-omitting iterator: the segment dictionary +// implements the optional interface, and the collected candidate entries have +// their (discarded) Count left at zero rather than incurring a postings read. +func TestFieldDictFuzzyAutomatonOmitsCount(t *testing.T) { + cfg := CreateConfig("TestFieldDictFuzzyAutomatonOmitsCount") + if err := InitTest(cfg); err != nil { + t.Fatal(err) + } + defer func() { _ = DestroyTest(cfg) }() + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + if err = idx.Open(); err != nil { + t.Fatal(err) + } + defer func() { _ = idx.Close() }() + + // Each doc's "desc" field is a single token (nil analyzer => whole value is + // one term). Repeating terms across docs yields multi-doc postings lists. + b := index.NewBatch() + for d := 0; d < 300; d++ { + doc := document.NewDocument(fmt.Sprintf("%d", d)) + doc.AddField(document.NewTextField("desc", nil, []byte(fmt.Sprintf("term%04d", d%100)))) + b.Update(doc) + } + if err = idx.Batch(b); err != nil { + t.Fatal(err) + } + + r, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + defer func() { _ = r.Close() }() + is := r.(*IndexSnapshot) + + // the segment dictionary must implement the count-omit optional interface, + // otherwise the wiring silently falls back to the count-reading path. + seg := is.segment[0].segment + dict, err := seg.Dictionary("desc") + if err != nil { + t.Fatal(err) + } + if _, ok := dict.(termDictionaryOmitCount); !ok { + // The omit-count wiring only activates once the zapx dependency ships + // AutomatonIteratorOmitCount; until then the code falls back to the + // count-reading path, so there is nothing to assert. + t.Skipf("segment dict %T does not implement termDictionaryOmitCount "+ + "(zapx dependency predates the omit-count iterator)", dict) + } + + fd, _, err := is.FieldDictFuzzyAutomaton("desc", "term0050", 2, "") + if err != nil { + t.Fatal(err) + } + n := 0 + tfd, err := fd.Next() + for err == nil && tfd != nil { + n++ + if tfd.Count != 0 { + t.Fatalf("expected omitted (zero) count for %q, got %d", tfd.Term, tfd.Count) + } + tfd, err = fd.Next() + } + if err != nil { + t.Fatal(err) + } + if n == 0 { + t.Fatal("expected at least one fuzzy candidate term") + } + t.Logf("collected %d candidate terms, all with omitted count", n) +} diff --git a/index/scorch/int.go b/index/scorch/int.go new file mode 100644 index 0000000..4fa6d7f --- /dev/null +++ b/index/scorch/int.go @@ -0,0 +1,92 @@ +// Copyright 2014 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +// This code originated from: +// https://github.com/cockroachdb/cockroach/blob/2dd65dde5d90c157f4b93f92502ca1063b904e1d/pkg/util/encoding/encoding.go + +// Modified to not use pkg/errors + +package scorch + +import "fmt" + +const ( + // intMin is chosen such that the range of int tags does not overlap the + // ascii character set that is frequently used in testing. + intMin = 0x80 // 128 + intMaxWidth = 8 + intZero = intMin + intMaxWidth // 136 + intSmall = intMax - intZero - intMaxWidth // 109 + // intMax is the maximum int tag value. + intMax = 0xfd // 253 +) + +// encodeUvarintAscending encodes the uint64 value using a variable length +// (length-prefixed) representation. The length is encoded as a single +// byte indicating the number of encoded bytes (-8) to follow. See +// EncodeVarintAscending for rationale. The encoded bytes are appended to the +// supplied buffer and the final buffer is returned. +func encodeUvarintAscending(b []byte, v uint64) []byte { + switch { + case v <= intSmall: + return append(b, intZero+byte(v)) + case v <= 0xff: + return append(b, intMax-7, byte(v)) + case v <= 0xffff: + return append(b, intMax-6, byte(v>>8), byte(v)) + case v <= 0xffffff: + return append(b, intMax-5, byte(v>>16), byte(v>>8), byte(v)) + case v <= 0xffffffff: + return append(b, intMax-4, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) + case v <= 0xffffffffff: + return append(b, intMax-3, byte(v>>32), byte(v>>24), byte(v>>16), byte(v>>8), + byte(v)) + case v <= 0xffffffffffff: + return append(b, intMax-2, byte(v>>40), byte(v>>32), byte(v>>24), byte(v>>16), + byte(v>>8), byte(v)) + case v <= 0xffffffffffffff: + return append(b, intMax-1, byte(v>>48), byte(v>>40), byte(v>>32), byte(v>>24), + byte(v>>16), byte(v>>8), byte(v)) + default: + return append(b, intMax, byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32), + byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) + } +} + +// decodeUvarintAscending decodes a varint encoded uint64 from the input +// buffer. The remainder of the input buffer and the decoded uint64 +// are returned. +func decodeUvarintAscending(b []byte) ([]byte, uint64, error) { + if len(b) == 0 { + return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value") + } + length := int(b[0]) - intZero + b = b[1:] // skip length byte + if length <= intSmall { + return b, uint64(length), nil + } + length -= intSmall + if length < 0 || length > 8 { + return nil, 0, fmt.Errorf("invalid uvarint length of %d", length) + } else if len(b) < length { + return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value: %q", b) + } + var v uint64 + // It is faster to range over the elements in a slice than to index + // into the slice on each loop iteration. + for _, t := range b[:length] { + v = (v << 8) | uint64(t) + } + return b[length:], v, nil +} diff --git a/index/scorch/int_test.go b/index/scorch/int_test.go new file mode 100644 index 0000000..202b157 --- /dev/null +++ b/index/scorch/int_test.go @@ -0,0 +1,96 @@ +// Copyright 2014 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +// This code originated from: +// https://github.com/cockroachdb/cockroach/blob/2dd65dde5d90c157f4b93f92502ca1063b904e1d/pkg/util/encoding/encoding_test.go + +// Modified to only test the parts we borrowed + +package scorch + +import ( + "bytes" + "math" + "testing" +) + +type testCaseUint64 struct { + value uint64 + expEnc []byte +} + +func TestEncodeDecodeUvarint(t *testing.T) { + testBasicEncodeDecodeUint64(encodeUvarintAscending, decodeUvarintAscending, false, t) + testCases := []testCaseUint64{ + {0, []byte{0x88}}, + {1, []byte{0x89}}, + {109, []byte{0xf5}}, + {110, []byte{0xf6, 0x6e}}, + {1 << 8, []byte{0xf7, 0x01, 0x00}}, + {math.MaxUint64, []byte{0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + } + testCustomEncodeUint64(testCases, encodeUvarintAscending, t) +} + +func testBasicEncodeDecodeUint64( + encFunc func([]byte, uint64) []byte, + decFunc func([]byte) ([]byte, uint64, error), + descending bool, t *testing.T, +) { + testCases := []uint64{ + 0, 1, + 1<<8 - 1, 1 << 8, + 1<<16 - 1, 1 << 16, + 1<<24 - 1, 1 << 24, + 1<<32 - 1, 1 << 32, + 1<<40 - 1, 1 << 40, + 1<<48 - 1, 1 << 48, + 1<<56 - 1, 1 << 56, + math.MaxUint64 - 1, math.MaxUint64, + } + + var lastEnc []byte + for i, v := range testCases { + enc := encFunc(nil, v) + if i > 0 { + if (descending && bytes.Compare(enc, lastEnc) >= 0) || + (!descending && bytes.Compare(enc, lastEnc) < 0) { + t.Errorf("ordered constraint violated for %d: [% x] vs. [% x]", v, enc, lastEnc) + } + } + b, decode, err := decFunc(enc) + if err != nil { + t.Error(err) + continue + } + if len(b) != 0 { + t.Errorf("leftover bytes: [% x]", b) + } + if decode != v { + t.Errorf("decode yielded different value than input: %d vs. %d", decode, v) + } + lastEnc = enc + } +} + +func testCustomEncodeUint64( + testCases []testCaseUint64, encFunc func([]byte, uint64) []byte, t *testing.T, +) { + for _, test := range testCases { + enc := encFunc(nil, test.value) + if !bytes.Equal(enc, test.expEnc) { + t.Errorf("expected [% x]; got [% x] (value: %d)", test.expEnc, enc, test.value) + } + } +} diff --git a/index/scorch/introducer.go b/index/scorch/introducer.go new file mode 100644 index 0000000..fd9f7a8 --- /dev/null +++ b/index/scorch/introducer.go @@ -0,0 +1,532 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + "path/filepath" + "sync/atomic" + + "github.com/RoaringBitmap/roaring/v2" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +const introducer = "introducer" + +type segmentIntroduction struct { + id uint64 + data segment.Segment + obsoletes map[uint64]*roaring.Bitmap + ids []string + internal map[string][]byte + + applied chan error + persisted chan error + persistedCallback index.BatchCallback +} + +type persistIntroduction struct { + persisted map[uint64]segment.Segment + applied notificationChan +} + +type epochWatcher struct { + epoch uint64 + notifyCh notificationChan +} + +func (s *Scorch) introducerLoop() { + defer func() { + if r := recover(); r != nil { + s.fireAsyncError(NewScorchError( + introducer, + fmt.Sprintf("panic: %v, path: %s", r, s.path), + ErrAsyncPanic, + )) + } + + s.asyncTasks.Done() + }() + + var epochWatchers []*epochWatcher +OUTER: + for { + atomic.AddUint64(&s.stats.TotIntroduceLoop, 1) + + select { + case <-s.closeCh: + break OUTER + + case epochWatcher := <-s.introducerNotifier: + epochWatchers = append(epochWatchers, epochWatcher) + + case nextMerge := <-s.merges: + s.introduceMerge(nextMerge) + + case next := <-s.introductions: + err := s.introduceSegment(next) + if err != nil { + continue OUTER + } + + case persist := <-s.persists: + s.introducePersist(persist) + + } + + var epochCurr uint64 + s.rootLock.RLock() + if s.root != nil { + epochCurr = s.root.epoch + } + s.rootLock.RUnlock() + var epochWatchersNext []*epochWatcher + for _, w := range epochWatchers { + if w.epoch < epochCurr { + close(w.notifyCh) + } else { + epochWatchersNext = append(epochWatchersNext, w) + } + } + epochWatchers = epochWatchersNext + } +} + +func (s *Scorch) introduceSegment(next *segmentIntroduction) error { + atomic.AddUint64(&s.stats.TotIntroduceSegmentBeg, 1) + defer atomic.AddUint64(&s.stats.TotIntroduceSegmentEnd, 1) + + s.rootLock.RLock() + root := s.root + root.AddRef() + s.rootLock.RUnlock() + + defer func() { _ = root.DecRef() }() + + nsegs := len(root.segment) + + // prepare new index snapshot + newSnapshot := &IndexSnapshot{ + parent: s, + segment: make([]*SegmentSnapshot, 0, nsegs+1), + offsets: make([]uint64, 0, nsegs+1), + internal: make(map[string][]byte, len(root.internal)), + refs: 1, + creator: "introduceSegment", + } + + // iterate through current segments + var running uint64 + var docsToPersistCount, memSegments, fileSegments uint64 + var droppedSegmentFiles []string + for i := range root.segment { + // see if optimistic work included this segment + delta, ok := next.obsoletes[root.segment[i].id] + if !ok { + var err error + delta, err = root.segment[i].segment.DocNumbers(next.ids) + if err != nil { + next.applied <- fmt.Errorf("error computing doc numbers: %v", err) + close(next.applied) + _ = newSnapshot.DecRef() + return err + } + } + + newss := &SegmentSnapshot{ + id: root.segment[i].id, + segment: root.segment[i].segment, + stats: root.segment[i].stats, + cachedDocs: root.segment[i].cachedDocs, + cachedMeta: root.segment[i].cachedMeta, + creator: root.segment[i].creator, + } + + // apply new obsoletions + if root.segment[i].deleted == nil { + newss.deleted = delta + } else { + newss.deleted = roaring.Or(root.segment[i].deleted, delta) + } + if newss.deleted.IsEmpty() { + newss.deleted = nil + } + + // update the deleted bitmap to include any nested/sub-documents as well + // if the segment supports that + if ns, ok := newss.segment.(segment.NestedSegment); ok { + newss.deleted = ns.AddNestedDocuments(newss.deleted) + } + // check for live size before copying + if newss.LiveSize() > 0 { + newSnapshot.segment = append(newSnapshot.segment, newss) + root.segment[i].segment.AddRef() + newSnapshot.offsets = append(newSnapshot.offsets, running) + running += newss.segment.Count() + } else if seg, ok := newss.segment.(segment.PersistedSegment); ok { + droppedSegmentFiles = append(droppedSegmentFiles, + filepath.Base(seg.Path())) + } + + if isMemorySegment(root.segment[i]) { + docsToPersistCount += root.segment[i].Count() + memSegments++ + } else { + fileSegments++ + } + } + + atomic.StoreUint64(&s.stats.TotItemsToPersist, docsToPersistCount) + atomic.StoreUint64(&s.stats.TotMemorySegmentsAtRoot, memSegments) + atomic.StoreUint64(&s.stats.TotFileSegmentsAtRoot, fileSegments) + + // append new segment, if any, to end of the new index snapshot + if next.data != nil { + stats := newFieldStats() + if fsr, ok := next.data.(segment.FieldStatsReporter); ok { + fsr.UpdateFieldStats(stats) + } + newSegmentSnapshot := &SegmentSnapshot{ + id: next.id, + segment: next.data, // take ownership of next.data's ref-count + stats: stats, + cachedDocs: &cachedDocs{cache: nil}, + cachedMeta: newCachedMeta(), + creator: "introduceSegment", + } + newSnapshot.segment = append(newSnapshot.segment, newSegmentSnapshot) + newSnapshot.offsets = append(newSnapshot.offsets, running) + + // increment numItemsIntroduced which tracks the number of items + // queued for persistence. + atomic.AddUint64(&s.stats.TotIntroducedItems, newSegmentSnapshot.Count()) + atomic.AddUint64(&s.stats.TotIntroducedSegmentsBatch, 1) + } + // copy old values + for key, oldVal := range root.internal { + newSnapshot.internal[key] = oldVal + } + // set new values and apply deletes + for key, newVal := range next.internal { + if newVal != nil { + newSnapshot.internal[key] = newVal + } else { + delete(newSnapshot.internal, key) + } + } + + newSnapshot.updateSize() + s.rootLock.Lock() + if next.persisted != nil { + s.rootPersisted = append(s.rootPersisted, next.persisted) + } + if next.persistedCallback != nil { + s.persistedCallbacks = append(s.persistedCallbacks, next.persistedCallback) + } + // swap in new index snapshot + newSnapshot.epoch = s.nextSnapshotEpoch + s.nextSnapshotEpoch++ + rootPrev := s.root + s.root = newSnapshot + atomic.StoreUint64(&s.stats.CurRootEpoch, s.root.epoch) + // release lock + s.rootLock.Unlock() + + if rootPrev != nil { + _ = rootPrev.DecRef() + } + + // update the removal eligibility for those segment files + // that are not a part of the latest root. + for _, filename := range droppedSegmentFiles { + s.unmarkIneligibleForRemoval(filename) + } + + close(next.applied) + + return nil +} + +func (s *Scorch) introducePersist(persist *persistIntroduction) { + atomic.AddUint64(&s.stats.TotIntroducePersistBeg, 1) + defer atomic.AddUint64(&s.stats.TotIntroducePersistEnd, 1) + + s.rootLock.Lock() + root := s.root + root.AddRef() + nextSnapshotEpoch := s.nextSnapshotEpoch + s.nextSnapshotEpoch++ + s.rootLock.Unlock() + + defer func() { _ = root.DecRef() }() + + newIndexSnapshot := &IndexSnapshot{ + parent: s, + epoch: nextSnapshotEpoch, + segment: make([]*SegmentSnapshot, len(root.segment)), + offsets: make([]uint64, len(root.offsets)), + internal: make(map[string][]byte, len(root.internal)), + refs: 1, + creator: "introducePersist", + } + + var docsToPersistCount, memSegments, fileSegments uint64 + for i, segmentSnapshot := range root.segment { + // see if this segment has been replaced + if replacement, ok := persist.persisted[segmentSnapshot.id]; ok { + newSegmentSnapshot := &SegmentSnapshot{ + id: segmentSnapshot.id, + segment: replacement, + deleted: segmentSnapshot.deleted, + stats: segmentSnapshot.stats, + cachedDocs: segmentSnapshot.cachedDocs, + cachedMeta: segmentSnapshot.cachedMeta, + creator: "introducePersist", + mmaped: 1, + } + newIndexSnapshot.segment[i] = newSegmentSnapshot + delete(persist.persisted, segmentSnapshot.id) + + // update items persisted in case of a new segment snapshot + atomic.AddUint64(&s.stats.TotPersistedItems, newSegmentSnapshot.Count()) + atomic.AddUint64(&s.stats.TotPersistedSegments, 1) + fileSegments++ + } else { + newIndexSnapshot.segment[i] = root.segment[i] + newIndexSnapshot.segment[i].segment.AddRef() + + if isMemorySegment(root.segment[i]) { + docsToPersistCount += root.segment[i].Count() + memSegments++ + } else { + fileSegments++ + } + } + newIndexSnapshot.offsets[i] = root.offsets[i] + } + + for k, v := range root.internal { + newIndexSnapshot.internal[k] = v + } + + atomic.StoreUint64(&s.stats.TotItemsToPersist, docsToPersistCount) + atomic.StoreUint64(&s.stats.TotMemorySegmentsAtRoot, memSegments) + atomic.StoreUint64(&s.stats.TotFileSegmentsAtRoot, fileSegments) + newIndexSnapshot.updateSize() + s.rootLock.Lock() + rootPrev := s.root + s.root = newIndexSnapshot + atomic.StoreUint64(&s.stats.CurRootEpoch, s.root.epoch) + s.rootLock.Unlock() + + if rootPrev != nil { + _ = rootPrev.DecRef() + } + + close(persist.applied) +} + +// The introducer should definitely handle the segmentMerge.notify +// channel before exiting the introduceMerge. +func (s *Scorch) introduceMerge(nextMerge *segmentMerge) { + atomic.AddUint64(&s.stats.TotIntroduceMergeBeg, 1) + defer atomic.AddUint64(&s.stats.TotIntroduceMergeEnd, 1) + + s.rootLock.RLock() + root := s.root + root.AddRef() + s.rootLock.RUnlock() + + defer func() { _ = root.DecRef() }() + + newSnapshot := &IndexSnapshot{ + parent: s, + internal: root.internal, + refs: 1, + creator: "introduceMerge", + } + + var running, docsToPersistCount, memSegments, fileSegments uint64 + var droppedSegmentFiles []string + newSegmentsDeleted := make([]*roaring.Bitmap, len(nextMerge.newSegments)) + for i := range newSegmentsDeleted { + // create a bitmaps to track the obsoletes per newly merged segments + newSegmentsDeleted[i] = roaring.NewBitmap() + } + + // iterate through current segments + for i := range root.segment { + segmentID := root.segment[i].id + if history, ok := nextMerge.mergedSegHistory[segmentID]; ok { + // this segment is going away, see if anything else was deleted since we started the merge + if history != nil && root.segment[i].deleted != nil { + // assume all these deletes are new + deletedSince := root.segment[i].deleted + // if we already knew about some of them, remove + if history.oldSegment.deleted != nil { + deletedSince = roaring.AndNot(root.segment[i].deleted, history.oldSegment.deleted) + } + deletedSinceItr := deletedSince.Iterator() + for deletedSinceItr.HasNext() { + oldDocNum := deletedSinceItr.Next() + newDocNum := history.oldNewDocIDs[oldDocNum] + newSegmentsDeleted[history.batchID].Add(uint32(newDocNum)) + } + } + + // clean up the old segment map to figure out the + // obsolete segments wrt root in meantime, whatever + // segments left behind in old map after processing + // the root segments would be the obsolete segment set + delete(nextMerge.mergedSegHistory, segmentID) + } else if root.segment[i].LiveSize() > 0 { + // this segment is staying + newSnapshot.segment = append(newSnapshot.segment, &SegmentSnapshot{ + id: root.segment[i].id, + segment: root.segment[i].segment, + deleted: root.segment[i].deleted, + stats: root.segment[i].stats, + cachedDocs: root.segment[i].cachedDocs, + cachedMeta: root.segment[i].cachedMeta, + creator: root.segment[i].creator, + }) + root.segment[i].segment.AddRef() + newSnapshot.offsets = append(newSnapshot.offsets, running) + running += root.segment[i].segment.Count() + + if isMemorySegment(root.segment[i]) { + docsToPersistCount += root.segment[i].Count() + memSegments++ + } else { + fileSegments++ + } + } else if root.segment[i].LiveSize() == 0 { + if seg, ok := root.segment[i].segment.(segment.PersistedSegment); ok { + droppedSegmentFiles = append(droppedSegmentFiles, + filepath.Base(seg.Path())) + } + } + } + // before the newMerge introduction, need to clean the newly + // merged segment wrt the current root segments, hence + // applying the obsolete segment contents to newly merged segment + for _, ss := range nextMerge.mergedSegHistory { + obsoleted := ss.oldSegment.DocNumbersLive() + if obsoleted != nil { + obsoletedIter := obsoleted.Iterator() + for obsoletedIter.HasNext() { + oldDocNum := obsoletedIter.Next() + newDocNum := ss.oldNewDocIDs[oldDocNum] + newSegmentsDeleted[ss.batchID].Add(uint32(newDocNum)) + } + } + } + + skipped := make([]bool, len(nextMerge.newSegments)) + // make the newly merged segments part of the newSnapshot being constructed + for i, newMergedSegment := range nextMerge.newSegments { + if newMergedSegment == nil { + if nextMerge.fileMerge { + atomic.AddUint64(&s.stats.TotFileMergeIntroductionsSkipped, 1) + } else { + atomic.AddUint64(&s.stats.TotMemMergeIntroductionsSkipped, 1) + } + skipped[i] = true + continue + } + // checking if this newly merged segment is worth keeping based on + // obsoleted doc count since the merge intro started + if newMergedSegment.Count() > newSegmentsDeleted[i].GetCardinality() { + stats := newFieldStats() + if fsr, ok := newMergedSegment.(segment.FieldStatsReporter); ok { + fsr.UpdateFieldStats(stats) + } + + // put the merged segment at the end of newSnapshot + newSnapshot.segment = append(newSnapshot.segment, &SegmentSnapshot{ + id: nextMerge.newSegmentIDs[i], + segment: newMergedSegment, // take ownership for newMergedSegment's ref-count + deleted: newSegmentsDeleted[i], + stats: stats, + cachedDocs: &cachedDocs{cache: nil}, + cachedMeta: newCachedMeta(), + creator: "introduceMerge", + mmaped: nextMerge.mmaped, + }) + newSnapshot.offsets = append(newSnapshot.offsets, running) + running += newMergedSegment.Count() + + switch newMergedSegment.(type) { + case segment.PersistedSegment: + fileSegments++ + default: + docsToPersistCount += newMergedSegment.Count() - newSegmentsDeleted[i].GetCardinality() + memSegments++ + } + atomic.AddUint64(&s.stats.TotIntroducedSegmentsMerge, 1) + skipped[i] = false + } else { + if nextMerge.fileMerge { + atomic.AddUint64(&s.stats.TotFileMergeIntroductionsObsoleted, 1) + } else { + atomic.AddUint64(&s.stats.TotMemMergeIntroductionsObsoleted, 1) + } + skipped[i] = true + } + } + + atomic.StoreUint64(&s.stats.TotItemsToPersist, docsToPersistCount) + atomic.StoreUint64(&s.stats.TotMemorySegmentsAtRoot, memSegments) + atomic.StoreUint64(&s.stats.TotFileSegmentsAtRoot, fileSegments) + + newSnapshot.AddRef() // 1 ref for the nextMerge.notify response + + newSnapshot.updateSize() + s.rootLock.Lock() + // swap in new index snapshot + newSnapshot.epoch = s.nextSnapshotEpoch + s.nextSnapshotEpoch++ + rootPrev := s.root + s.root = newSnapshot + atomic.StoreUint64(&s.stats.CurRootEpoch, s.root.epoch) + // release lock + s.rootLock.Unlock() + + if rootPrev != nil { + _ = rootPrev.DecRef() + } + + // update the removal eligibility for those segment files + // that are not a part of the latest root. + for _, filename := range droppedSegmentFiles { + s.unmarkIneligibleForRemoval(filename) + } + + // notify requester that we incorporated this + nextMerge.notifyCh <- &mergeTaskIntroStatus{ + indexSnapshot: newSnapshot, + skipped: skipped} + close(nextMerge.notifyCh) +} + +func isMemorySegment(s *SegmentSnapshot) bool { + switch s.segment.(type) { + case segment.PersistedSegment: + return false + default: + return true + } +} diff --git a/index/scorch/merge.go b/index/scorch/merge.go new file mode 100644 index 0000000..20d74c5 --- /dev/null +++ b/index/scorch/merge.go @@ -0,0 +1,730 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/RoaringBitmap/roaring/v2" + "github.com/blevesearch/bleve/v2/index/scorch/mergeplan" + "github.com/blevesearch/bleve/v2/util" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +const merger = "merger" + +// used in the context of mergerCtrl to provide a way to verify +// the completion of a merge operation +const mergeDoneKey = "mergeDone" + +type mergeDoneChan chan error + +// used in the context of mergerCtrl to provide a way to use +// a custom merge plan instead of the one generated by the +// default merge planner +const mergePlanFuncKey = "mergePlanFunc" + +type mergePlanFunc func(*IndexSnapshot) (*mergeplan.MergePlan, error) + +func (s *Scorch) mergerLoop() { + defer func() { + if r := recover(); r != nil { + s.fireAsyncError(NewScorchError( + merger, + fmt.Sprintf("panic: %v, path: %s", r, s.path), + ErrAsyncPanic, + )) + } + + s.asyncTasks.Done() + }() + + var lastEpochMergePlanned uint64 + var ctrlMsg *mergerCtrl + ctrlMsgDflt := &mergerCtrl{ctx: context.Background(), + options: s.mergePlannerOptions, + doneCh: nil} + +OUTER: + for { + atomic.AddUint64(&s.stats.TotFileMergeLoopBeg, 1) + + select { + case <-s.closeCh: + break OUTER + + default: + // check to see if there is a new snapshot to persist + s.rootLock.Lock() + ourSnapshot := s.root + ourSnapshot.AddRef() + atomic.StoreUint64(&s.iStats.mergeSnapshotSize, uint64(ourSnapshot.Size())) + atomic.StoreUint64(&s.iStats.mergeEpoch, ourSnapshot.epoch) + s.rootLock.Unlock() + + if ctrlMsg == nil && ourSnapshot.epoch != lastEpochMergePlanned { + ctrlMsg = ctrlMsgDflt + } + if ctrlMsg != nil { + continueMerge := s.fireEvent(EventKindPreMergeCheck, 0) + // The default, if there's no handler, is to continue the merge. + if !continueMerge { + // If it's decided that this merge can't take place now, + // begin the merge process all over again. + // Retry instead of blocking/waiting here since a long wait + // can result in more segments introduced i.e. s.root will + // be updated. + + // decrement the ref count since its no longer needed in this + // iteration + _ = ourSnapshot.DecRef() + continue OUTER + } + + // lets get started + startTime := time.Now() + err := s.planMergeAtSnapshot(ctrlMsg, ourSnapshot) + if err != nil { + atomic.StoreUint64(&s.iStats.mergeEpoch, 0) + if err == segment.ErrClosed { + // index has been closed + _ = ourSnapshot.DecRef() + + // continue the workloop on a user triggered cancel + if ctrlMsg.doneCh != nil { + close(ctrlMsg.doneCh) + ctrlMsg = nil + continue OUTER + } + + // exit the workloop on index closure + ctrlMsg = nil + break OUTER + } + + s.fireAsyncError(NewScorchError( + merger, + fmt.Sprintf("merging err: %v", err), + ErrPersist, + )) + _ = ourSnapshot.DecRef() + atomic.AddUint64(&s.stats.TotFileMergeLoopErr, 1) + continue OUTER + } + + if ctrlMsg.doneCh != nil { + close(ctrlMsg.doneCh) + } + ctrlMsg = nil + + lastEpochMergePlanned = ourSnapshot.epoch + + atomic.StoreUint64(&s.stats.LastMergedEpoch, ourSnapshot.epoch) + + s.fireEvent(EventKindMergerProgress, time.Since(startTime)) + } + _ = ourSnapshot.DecRef() + + // tell the persister we're waiting for changes + // first make a epochWatcher chan + ew := &epochWatcher{ + epoch: lastEpochMergePlanned, + notifyCh: make(notificationChan, 1), + } + + // give it to the persister + select { + case <-s.closeCh: + break OUTER + case s.persisterNotifier <- ew: + case ctrlMsg = <-s.forceMergeRequestCh: + continue OUTER + } + + // now wait for persister (but also detect close) + select { + case <-s.closeCh: + break OUTER + case <-ew.notifyCh: + case ctrlMsg = <-s.forceMergeRequestCh: + } + } + + atomic.AddUint64(&s.stats.TotFileMergeLoopEnd, 1) + } +} + +type mergerCtrl struct { + ctx context.Context + options *mergeplan.MergePlanOptions + doneCh chan struct{} +} + +// ForceMerge helps users trigger a merge operation on +// an online scorch index. +func (s *Scorch) ForceMerge(ctx context.Context, + mo *mergeplan.MergePlanOptions) error { + // check whether force merge is already under processing + s.rootLock.Lock() + if atomic.LoadUint64(&s.stats.TotFileMergeForceOpsStarted) > + atomic.LoadUint64(&s.stats.TotFileMergeForceOpsCompleted) { + s.rootLock.Unlock() + return fmt.Errorf("force merge already in progress") + } + + atomic.AddUint64(&s.stats.TotFileMergeForceOpsStarted, 1) + s.rootLock.Unlock() + + if mo != nil { + err := mergeplan.ValidateMergePlannerOptions(mo) + if err != nil { + return err + } + } else { + // assume the default single segment merge policy + mo = &mergeplan.SingleSegmentMergePlanOptions + } + msg := &mergerCtrl{options: mo, + doneCh: make(chan struct{}), + ctx: ctx, + } + + // request the merger perform a force merge + select { + case s.forceMergeRequestCh <- msg: + case <-s.closeCh: + return nil + } + + // wait for the force merge operation completion + select { + case <-msg.doneCh: + atomic.AddUint64(&s.stats.TotFileMergeForceOpsCompleted, 1) + case <-s.closeCh: + } + + return nil +} + +func (s *Scorch) parseMergePlannerOptions(po *persisterOptions) (*mergeplan.MergePlanOptions, + error) { + mergePlannerOptions := mergeplan.DefaultMergePlanOptions + // by default use the MaxSizeInMemoryMergePerWorker from the persister option + // as the FloorSegmentFileSize for the merge planner which would be the + // first tier size in the planning. If the value is 0, then we don't use the + // file size in the planning. + mergePlannerOptions.FloorSegmentFileSize = int64(po.MaxSizeInMemoryMergePerWorker) + + if v, ok := s.config["scorchMergePlanOptions"]; ok { + b, err := util.MarshalJSON(v) + if err != nil { + return &mergePlannerOptions, err + } + + err = util.UnmarshalJSON(b, &mergePlannerOptions) + if err != nil { + return &mergePlannerOptions, err + } + + err = mergeplan.ValidateMergePlannerOptions(&mergePlannerOptions) + if err != nil { + return nil, err + } + } + return &mergePlannerOptions, nil +} + +type closeChWrapper struct { + ch1 chan struct{} + ctx context.Context + closeCh chan struct{} + cancelCh chan struct{} +} + +func newCloseChWrapper(ch1 chan struct{}, + ctx context.Context) *closeChWrapper { + return &closeChWrapper{ + ch1: ch1, + ctx: ctx, + closeCh: make(chan struct{}), + cancelCh: make(chan struct{}), + } +} + +func (w *closeChWrapper) close() { + close(w.closeCh) +} + +func (w *closeChWrapper) listen() { + select { + case <-w.ch1: + close(w.cancelCh) + case <-w.ctx.Done(): + close(w.cancelCh) + case <-w.closeCh: + } +} + +// mergeBatch represents a batch of segments to be merged together, +// along with the new segment that will be created as a result of the merge. +type mergeBatch struct { + snapshots []*SegmentSnapshot + filenames []string + segments []segment.Segment + drops []*roaring.Bitmap + + new segment.Segment + newID uint64 + newDocNums [][]uint64 + newFilename string + newTime uint64 +} + +// planMergeAtSnapshot plans and executes the merge operations for a given snapshot +// if there is a custom merge plan function provided, it uses that to get the merge plan +// otherwise, it builds the merge plan using the default planner and executes the merge tasks in the plan. +func (s *Scorch) planMergeAtSnapshot(ctrlMsg *mergerCtrl, ourSnapshot *IndexSnapshot) error { + var mergePlan *mergeplan.MergePlan + // if a merge plan function is provided in the context, use it to get the merge plan + if mergePlanFunc, ok := ctrlMsg.ctx.Value(mergePlanFuncKey).(mergePlanFunc); ok { + var err error + mergePlan, err = mergePlanFunc(ourSnapshot) + if err != nil { + atomic.AddUint64(&s.stats.TotFileMergePlanErr, 1) + return fmt.Errorf("merge planning err: %v", err) + } + } + + // default to making a merge plan if a custom one is not provided + if mergePlan == nil || len(mergePlan.Tasks) == 0 { + // build list of persisted segments in this snapshot + var onlyPersistedSnapshots []mergeplan.Segment + for _, segmentSnapshot := range ourSnapshot.segment { + if _, ok := segmentSnapshot.segment.(segment.PersistedSegment); ok { + onlyPersistedSnapshots = append(onlyPersistedSnapshots, segmentSnapshot) + } + } + + atomic.AddUint64(&s.stats.TotFileMergePlan, 1) + + // give this list to the planner + var err error + mergePlan, err = mergeplan.Plan(onlyPersistedSnapshots, ctrlMsg.options) + if err != nil { + atomic.AddUint64(&s.stats.TotFileMergePlanErr, 1) + return fmt.Errorf("merge planning err: %v", err) + } + + if mergePlan == nil || len(mergePlan.Tasks) == 0 { + // nothing to do + atomic.AddUint64(&s.stats.TotFileMergePlanNone, 1) + return nil + } + } + + atomic.AddUint64(&s.stats.TotFileMergePlanOk, 1) + atomic.AddUint64(&s.stats.TotFileMergePlanTasks, uint64(len(mergePlan.Tasks))) + + cw := newCloseChWrapper(s.closeCh, ctrlMsg.ctx) + defer cw.close() + go cw.listen() + + var err error + defer func() { + // send error to done channel if present + if done, ok := cw.ctx.Value(mergeDoneKey).(chan error); ok { + done <- err + } + }() + + numBatches := len(mergePlan.Tasks) + mergeBatches := make([]*mergeBatch, numBatches) + defer func() { + for batchID := 0; batchID < numBatches; batchID++ { + batch := mergeBatches[batchID] + if batch == nil { + continue + } + if err == nil { + s.rootLock.Lock() + for _, filename := range batch.filenames { + delete(s.ineligibleForRemoval, filename) + } + s.rootLock.Unlock() + } else { + if batch.new != nil { + _ = batch.new.Close() + } + s.unmarkIneligibleForRemoval(batch.newFilename) + } + } + }() + + numMergedSegments := 0 + for batchID := 0; batchID < numBatches; batchID++ { + task := mergePlan.Tasks[batchID] + if len(task.Segments) == 0 { + atomic.AddUint64(&s.stats.TotFileMergePlanTasksSegmentsEmpty, 1) + } + atomic.AddUint64(&s.stats.TotFileMergePlanTasksSegments, uint64(len(task.Segments))) + + newSegmentID := atomic.AddUint64(&s.nextSegmentID, 1) + batch := &mergeBatch{ + snapshots: make([]*SegmentSnapshot, 0, len(task.Segments)), + filenames: make([]string, 0, len(task.Segments)), + segments: make([]segment.Segment, 0, len(task.Segments)), + drops: make([]*roaring.Bitmap, 0, len(task.Segments)), + + new: nil, + newID: newSegmentID, + newDocNums: nil, + newFilename: zapFileName(newSegmentID), + newTime: 0, + } + mergeBatches[batchID] = batch + + for _, planSegment := range task.Segments { + if segSnapshot, ok := planSegment.(*SegmentSnapshot); ok { + if persistedSeg, ok := segSnapshot.segment.(segment.PersistedSegment); ok { + if segSnapshot.LiveSize() == 0 { + atomic.AddUint64(&s.stats.TotFileMergeSegmentsEmpty, 1) + } else { + batch.snapshots = append(batch.snapshots, segSnapshot) + batch.segments = append(batch.segments, segSnapshot.segment) + batch.drops = append(batch.drops, segSnapshot.deleted) + } + // track the files getting merged for unsetting the + // removal ineligibility. This helps to unflip files + // even with fast merger, slow persister work flows. + path := persistedSeg.Path() + batch.filenames = append(batch.filenames, + strings.TrimPrefix(path, s.path+string(os.PathSeparator))) + } + } + } + + if len(batch.segments) == 0 { + continue + } + + numMergedSegments += len(batch.snapshots) + s.markIneligibleForRemoval(batch.newFilename) + path := s.path + string(os.PathSeparator) + batch.newFilename + + prevBytesReadTotal := cumulateBytesRead(batch.segments) + fileMergeZapStartTime := time.Now() + atomic.AddUint64(&s.stats.TotFileMergeZapBeg, 1) + batch.newDocNums, _, err = s.segPlugin.MergeUsing(batch.segments, batch.drops, path, + cw.cancelCh, s, s.segmentConfig) + atomic.AddUint64(&s.stats.TotFileMergeZapEnd, 1) + batch.newTime = uint64(time.Since(fileMergeZapStartTime)) + atomic.AddUint64(&s.stats.TotFileMergeZapTime, batch.newTime) + if atomic.LoadUint64(&s.stats.MaxFileMergeZapTime) < batch.newTime { + atomic.StoreUint64(&s.stats.MaxFileMergeZapTime, batch.newTime) + } + if err != nil { + atomic.AddUint64(&s.stats.TotFileMergePlanTasksErr, 1) + return err + } + + batch.new, err = s.segPlugin.OpenUsing(path, s.segmentConfig) + if err != nil { + atomic.AddUint64(&s.stats.TotFileMergePlanTasksErr, 1) + return err + } + + totalBytesRead := batch.new.BytesRead() + prevBytesReadTotal + batch.new.ResetBytesRead(totalBytesRead) + atomic.AddUint64(&s.stats.TotFileMergeSegments, uint64(len(batch.segments))) + } + + newSegmentIDs := make([]uint64, numBatches) + newSegments := make([]segment.Segment, numBatches) + mergedSegHistory := make(map[uint64]*mergedSegmentHistory, numMergedSegments) + for batchID := 0; batchID < numBatches; batchID++ { + batch := mergeBatches[batchID] + newSegmentIDs[batchID] = batch.newID + newSegments[batchID] = batch.new + for j, ss := range batch.snapshots { + mergedSegHistory[ss.id] = &mergedSegmentHistory{ + batchID: batchID, + oldNewDocIDs: batch.newDocNums[j], + oldSegment: ss, + } + } + } + + sm := &segmentMerge{ + newSegmentIDs: newSegmentIDs, + newSegments: newSegments, + mergedSegHistory: mergedSegHistory, + notifyCh: make(chan *mergeTaskIntroStatus), + mmaped: 1, + fileMerge: true, + } + + s.fireEvent(EventKindMergeTaskIntroductionStart, 0) + // give it to the introducer + select { + case <-s.closeCh: + err = segment.ErrClosed + return err + case s.merges <- sm: + atomic.AddUint64(&s.stats.TotFileMergeIntroductions, uint64(numBatches)) + } + + introStartTime := time.Now() + // it is safe to blockingly wait for the merge introduction + // here as the introducer is bound to handle the notify channel. + introStatus := <-sm.notifyCh + introTime := uint64(time.Since(introStartTime)) + atomic.AddUint64(&s.stats.TotFileMergeZapIntroductionTime, introTime) + if atomic.LoadUint64(&s.stats.MaxFileMergeZapIntroductionTime) < introTime { + atomic.StoreUint64(&s.stats.MaxFileMergeZapIntroductionTime, introTime) + } + atomic.AddUint64(&s.stats.TotFileMergeIntroductionsDone, uint64(numBatches)) + + _ = introStatus.indexSnapshot.DecRef() + for batchID, skipped := range introStatus.skipped { + if skipped { + batch := mergeBatches[batchID] + if batch.new != nil { + _ = batch.new.Close() + } + s.unmarkIneligibleForRemoval(batch.newFilename) + } + } + + atomic.AddUint64(&s.stats.TotFileMergePlanTasksDone, uint64(numBatches)) + s.fireEvent(EventKindMergeTaskIntroduction, 0) + return nil +} + +type mergeTaskIntroStatus struct { + indexSnapshot *IndexSnapshot + skipped []bool +} + +// this is important when it comes to introducing multiple merged segments in a +// single introducer channel push. That way there is a check to ensure that the +// file count doesn't explode during the index's lifetime. +type mergedSegmentHistory struct { + batchID int + oldNewDocIDs []uint64 + oldSegment *SegmentSnapshot +} + +type segmentMerge struct { + newSegmentIDs []uint64 + newSegments []segment.Segment + mergedSegHistory map[uint64]*mergedSegmentHistory + notifyCh chan *mergeTaskIntroStatus + mmaped uint32 + fileMerge bool +} + +func cumulateBytesRead(sbs []segment.Segment) uint64 { + var rv uint64 + for _, seg := range sbs { + rv += seg.BytesRead() + } + return rv +} + +func (s *Scorch) mergeAndPersistInMemorySegments(flushes []*flushable, po *persisterOptions) (*IndexSnapshot, map[uint64]struct{}, error) { + atomic.AddUint64(&s.stats.TotMemMergeBeg, 1) + + var err error + numBatches := len(flushes) + mergeBatches := make([]*mergeBatch, numBatches) + defer func() { + if err != nil { + for batchID := 0; batchID < numBatches; batchID++ { + batch := mergeBatches[batchID] + if batch.new != nil { + _ = batch.new.Close() + } + s.unmarkIneligibleForRemoval(batch.newFilename) + } + } + }() + + var numMergedSegments uint64 + sem := make(chan struct{}, po.NumPersisterWorkers) + var wg sync.WaitGroup + var errM sync.Mutex + var errs []error + for batchID := 0; batchID < numBatches; batchID++ { + wg.Add(1) + sem <- struct{}{} + go func(batchID int) { + var err error + defer func() { + if err != nil { + atomic.AddUint64(&s.stats.TotMemMergeErr, 1) + errM.Lock() + errs = append(errs, err) + errM.Unlock() + } + <-sem + wg.Done() + }() + + flush := flushes[batchID] + newSegmentID := atomic.AddUint64(&s.nextSegmentID, 1) + batch := &mergeBatch{ + snapshots: flush.sbsBatchSnapshots, + filenames: nil, + segments: flush.sbsBatch, + drops: flush.sbsBatchDrops, + + new: nil, + newID: newSegmentID, + newDocNums: nil, + newFilename: zapFileName(newSegmentID), + newTime: 0, + } + mergeBatches[batchID] = batch + + s.markIneligibleForRemoval(batch.newFilename) + path := s.path + string(os.PathSeparator) + batch.newFilename + + memMergeZapStartTime := time.Now() + atomic.AddUint64(&s.stats.TotMemMergeZapBeg, 1) + batch.newDocNums, _, err = s.segPlugin.MergeUsing(batch.segments, batch.drops, path, + s.closeCh, s, s.segmentConfig) + atomic.AddUint64(&s.stats.TotMemMergeZapEnd, 1) + batch.newTime = uint64(time.Since(memMergeZapStartTime)) + atomic.AddUint64(&s.stats.TotMemMergeZapTime, batch.newTime) + if err != nil { + return + } + batch.new, err = s.segPlugin.OpenUsing(path, s.segmentConfig) + if err != nil { + return + } + atomic.AddUint64(&numMergedSegments, uint64(len(batch.segments))) + }(batchID) + } + wg.Wait() + close(sem) + + var maxZapTime uint64 + for _, batch := range mergeBatches { + if batch.newTime > maxZapTime { + maxZapTime = batch.newTime + } + } + if maxZapTime > atomic.LoadUint64(&s.stats.MaxMemMergeZapTime) { + atomic.StoreUint64(&s.stats.MaxMemMergeZapTime, maxZapTime) + } + atomic.AddUint64(&s.stats.TotMemMergeSegments, numMergedSegments) + + if len(errs) > 0 { + var errf error + numClosed := 0 + for _, e := range errs { + if e == segment.ErrClosed { + numClosed++ + } + errf = errors.Join(errf, e) + } + if numClosed == len(errs) { + // the index snapshot was closed which will be handled gracefully + // by retrying the whole merge+flush operation in a later iteration + // so its safe to early exit the same error. + err = segment.ErrClosed + } else { + err = errf + } + return nil, nil, err + } + + newSegmentIDs := make([]uint64, numBatches) + newSegments := make([]segment.Segment, numBatches) + mergedSegHistory := make(map[uint64]*mergedSegmentHistory, numMergedSegments) + for batchID := 0; batchID < numBatches; batchID++ { + batch := mergeBatches[batchID] + newSegmentIDs[batchID] = batch.newID + newSegments[batchID] = batch.new + for j, ss := range batch.snapshots { + mergedSegHistory[ss.id] = &mergedSegmentHistory{ + batchID: batchID, + oldNewDocIDs: batch.newDocNums[j], + oldSegment: ss, + } + } + } + + sm := &segmentMerge{ + newSegmentIDs: newSegmentIDs, + newSegments: newSegments, + mergedSegHistory: mergedSegHistory, + notifyCh: make(chan *mergeTaskIntroStatus), + mmaped: 1, + fileMerge: false, + } + + select { + case <-s.closeCh: + err = segment.ErrClosed + return nil, nil, err + case s.merges <- sm: + atomic.AddUint64(&s.stats.TotMemMergeIntroductions, uint64(numBatches)) + } + + introStartTime := time.Now() + // blockingly wait for the introduction to complete + introStatus := <-sm.notifyCh + introTime := uint64(time.Since(introStartTime)) + atomic.AddUint64(&s.stats.TotMemMergeZapIntroductionTime, introTime) + if atomic.LoadUint64(&s.stats.MaxMemMergeZapIntroductionTime) < introTime { + atomic.StoreUint64(&s.stats.MaxMemMergeZapIntroductionTime, introTime) + } + atomic.AddUint64(&s.stats.TotMemMergeIntroductionsDone, uint64(numBatches)) + introducedSnapshot := introStatus.indexSnapshot + introducedSegmentIDs := make(map[uint64]struct{}, numBatches) + for batchID, skipped := range introStatus.skipped { + batch := mergeBatches[batchID] + if skipped { + _ = batch.new.Close() + s.unmarkIneligibleForRemoval(batch.newFilename) + } else { + introducedSegmentIDs[batch.newID] = struct{}{} + } + } + + atomic.AddUint64(&s.stats.TotMemMergeDone, 1) + + // if we could not introduce all of the newly merged segments, + // then we cannot persist a snapshot with any merged segments at all + if len(introducedSegmentIDs) != numBatches { + _ = introducedSnapshot.DecRef() + return nil, nil, nil + } + + return introducedSnapshot, introducedSegmentIDs, nil +} + +func (s *Scorch) ReportBytesWritten(bytesWritten uint64) { + atomic.AddUint64(&s.stats.TotFileMergeWrittenBytes, bytesWritten) +} diff --git a/index/scorch/merge_test.go b/index/scorch/merge_test.go new file mode 100644 index 0000000..b324cb7 --- /dev/null +++ b/index/scorch/merge_test.go @@ -0,0 +1,157 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "sync" + "sync/atomic" + "testing" + + "github.com/blevesearch/bleve/v2/document" + index "github.com/blevesearch/bleve_index_api" +) + +func TestObsoleteSegmentMergeIntroduction(t *testing.T) { + testConfig := CreateConfig("TestObsoleteSegmentMergeIntroduction") + err := InitTest(testConfig) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(testConfig) + if err != nil { + t.Fatal(err) + } + }() + + var introComplete, mergeIntroStart, mergeIntroComplete sync.WaitGroup + introComplete.Add(1) + mergeIntroStart.Add(1) + mergeIntroComplete.Add(1) + var segIntroCompleted int + RegistryEventCallbacks["test"] = func(e Event) bool { + switch e.Kind { + case EventKindBatchIntroduction: + segIntroCompleted++ + if segIntroCompleted == 3 { + // all 3 segments introduced + introComplete.Done() + } + case EventKindMergeTaskIntroductionStart: + // signal the start of merge task introduction so that + // we can introduce a new batch which obsoletes the + // merged segment's contents. + mergeIntroStart.Done() + // hold the merge task introduction until the merged segment contents + // are obsoleted with the next batch/segment introduction. + introComplete.Wait() + case EventKindMergeTaskIntroduction: + // signal the completion of the merge task introduction. + mergeIntroComplete.Done() + + } + + return true + } + + ourConfig := make(map[string]interface{}, len(testConfig)) + for k, v := range testConfig { + ourConfig[k] = v + } + ourConfig["eventCallbackName"] = "test" + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, ourConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + // first introduce two documents over two batches. + batch := index.NewBatch() + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test3"))) + batch.Update(doc) + err = idx.Batch(batch) + if err != nil { + t.Error(err) + } + + batch.Reset() + doc = document.NewDocument("2") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test2updated"))) + batch.Update(doc) + err = idx.Batch(batch) + if err != nil { + t.Error(err) + } + + // wait until the merger trying to introduce the new merged segment. + mergeIntroStart.Wait() + + // execute another batch which obsoletes the contents of the new merged + // segment awaiting introduction. + batch.Reset() + batch.Delete("1") + batch.Delete("2") + doc = document.NewDocument("3") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test3updated"))) + batch.Update(doc) + err = idx.Batch(batch) + if err != nil { + t.Error(err) + } + + // wait until the merge task introduction complete. + mergeIntroComplete.Wait() + + idxr, err := idx.Reader() + if err != nil { + t.Error(err) + } + + numSegments := len(idxr.(*IndexSnapshot).segment) + if numSegments != 1 { + t.Errorf("expected one segment at the root, got: %d", numSegments) + } + + skipIntroCount := atomic.LoadUint64(&idxr.(*IndexSnapshot).parent.stats.TotFileMergeIntroductionsObsoleted) + if skipIntroCount != 1 { + t.Errorf("expected one obsolete merge segment skipping the introduction, got: %d", skipIntroCount) + } + + docCount, err := idxr.DocCount() + if err != nil { + t.Fatal(err) + } + if docCount != 1 { + t.Errorf("Expected document count to be %d got %d", 1, docCount) + } + + err = idxr.Close() + if err != nil { + t.Fatal(err) + } +} diff --git a/index/scorch/mergeplan/merge_plan.go b/index/scorch/mergeplan/merge_plan.go new file mode 100644 index 0000000..a9bfe4c --- /dev/null +++ b/index/scorch/mergeplan/merge_plan.go @@ -0,0 +1,542 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package mergeplan provides a segment merge planning approach that's +// inspired by Lucene's TieredMergePolicy.java and descriptions like +// http://blog.mikemccandless.com/2011/02/visualizing-lucenes-segment-merges.html +package mergeplan + +import ( + "errors" + "fmt" + "math" + "sort" + "strings" +) + +// A Segment represents the information that the planner needs to +// calculate segment merging. +type Segment interface { + // Unique id of the segment -- used for sorting. + Id() uint64 + + // Full segment size (the size before any logical deletions). + FullSize() int64 + + // Size of the live data of the segment; i.e., FullSize() minus + // any logical deletions. + LiveSize() int64 + + HasVector() bool + + // Size of the persisted segment file. + FileSize() int64 + + // Size of the persisted segment file + // accounting for any logical deletions. + LiveFileSize() int64 +} + +// Plan() will functionally compute a merge plan. A segment will be +// assigned to at most a single MergeTask in the output MergePlan. A +// segment not assigned to any MergeTask means the segment should +// remain unmerged. +func Plan(segments []Segment, o *MergePlanOptions) (*MergePlan, error) { + return plan(segments, o) +} + +// A MergePlan is the result of the Plan() API. +// +// The planner doesn't know how or whether these tasks are executed -- +// that's up to a separate merge execution system, which might execute +// these tasks concurrently or not, and which might execute all the +// tasks or not. +type MergePlan struct { + Tasks []*MergeTask +} + +// A MergeTask represents several segments that should be merged +// together into a single segment. +type MergeTask struct { + Segments []Segment +} + +// The MergePlanOptions is designed to be reusable between planning calls. +type MergePlanOptions struct { + // Max # segments per logarithmic tier, or max width of any + // logarithmic "step". Smaller values mean more merging but fewer + // segments. Should be >= SegmentsPerMergeTask, else you'll have + // too much merging. + MaxSegmentsPerTier int + + // Max size of any segment produced after merging. Actual + // merging, however, may produce segment sizes different than the + // planner's predicted sizes. + MaxSegmentSize int64 + + // Max size (in bytes) of the persisted segment file that contains the + // vectors. This is used to prevent merging of segments that + // contain vectors that are too large. + MaxSegmentFileSize int64 + + // The growth factor for each tier in a staircase of idealized + // segments computed by CalcBudget(). + TierGrowth float64 + + // The number of segments in any resulting MergeTask. e.g., + // len(result.Tasks[ * ].Segments) == SegmentsPerMergeTask. + SegmentsPerMergeTask int + + // Small segments are rounded up to this size, i.e., treated as + // equal (floor) size for consideration. This is to prevent lots + // of tiny segments from resulting in a long tail in the index. + FloorSegmentSize int64 + + // Small segments' file size are rounded up to this size to prevent lot + // of tiny segments causing a long tail in the index. + FloorSegmentFileSize int64 + + // Controls how aggressively merges that reclaim more deletions + // are favored. Higher values will more aggressively target + // merges that reclaim deletions, but be careful not to go so high + // that way too much merging takes place; a value of 3.0 is + // probably nearly too high. A value of 0.0 means deletions don't + // impact merge selection. + ReclaimDeletesWeight float64 + + // Optional, defaults to mergeplan.CalcBudget(). + CalcBudget func(totalSize int64, firstTierSize int64, + o *MergePlanOptions) (budgetNumSegments int) + + // Optional, defaults to mergeplan.ScoreSegments(). + ScoreSegments func(segments []Segment, o *MergePlanOptions) float64 + + // Optional. + Logger func(string) +} + +// Returns the higher of the input or FloorSegmentSize. +func (o *MergePlanOptions) RaiseToFloorSegmentSize(s int64) int64 { + if s > o.FloorSegmentSize { + return s + } + return o.FloorSegmentSize +} + +func (o *MergePlanOptions) RaiseToFloorSegmentFileSize(s int64) int64 { + if s > o.FloorSegmentFileSize { + return s + } + return o.FloorSegmentFileSize +} + +func (o *MergePlanOptions) BudgetCurrency() BudgetCurrency { + if o.FloorSegmentFileSize > 0 { + return FileSizeBudget + } + return LiveSizeBudget +} + +// MaxSegmentSizeLimit represents the maximum size of a segment, +// this limit comes with hit-1 optimisation/max encoding limit uint31. +const MaxSegmentSizeLimit = 1<<31 - 1 + +// ErrMaxSegmentSizeTooLarge is returned when the size of the segment +// exceeds the MaxSegmentSizeLimit +var ErrMaxSegmentSizeTooLarge = errors.New("MaxSegmentSize exceeds the size limit") + +// DefaultMergePlanOptions suggests the default options. +var DefaultMergePlanOptions = MergePlanOptions{ + MaxSegmentsPerTier: 10, + MaxSegmentSize: 5000000, + MaxSegmentFileSize: 4 * 1024 * 1024 * 1024, // 4GiB + TierGrowth: 10.0, + SegmentsPerMergeTask: 10, + FloorSegmentSize: 2000, + ReclaimDeletesWeight: 2.0, +} + +// SingleSegmentMergePlanOptions helps in creating a +// single segment index. +var SingleSegmentMergePlanOptions = MergePlanOptions{ + MaxSegmentsPerTier: 1, + MaxSegmentSize: 1 << 30, + MaxSegmentFileSize: 1 << 40, + TierGrowth: 1.0, + SegmentsPerMergeTask: 10, + FloorSegmentSize: 1 << 30, + ReclaimDeletesWeight: 2.0, + FloorSegmentFileSize: 1 << 40, +} + +// ------------------------------------------- + +// BudgetCurrency is the currency used for budget +// calculations in the merge planner. +type BudgetCurrency int + +const ( + // FileSizeBudget uses the file size + // of the segments to determine the budget. + FileSizeBudget BudgetCurrency = iota + + // LiveSizeBudget uses the live size + // of the segments to determine the budget. + LiveSizeBudget +) + +var ErrUnknownCurrency = errors.New("unknown budget currency") + +// ------------------------------------------- + +// rosterCandidate is a potential roster of segments considered while +// planning, along with its computed score. +type rosterCandidate struct { + segments []Segment + score float64 // Lower is better. +} + +func (c *rosterCandidate) betterThan(other *rosterCandidate) bool { + ourSegmentCount := c.count() + otherSegmentCount := other.count() + // if we have more than one segment and the other has only one, and that + // one segment is fully live, then we are better than the other. + if ourSegmentCount > 1 && otherSegmentCount == 1 { + solo := other.segments[0] + if solo.LiveSize() == solo.FullSize() { + return true + } + } + // if we have only one segment and the other has more than one, and that + // one segment is fully live, then we are worse than the other. + if ourSegmentCount == 1 && otherSegmentCount > 1 { + solo := c.segments[0] + if solo.LiveSize() == solo.FullSize() { + return false + } + } + return c.score < other.score +} + +func (c *rosterCandidate) count() int { + return len(c.segments) +} + +// ------------------------------------------- + +func plan(segmentsIn []Segment, o *MergePlanOptions) (*MergePlan, error) { + if len(segmentsIn) <= 1 { + return nil, nil + } + + if o == nil { + o = &DefaultMergePlanOptions + } + + segments := append([]Segment(nil), segmentsIn...) // Copy. + + sort.Sort(byLiveSizeDescending(segments)) + + var minLiveSize int64 = math.MaxInt64 + var minLiveFileSize int64 = math.MaxInt64 + + var eligibles []Segment + var eligiblesLiveSize int64 + var eligiblesLiveFileSize int64 + + for _, segment := range segments { + liveSize := segment.LiveSize() + liveFileSize := segment.LiveFileSize() + + if minLiveSize > liveSize { + minLiveSize = liveSize + } + + if minLiveFileSize > liveFileSize { + minLiveFileSize = liveFileSize + } + + isEligible := liveSize < o.MaxSegmentSize/2 + // An eligible segment (based on #documents) may be too large + // and thus need a stricter check based on the file size. + // This is particularly important for segments that contain + // vectors. + if segment.HasVector() { + isEligible = isEligible && liveFileSize < o.MaxSegmentFileSize/2 + } + + // Only small-enough segments are eligible. + if isEligible { + eligibles = append(eligibles, segment) + eligiblesLiveSize += liveSize + eligiblesLiveFileSize += liveFileSize + } + } + + calcBudget := o.CalcBudget + if calcBudget == nil { + calcBudget = CalcBudget + } + + currency := o.BudgetCurrency() + + var budgetNumSegments int + switch currency { + case FileSizeBudget: + minLiveFileSize = o.RaiseToFloorSegmentFileSize(minLiveFileSize) + budgetNumSegments = calcBudget(eligiblesLiveFileSize, minLiveFileSize, o) + case LiveSizeBudget: + minLiveSize = o.RaiseToFloorSegmentSize(minLiveSize) + budgetNumSegments = calcBudget(eligiblesLiveSize, minLiveSize, o) + default: + return nil, ErrUnknownCurrency + } + + scoreSegments := o.ScoreSegments + if scoreSegments == nil { + scoreSegments = ScoreSegments + } + + rv := &MergePlan{} + + var empties []Segment + for _, eligible := range eligibles { + if eligible.LiveSize() <= 0 { + empties = append(empties, eligible) + } + } + if len(empties) > 0 { + rv.Tasks = append(rv.Tasks, &MergeTask{Segments: empties}) + eligibles = removeSegments(eligibles, empties) + } + + numMergeTasks := 0 + + // While we're over budget, keep looping, which might produce + // another MergeTask. + for len(eligibles) > 0 && (len(eligibles)+numMergeTasks) > budgetNumSegments { + // Track the current best candidate as we examine and score + // potential rosters of merges. + var best *rosterCandidate + + for startIdx := 0; startIdx < len(eligibles); startIdx++ { + var roster []Segment + var rosterLiveSize int64 + var rosterLiveFileSize int64 + + for idx := startIdx; idx < len(eligibles) && len(roster) < o.SegmentsPerMergeTask; idx++ { + eligible := eligibles[idx] + + liveSize := eligible.LiveSize() + + if rosterLiveSize+liveSize >= o.MaxSegmentSize { + continue + } + + if eligible.HasVector() { + liveFileSize := eligible.LiveFileSize() + + if rosterLiveFileSize+liveFileSize >= o.MaxSegmentFileSize { + continue + } + + rosterLiveFileSize += liveFileSize + } + + roster = append(roster, eligible) + + rosterLiveSize += liveSize + } + + if len(roster) > 0 { + rosterScore := scoreSegments(roster, o) + + candidate := &rosterCandidate{ + segments: roster, + score: rosterScore, + } + + if best == nil || candidate.betterThan(best) { + best = candidate + } + } + } + + if best == nil { + return rv, nil + } + + bestRoster := best.segments + + // create tasks with valid merges - i.e. there should be at least 2 non-empty segments + // or the bestRoster should have 1 segment with deletes to reclaim. + if len(bestRoster) > 1 || bestRoster[0].LiveSize() < bestRoster[0].FullSize() { + rv.Tasks = append(rv.Tasks, &MergeTask{Segments: bestRoster}) + numMergeTasks++ + } + + eligibles = removeSegments(eligibles, bestRoster) + } + + return rv, nil +} + +// Compute the number of segments that would be needed to cover the +// totalSize, by climbing up a logarithmically growing staircase of +// segment tiers. +func CalcBudget(totalSize int64, firstTierSize int64, o *MergePlanOptions) ( + budgetNumSegments int) { + tierSize := firstTierSize + if tierSize < 1 { + tierSize = 1 + } + + maxSegmentsPerTier := o.MaxSegmentsPerTier + if maxSegmentsPerTier < 1 { + maxSegmentsPerTier = 1 + } + + tierGrowth := o.TierGrowth + if tierGrowth < 1.0 { + tierGrowth = 1.0 + } + + for totalSize > 0 { + segmentsInTier := float64(totalSize) / float64(tierSize) + if segmentsInTier < float64(maxSegmentsPerTier) { + budgetNumSegments += int(math.Ceil(segmentsInTier)) + break + } + + budgetNumSegments += maxSegmentsPerTier + totalSize -= int64(maxSegmentsPerTier) * tierSize + tierSize = int64(float64(tierSize) * tierGrowth) + } + + return budgetNumSegments +} + +// Of note, removeSegments() keeps the ordering of the results stable. +func removeSegments(segments []Segment, toRemove []Segment) []Segment { + rv := make([]Segment, 0, len(segments)-len(toRemove)) +OUTER: + for _, segment := range segments { + for _, r := range toRemove { + if segment == r { + continue OUTER + } + } + rv = append(rv, segment) + } + return rv +} + +// Smaller result score is better. +func ScoreSegments(segments []Segment, o *MergePlanOptions) float64 { + var totBeforeSize int64 + var totAfterSize int64 + var totAfterSizeFloored int64 + + for _, segment := range segments { + totBeforeSize += segment.FullSize() + totAfterSize += segment.LiveSize() + totAfterSizeFloored += o.RaiseToFloorSegmentSize(segment.LiveSize()) + } + + if totBeforeSize <= 0 || totAfterSize <= 0 || totAfterSizeFloored <= 0 { + return 0 + } + + // Roughly guess the "balance" of the segments -- whether the + // segments are about the same size. + balance := + float64(o.RaiseToFloorSegmentSize(segments[0].LiveSize())) / + float64(totAfterSizeFloored) + + // Gently favor smaller merges over bigger ones. We don't want to + // make the exponent too large else we end up with poor merges of + // small segments in order to avoid the large merges. + score := balance * math.Pow(float64(totAfterSize), 0.05) + + // Strongly favor merges that reclaim deletes. + nonDelRatio := float64(totAfterSize) / float64(totBeforeSize) + + score *= math.Pow(nonDelRatio, o.ReclaimDeletesWeight) + + return score +} + +// ------------------------------------------ + +// ToBarChart returns an ASCII rendering of the segments and the plan. +// The barMax is the max width of the bars in the bar chart. +func ToBarChart(prefix string, barMax int, segments []Segment, plan *MergePlan) string { + rv := make([]string, 0, len(segments)) + + var maxFullSize int64 + for _, segment := range segments { + if maxFullSize < segment.FullSize() { + maxFullSize = segment.FullSize() + } + } + if maxFullSize < 0 { + maxFullSize = 1 + } + + for _, segment := range segments { + barFull := int(segment.FullSize()) + barLive := int(segment.LiveSize()) + + if maxFullSize > int64(barMax) { + barFull = int(float64(barMax) * float64(barFull) / float64(maxFullSize)) + barLive = int(float64(barMax) * float64(barLive) / float64(maxFullSize)) + } + + barKind := " " + barChar := "." + + if plan != nil { + TASK_LOOP: + for taski, task := range plan.Tasks { + for _, taskSegment := range task.Segments { + if taskSegment == segment { + barKind = "*" + barChar = fmt.Sprintf("%d", taski) + break TASK_LOOP + } + } + } + } + + bar := + strings.Repeat(barChar, barLive)[0:barLive] + + strings.Repeat("x", barFull-barLive)[0:barFull-barLive] + + rv = append(rv, fmt.Sprintf("%s %5d: %5d /%5d - %s %s", prefix, + segment.Id(), + segment.LiveSize(), + segment.FullSize(), + barKind, bar)) + } + + return strings.Join(rv, "\n") +} + +// ValidateMergePlannerOptions validates the merge planner options +func ValidateMergePlannerOptions(options *MergePlanOptions) error { + if options.MaxSegmentSize > MaxSegmentSizeLimit { + return ErrMaxSegmentSizeTooLarge + } + return nil +} diff --git a/index/scorch/mergeplan/merge_plan_test.go b/index/scorch/mergeplan/merge_plan_test.go new file mode 100644 index 0000000..25593fb --- /dev/null +++ b/index/scorch/mergeplan/merge_plan_test.go @@ -0,0 +1,1127 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mergeplan + +import ( + "encoding/json" + "fmt" + "math/rand" + "os" + "reflect" + "slices" + "sort" + "testing" + "time" +) + +// Implements the Segment interface for testing, +type segment struct { + MyId uint64 + MyFullSize int64 + MyLiveSize int64 + + MyHasVector bool + MyFileSize int64 +} + +func (s *segment) Id() uint64 { return s.MyId } +func (s *segment) FullSize() int64 { return s.MyFullSize } +func (s *segment) LiveSize() int64 { return s.MyLiveSize } +func (s *segment) HasVector() bool { return s.MyHasVector } +func (s *segment) FileSize() int64 { return s.MyFileSize } +func (s *segment) LiveFileSize() int64 { + // LiveFileSize is an estimate of the live portion of the file size, + // based on the ratio of live to full size. + fullSize := float64(s.MyFullSize) + if fullSize <= 0 { + return 0 + } + liveSize := float64(s.MyLiveSize) + if liveSize <= 0 { + return 0 + } + fileSize := float64(s.MyFileSize) + if fileSize <= 0 { + return 0 + } + liveRatio := liveSize / fullSize + return int64(fileSize * liveRatio) +} + +func makeLinearSegments(n int) (rv []Segment) { + for i := 0; i < n; i++ { + rv = append(rv, &segment{ + MyId: uint64(i), + MyFullSize: int64(i), + MyLiveSize: int64(i), + }) + } + return rv +} + +// ---------------------------------------- + +func TestSimplePlan(t *testing.T) { + segs := makeLinearSegments(10) + + tests := []struct { + Desc string + Segments []Segment + Options *MergePlanOptions + ExpectPlan *MergePlan + ExpectErr error + }{ + { + "nil segments", + nil, nil, nil, nil, + }, + { + "empty segments", + []Segment{}, + nil, nil, nil, + }, + { + "1 segment", + []Segment{segs[1]}, + nil, + nil, + nil, + }, + { + "2 segments", + []Segment{ + segs[1], + segs[2], + }, + nil, + &MergePlan{ + Tasks: []*MergeTask{ + { + Segments: []Segment{ + segs[2], + segs[1], + }, + }, + }, + }, + nil, + }, + { + "3 segments", + []Segment{ + segs[1], + segs[2], + segs[9], + }, + nil, + &MergePlan{ + Tasks: []*MergeTask{ + { + Segments: []Segment{ + segs[9], + segs[2], + segs[1], + }, + }, + }, + }, + nil, + }, + { + "many segments", + []Segment{ + segs[1], + segs[2], + segs[3], + segs[4], + segs[5], + segs[6], + }, + &MergePlanOptions{ + MaxSegmentsPerTier: 1, + MaxSegmentSize: 1000, + TierGrowth: 2.0, + SegmentsPerMergeTask: 2, + FloorSegmentSize: 1, + }, + &MergePlan{ + Tasks: []*MergeTask{ + { + Segments: []Segment{ + segs[6], + segs[5], + }, + }, + }, + }, + nil, + }, + } + + for testi, test := range tests { + plan, err := Plan(test.Segments, test.Options) + + if err != test.ExpectErr { + testj, _ := json.Marshal(&test) + + t.Errorf("testi: %d, test: %s, got err: %v", testi, testj, err) + } + + if !reflect.DeepEqual(plan, test.ExpectPlan) { + testj, _ := json.Marshal(&test) + + planj, _ := json.Marshal(&plan) + + t.Errorf("testi: %d, test: %s, got plan: %s", + testi, testj, planj) + } + } +} + +// ---------------------------------------- + +func TestSort(t *testing.T) { + segs := makeLinearSegments(10) + + sort.Sort(byLiveSizeDescending(segs)) + + for i := 1; i < len(segs); i++ { + if segs[i].LiveSize() >= segs[i-1].LiveSize() { + t.Errorf("not descending") + } + } +} + +// ---------------------------------------- + +func TestCalcBudget(t *testing.T) { + tests := []struct { + totalSize int64 + firstTierSize int64 + o MergePlanOptions + expect int + }{ + {0, 0, MergePlanOptions{}, 0}, + {1, 0, MergePlanOptions{}, 1}, + {9, 0, MergePlanOptions{}, 9}, + { + 1, 1, + MergePlanOptions{ + MaxSegmentsPerTier: 1, + MaxSegmentSize: 1000, + TierGrowth: 2.0, + SegmentsPerMergeTask: 2, + FloorSegmentSize: 1, + }, + 1, + }, + { + 21, 1, + MergePlanOptions{ + MaxSegmentsPerTier: 1, + MaxSegmentSize: 1000, + TierGrowth: 2.0, + SegmentsPerMergeTask: 2, + FloorSegmentSize: 1, + }, + 5, + }, + { + 21, 1, + MergePlanOptions{ + MaxSegmentsPerTier: 2, + MaxSegmentSize: 1000, + TierGrowth: 2.0, + SegmentsPerMergeTask: 2, + FloorSegmentSize: 1, + }, + 7, + }, + { + 1000, 2000, DefaultMergePlanOptions, + 1, + }, + { + 5000, 2000, DefaultMergePlanOptions, + 3, + }, + { + 10000, 2000, DefaultMergePlanOptions, + 5, + }, + { + 30000, 2000, DefaultMergePlanOptions, + 11, + }, + { + 1000000, 2000, DefaultMergePlanOptions, + 24, + }, + { + 1000000000, 2000, DefaultMergePlanOptions, + 54, + }, + } + + for testi, test := range tests { + res := CalcBudget(test.totalSize, test.firstTierSize, &test.o) + if res != test.expect { + t.Errorf("testi: %d, test: %#v, res: %v", + testi, test, res) + } + } +} + +func TestCalcBudgetForSingleSegmentMergePolicy(t *testing.T) { + mpolicy := MergePlanOptions{ + MaxSegmentsPerTier: 1, + MaxSegmentSize: 1 << 30, // ~ 1 Billion + SegmentsPerMergeTask: 10, + FloorSegmentSize: 1 << 30, + } + + tests := []struct { + totalSize int64 + firstTierSize int64 + o MergePlanOptions + expect int + }{ + {0, mpolicy.RaiseToFloorSegmentSize(0), mpolicy, 0}, + {1, mpolicy.RaiseToFloorSegmentSize(1), mpolicy, 1}, + {9, mpolicy.RaiseToFloorSegmentSize(0), mpolicy, 1}, + {1, mpolicy.RaiseToFloorSegmentSize(1), mpolicy, 1}, + {21, mpolicy.RaiseToFloorSegmentSize(21), mpolicy, 1}, + {21, mpolicy.RaiseToFloorSegmentSize(21), mpolicy, 1}, + {1000, mpolicy.RaiseToFloorSegmentSize(2000), mpolicy, 1}, + {5000, mpolicy.RaiseToFloorSegmentSize(5000), mpolicy, 1}, + {10000, mpolicy.RaiseToFloorSegmentSize(10000), mpolicy, 1}, + {30000, mpolicy.RaiseToFloorSegmentSize(30000), mpolicy, 1}, + {1000000, mpolicy.RaiseToFloorSegmentSize(1000000), mpolicy, 1}, + {1000000000, 1 << 30, mpolicy, 1}, + {1013423541, 1 << 30, mpolicy, 1}, + {98765442, 1 << 30, mpolicy, 1}, + } + + for testi, test := range tests { + res := CalcBudget(test.totalSize, test.firstTierSize, &test.o) + if res != test.expect { + t.Errorf("testi: %d, test: %#v, res: %v", + testi, test, res) + } + } +} + +// ---------------------------------------- + +func TestInsert1SameSizedSegmentBetweenMerges(t *testing.T) { + o := &MergePlanOptions{ + MaxSegmentSize: 1000, + MaxSegmentsPerTier: 3, + TierGrowth: 3.0, + SegmentsPerMergeTask: 3, + } + + spec := testCyclesSpec{ + descrip: "i1sssbm", + verbose: os.Getenv("VERBOSE") == "i1sssbm" || os.Getenv("VERBOSE") == "y", + n: 200, + o: o, + beforePlan: func(spec *testCyclesSpec) { + spec.segments = append(spec.segments, &segment{ + MyId: spec.nextSegmentId, + MyFullSize: 1, + MyLiveSize: 1, + }) + spec.nextSegmentId++ + }, + } + + spec.runCycles(t) +} + +func TestInsertManySameSizedSegmentsBetweenMerges(t *testing.T) { + o := &MergePlanOptions{ + MaxSegmentSize: 1000, + MaxSegmentsPerTier: 3, + TierGrowth: 3.0, + SegmentsPerMergeTask: 3, + } + + spec := testCyclesSpec{ + descrip: "imsssbm", + verbose: os.Getenv("VERBOSE") == "imsssbm" || os.Getenv("VERBOSE") == "y", + n: 20, + o: o, + beforePlan: func(spec *testCyclesSpec) { + for i := 0; i < 10; i++ { + spec.segments = append(spec.segments, &segment{ + MyId: spec.nextSegmentId, + MyFullSize: 1, + MyLiveSize: 1, + }) + spec.nextSegmentId++ + } + }, + } + + spec.runCycles(t) +} + +func TestInsertManySameSizedSegmentsWithDeletionsBetweenMerges(t *testing.T) { + o := &MergePlanOptions{ + MaxSegmentSize: 1000, + MaxSegmentsPerTier: 3, + TierGrowth: 3.0, + SegmentsPerMergeTask: 3, + } + + spec := testCyclesSpec{ + descrip: "imssswdbm", + verbose: os.Getenv("VERBOSE") == "imssswdbm" || os.Getenv("VERBOSE") == "y", + n: 20, + o: o, + beforePlan: func(spec *testCyclesSpec) { + for i := 0; i < 10; i++ { + // Deletions are a shrinking of the live size. + for i, seg := range spec.segments { + if (spec.cycle+i)%5 == 0 { + s := seg.(*segment) + if s.MyLiveSize > 0 { + s.MyLiveSize -= 1 + } + } + } + + spec.segments = append(spec.segments, &segment{ + MyId: spec.nextSegmentId, + MyFullSize: 1, + MyLiveSize: 1, + }) + spec.nextSegmentId++ + } + }, + } + + spec.runCycles(t) +} + +func TestInsertManyDifferentSizedSegmentsBetweenMerges(t *testing.T) { + o := &MergePlanOptions{ + MaxSegmentSize: 1000, + MaxSegmentsPerTier: 3, + TierGrowth: 3.0, + SegmentsPerMergeTask: 3, + } + + spec := testCyclesSpec{ + descrip: "imdssbm", + verbose: os.Getenv("VERBOSE") == "imdssbm" || os.Getenv("VERBOSE") == "y", + n: 20, + o: o, + beforePlan: func(spec *testCyclesSpec) { + for i := 0; i < 10; i++ { + spec.segments = append(spec.segments, &segment{ + MyId: spec.nextSegmentId, + MyFullSize: int64(1 + (i % 5)), + MyLiveSize: int64(1 + (i % 5)), + }) + spec.nextSegmentId++ + } + }, + } + + spec.runCycles(t) +} + +func TestManySameSizedSegmentsWithDeletesBetweenMerges(t *testing.T) { + o := &MergePlanOptions{ + MaxSegmentSize: 1000, + MaxSegmentsPerTier: 3, + TierGrowth: 3.0, + SegmentsPerMergeTask: 3, + } + + var numPlansWithTasks int + + spec := testCyclesSpec{ + descrip: "mssswdbm", + verbose: os.Getenv("VERBOSE") == "mssswdbm" || os.Getenv("VERBOSE") == "y", + n: 20, + o: o, + beforePlan: func(spec *testCyclesSpec) { + // Deletions are a shrinking of the live size. + for i, seg := range spec.segments { + if (spec.cycle+i)%5 == 0 { + s := seg.(*segment) + if s.MyLiveSize > 0 { + s.MyLiveSize -= 1 + } + } + } + + for i := 0; i < 10; i++ { + spec.segments = append(spec.segments, &segment{ + MyId: spec.nextSegmentId, + MyFullSize: 1, + MyLiveSize: 1, + }) + spec.nextSegmentId++ + } + }, + afterPlan: func(spec *testCyclesSpec, plan *MergePlan) { + if plan != nil && len(plan.Tasks) > 0 { + numPlansWithTasks++ + } + }, + } + + spec.runCycles(t) + + if numPlansWithTasks <= 0 { + t.Errorf("expected some plans with tasks") + } +} + +func TestValidateMergePlannerOptions(t *testing.T) { + o := &MergePlanOptions{ + MaxSegmentSize: 1 << 32, + MaxSegmentsPerTier: 3, + TierGrowth: 3.0, + SegmentsPerMergeTask: 3, + } + err := ValidateMergePlannerOptions(o) + if err != ErrMaxSegmentSizeTooLarge { + t.Error("Validation expected to fail as the MaxSegmentSize exceeds limit") + } +} + +func TestPlanMaxSegmentSizeLimit(t *testing.T) { + o := &MergePlanOptions{ + MaxSegmentSize: 20, + MaxSegmentsPerTier: 5, + TierGrowth: 3.0, + SegmentsPerMergeTask: 5, + FloorSegmentSize: 5, + } + segments := makeLinearSegments(20) + + s := rand.NewSource(time.Now().UnixNano()) + r := rand.New(s) + + max := 20 + min := 5 + randomInRange := func() int64 { + return int64(r.Intn(max-min) + min) + } + for i := 1; i < 20; i++ { + o.MaxSegmentSize = randomInRange() + plans, err := Plan(segments, o) + if err != nil { + t.Errorf("Plan failed, err: %v", err) + } + if len(plans.Tasks) == 0 { + t.Errorf("expected some plans with tasks") + } + + for _, task := range plans.Tasks { + var totalLiveSize int64 + for _, segs := range task.Segments { + totalLiveSize += segs.LiveSize() + } + if totalLiveSize >= o.MaxSegmentSize { + t.Errorf("merged segments size: %d exceeding the MaxSegmentSize"+ + "limit: %d", totalLiveSize, o.MaxSegmentSize) + } + } + } +} + +// ---------------------------------------- + +type testCyclesSpec struct { + descrip string + verbose bool + + n int // Number of cycles to run. + o *MergePlanOptions + + converge bool + + beforePlan func(*testCyclesSpec) + afterPlan func(*testCyclesSpec, *MergePlan) + + cycle int + segments []Segment + nextSegmentId uint64 +} + +func (spec *testCyclesSpec) runCycles(t *testing.T) { + numPlansWithTasks := 0 + + for spec.cycle < spec.n { + if spec.verbose { + emit(spec.descrip, spec.cycle, 0, spec.segments, nil) + } + + if spec.beforePlan != nil { + spec.beforePlan(spec) + } + + if spec.verbose { + emit(spec.descrip, spec.cycle, 1, spec.segments, nil) + } + + plan, err := Plan(spec.segments, spec.o) + if err != nil { + t.Fatalf("expected no err, got: %v", err) + } + + if spec.afterPlan != nil { + spec.afterPlan(spec, plan) + } + + if spec.verbose { + emit(spec.descrip, spec.cycle, 2, spec.segments, plan) + } + + if plan == nil || len(plan.Tasks) == 0 { + if spec.converge { + return + } + spec.cycle++ + continue + } + + numPlansWithTasks++ + before := len(spec.segments) + + var beforeFullSize int64 + for _, s := range spec.segments { + beforeFullSize += s.FullSize() + } + + for _, task := range plan.Tasks { + spec.segments = removeSegments(spec.segments, task.Segments) + + var totLiveSize int64 + var totLiveFileSize int64 + var hasVector bool + for _, segment := range task.Segments { + totLiveSize += segment.LiveSize() + totLiveFileSize += segment.LiveFileSize() + if segment.HasVector() { + hasVector = true + } + } + + if totLiveSize > 0 { + spec.segments = append(spec.segments, &segment{ + MyId: spec.nextSegmentId, + MyFullSize: totLiveSize, + MyLiveSize: totLiveSize, + MyFileSize: totLiveFileSize, + MyHasVector: hasVector, + }) + spec.nextSegmentId++ + } + } + + after := len(spec.segments) + + var afterFullSize int64 + for _, s := range spec.segments { + afterFullSize += s.FullSize() + } + + if after > before { + t.Errorf("cycle %d: plan produced %d task(s) but increased segment count (%d segs -> %d segs)", + spec.cycle, len(plan.Tasks), before, after) + } + + if after == before && afterFullSize >= beforeFullSize { + t.Errorf("cycle %d: plan produced %d task(s) but made no progress "+ + "(%d segs / %d bytes -> %d segs / %d bytes)", + spec.cycle, len(plan.Tasks), before, beforeFullSize, + len(spec.segments), afterFullSize) + } + + spec.cycle++ + } + + if spec.converge { + t.Errorf("index did not converge within %d cycles (%d segments remain); "+ + "the planner keeps producing tasks, likely an infinite loop", + spec.n, len(spec.segments)) + } + + if numPlansWithTasks <= 0 { + t.Errorf("expected some plans with tasks") + } +} + +func emit(descrip string, cycle int, step int, segments []Segment, plan *MergePlan) { + if os.Getenv("VERBOSE") == "" { + return + } + + suffix := "" + if plan != nil && len(plan.Tasks) > 0 { + suffix = "hasPlan" + } + + fmt.Printf("%s %d.%d ---------- %s\n", descrip, cycle, step, suffix) + fmt.Printf("%s\n", ToBarChart(descrip, 100, segments, plan)) +} + +// ---------------------------------------- + +func TestPlanMaxSegmentFileSize(t *testing.T) { + tests := []struct { + segments []Segment + o *MergePlanOptions + expectedFinalSegments []Segment + }{ + { + []Segment{ + &segment{ // ineligible + MyId: 1, + MyFullSize: 4000, + MyLiveSize: 3900, + + MyHasVector: true, + MyFileSize: 4000 * 1000 * 4, // > 2MB + }, + &segment{ // ineligible + MyId: 2, + MyFullSize: 6000, + MyLiveSize: 5500, // > 5000 + + MyHasVector: true, + MyFileSize: 6000 * 1000 * 4, // > 2MB + }, + &segment{ // eligible + MyId: 3, + MyFullSize: 500, + MyLiveSize: 490, + + MyHasVector: true, + MyFileSize: 500 * 1000 * 4, + }, + &segment{ // eligible + MyId: 4, + MyFullSize: 500, + MyLiveSize: 480, + + MyHasVector: true, + MyFileSize: 500 * 1000 * 4, + }, + &segment{ // eligible + MyId: 5, + MyFullSize: 500, + MyLiveSize: 300, + + MyHasVector: true, + MyFileSize: 500 * 1000 * 4, + }, + &segment{ // eligible + MyId: 6, + MyFullSize: 500, + MyLiveSize: 400, + + MyHasVector: true, + MyFileSize: 500 * 1000 * 4, + }, + }, + &MergePlanOptions{ + MaxSegmentSize: 5000, // number of documents + // considering vector dimension as 1000 + // vectorBytes = 5000 * 1000 * 4 = 20MB, which is too large + // So, let's set the fileSize limit to 4MB + MaxSegmentFileSize: 4000000, // 4MB + MaxSegmentsPerTier: 1, + SegmentsPerMergeTask: 2, + TierGrowth: 2.0, + ReclaimDeletesWeight: 2, + FloorSegmentSize: 1, + }, + []Segment{ + &segment{ + MyId: 1, + MyFullSize: 4000, + MyLiveSize: 3900, + + MyHasVector: true, + MyFileSize: 4000 * 1000 * 4, + }, + &segment{ + MyId: 2, + MyFullSize: 6000, + MyLiveSize: 5500, + + MyHasVector: true, + MyFileSize: 6000 * 1000 * 4, + }, + &segment{ + MyId: 3, + MyFullSize: 500, + MyLiveSize: 490, + + MyHasVector: true, + MyFileSize: 500 * 1000 * 4, + }, + &segment{ + MyId: 4, + MyFullSize: 500, + MyLiveSize: 480, + + MyHasVector: true, + MyFileSize: 500 * 1000 * 4, + }, + &segment{ + MyId: 7, + MyFullSize: 700, + MyLiveSize: 700, + + MyHasVector: true, + MyFileSize: 700 * 1000 * 4, + }, + }, + }, + } + + for testi, test := range tests { + t.Run(fmt.Sprintf("Test-%d", testi), func(t *testing.T) { + var nextID uint64 + for _, seg := range test.segments { + if seg.Id() >= nextID { + nextID = seg.Id() + 1 + } + } + + spec := testCyclesSpec{ + descrip: fmt.Sprintf("pmsfs-%d", testi), + verbose: os.Getenv("VERBOSE") == "pmsfs" || os.Getenv("VERBOSE") == "y", + n: 10, + o: test.o, + converge: true, + segments: slices.Clone(test.segments), + nextSegmentId: nextID, + } + + spec.runCycles(t) + + if !reflect.DeepEqual(spec.segments, test.expectedFinalSegments) { + t.Fatalf("final layout mismatch, got: %+v", spec.segments) + } + }) + } +} + +func TestSingleSegmentTaskMergePlan(t *testing.T) { + o := DefaultMergePlanOptions + o.FloorSegmentFileSize = 209715200 + + spec := testCyclesSpec{ + descrip: "stmp", + verbose: os.Getenv("VERBOSE") == "stmp" || os.Getenv("VERBOSE") == "y", + n: 10, + o: &o, + converge: true, + segments: []Segment{ + &segment{ + MyId: 2, + MyFullSize: 78059, + MyLiveSize: 78059, + MyFileSize: 129475914, + }, + &segment{ + MyId: 1, + MyFullSize: 3959, + MyLiveSize: 3959, + MyFileSize: 24805725, + }, + }, + nextSegmentId: 3, + } + + spec.runCycles(t) + + expectedSegments := []Segment{ + &segment{ + MyId: 3, + MyFullSize: 78059 + 3959, + MyLiveSize: 78059 + 3959, + MyFileSize: 129475914 + 24805725, + }, + } + + if !reflect.DeepEqual(spec.segments, expectedSegments) { + t.Fatalf("final layout mismatch, got: %+v", spec.segments) + } +} + +func TestPlanLiveFileSizeEligibility(t *testing.T) { + o := &MergePlanOptions{ + MaxSegmentFileSize: 4 * 1000 * 1000, + MaxSegmentSize: 100 * 1000, + SegmentsPerMergeTask: 2, + TierGrowth: 2.0, + FloorSegmentSize: 2000, + } + + segments := []Segment{ + &segment{ + MyId: 1, + MyFullSize: 1000, + MyLiveSize: 500, + MyHasVector: true, + MyFileSize: 3 * 1000 * 1000, + }, + &segment{ + MyId: 2, + MyFullSize: 1000, + MyLiveSize: 500, + MyHasVector: true, + MyFileSize: 3 * 1000 * 1000, + }, + &segment{ + MyId: 3, + MyFullSize: 1000, + MyLiveSize: 1000, + MyHasVector: true, + MyFileSize: 3 * 1000 * 1000, + }, + } + + spec := testCyclesSpec{ + descrip: "plfse", + verbose: os.Getenv("VERBOSE") == "plfse" || os.Getenv("VERBOSE") == "y", + n: 10, + o: o, + converge: true, + segments: slices.Clone(segments), + nextSegmentId: 4, + } + + spec.runCycles(t) + + expectedSegments := []Segment{ + &segment{ + MyId: 3, + MyFullSize: 1000, + MyLiveSize: 1000, + MyHasVector: true, + MyFileSize: 3 * 1000 * 1000, + }, + &segment{ + MyId: 4, + MyFullSize: 1000, + MyLiveSize: 1000, + MyHasVector: true, + MyFileSize: 3 * 1000 * 1000, + }, + } + + if !reflect.DeepEqual(spec.segments, expectedSegments) { + t.Fatalf("final layout mismatch, got: %+v", spec.segments) + } +} + +func TestPlanLiveFileSizeBudget(t *testing.T) { + o := &MergePlanOptions{ + MaxSegmentsPerTier: 1, + SegmentsPerMergeTask: 2, + TierGrowth: 2.0, + FloorSegmentSize: 1, + FloorSegmentFileSize: 10 * 1000 * 1000, + MaxSegmentSize: 100 * 1000, + MaxSegmentFileSize: 4 * 1000 * 1000 * 1000, + } + + segments := []Segment{ + &segment{ + MyId: 1, + MyFullSize: 1000, + MyLiveSize: 100, + MyFileSize: 10 * 1000 * 1000, + }, + &segment{ + MyId: 2, + MyFullSize: 1000, + MyLiveSize: 100, + MyFileSize: 10 * 1000 * 1000, + }, + } + + spec := testCyclesSpec{ + descrip: "plfsb", + verbose: os.Getenv("VERBOSE") == "plfsb" || os.Getenv("VERBOSE") == "y", + n: 10, + o: o, + converge: true, + segments: slices.Clone(segments), + nextSegmentId: 3, + } + + spec.runCycles(t) + + expectedSegments := []Segment{ + &segment{ + MyId: 3, + MyFullSize: 200, + MyLiveSize: 200, + MyFileSize: 2 * 1000 * 1000, + }, + } + + if !reflect.DeepEqual(spec.segments, expectedSegments) { + t.Fatalf("final layout mismatch, got: %+v", spec.segments) + } +} + +func TestEmptySegmentsDoNotForceOverMerge(t *testing.T) { + o := &MergePlanOptions{ + MaxSegmentsPerTier: 10, + MaxSegmentSize: 1000000, + SegmentsPerMergeTask: 10, + FloorSegmentSize: 1, + CalcBudget: func(totalSize, firstTierSize int64, o *MergePlanOptions) int { + return 2 + }, + } + + spec := testCyclesSpec{ + descrip: "esdnfom", + verbose: os.Getenv("VERBOSE") == "esdnfom" || os.Getenv("VERBOSE") == "y", + n: 10, + o: o, + converge: true, + segments: []Segment{ + &segment{MyId: 1, MyFullSize: 100, MyLiveSize: 100}, + &segment{MyId: 2, MyFullSize: 100, MyLiveSize: 100}, + &segment{MyId: 3, MyFullSize: 100, MyLiveSize: 0}, + }, + nextSegmentId: 4, + } + + spec.runCycles(t) + + expectedSegments := []Segment{ + &segment{MyId: 1, MyFullSize: 100, MyLiveSize: 100}, + &segment{MyId: 2, MyFullSize: 100, MyLiveSize: 100}, + } + + if !reflect.DeepEqual(spec.segments, expectedSegments) { + t.Fatalf("final layout mismatch, got: %+v", spec.segments) + } +} + +func TestOnlyEmptySegments(t *testing.T) { + o := &MergePlanOptions{ + MaxSegmentsPerTier: 10, + MaxSegmentSize: 1000000, + SegmentsPerMergeTask: 10, + FloorSegmentSize: 1, + } + + spec := testCyclesSpec{ + descrip: "oes", + verbose: os.Getenv("VERBOSE") == "oes" || os.Getenv("VERBOSE") == "y", + n: 10, + o: o, + converge: true, + segments: []Segment{ + &segment{MyId: 1, MyFullSize: 100, MyLiveSize: 0}, + &segment{MyId: 2, MyFullSize: 100, MyLiveSize: 0}, + &segment{MyId: 3, MyFullSize: 100, MyLiveSize: 0}, + }, + nextSegmentId: 4, + } + + spec.runCycles(t) + + expectedSegments := []Segment{} + + if !reflect.DeepEqual(spec.segments, expectedSegments) { + t.Fatalf("final layout mismatch, got: %+v", spec.segments) + } +} + +func TestEligibleAndFilteredSegments(t *testing.T) { + o := &MergePlanOptions{ + MaxSegmentsPerTier: 10, + MaxSegmentSize: 1000000, + MaxSegmentFileSize: 1000, + SegmentsPerMergeTask: 10, + FloorSegmentSize: 1, + FloorSegmentFileSize: 2000, + } + + spec := testCyclesSpec{ + descrip: "eafs", + verbose: os.Getenv("VERBOSE") == "eafs" || os.Getenv("VERBOSE") == "y", + n: 10, + o: o, + converge: true, + segments: []Segment{ + &segment{MyId: 1, MyFullSize: 100, MyLiveSize: 100, MyFileSize: 400, MyHasVector: true}, + &segment{MyId: 2, MyFullSize: 100, MyLiveSize: 100, MyFileSize: 400, MyHasVector: true}, + &segment{MyId: 3, MyFullSize: 100, MyLiveSize: 100, MyFileSize: 400, MyHasVector: true}, + &segment{MyId: 4, MyFullSize: 100, MyLiveSize: 100, MyFileSize: 400, MyHasVector: true}, + }, + nextSegmentId: 5, + } + + spec.runCycles(t) + + expectedSegments := []Segment{ + &segment{MyId: 5, MyFullSize: 200, MyLiveSize: 200, MyFileSize: 800, MyHasVector: true}, + &segment{MyId: 6, MyFullSize: 200, MyLiveSize: 200, MyFileSize: 800, MyHasVector: true}, + } + + if !reflect.DeepEqual(spec.segments, expectedSegments) { + t.Fatalf("final layout mismatch, got: %+v", spec.segments) + } +} + +func TestPlanWithNoBestRoster(t *testing.T) { + o := DefaultMergePlanOptions + o.SegmentsPerMergeTask = 0 + + spec := testCyclesSpec{ + descrip: "pwnbr", + verbose: os.Getenv("VERBOSE") == "pwnbr" || os.Getenv("VERBOSE") == "y", + n: 10, + o: &o, + converge: true, + segments: []Segment{ + &segment{MyId: 1, MyFullSize: 100, MyLiveSize: 0, MyFileSize: 400}, + &segment{MyId: 2, MyFullSize: 100, MyLiveSize: 100, MyFileSize: 400}, + &segment{MyId: 3, MyFullSize: 100, MyLiveSize: 50, MyFileSize: 4000}, + }, + nextSegmentId: 5, + } + + spec.runCycles(t) + + expectedSegments := []Segment{ + &segment{MyId: 2, MyFullSize: 100, MyLiveSize: 100, MyFileSize: 400}, + &segment{MyId: 3, MyFullSize: 100, MyLiveSize: 50, MyFileSize: 4000}, + } + + if !reflect.DeepEqual(spec.segments, expectedSegments) { + t.Fatalf("final layout mismatch, got: %+v", spec.segments) + } +} diff --git a/index/scorch/mergeplan/sort.go b/index/scorch/mergeplan/sort.go new file mode 100644 index 0000000..d044b8d --- /dev/null +++ b/index/scorch/mergeplan/sort.go @@ -0,0 +1,28 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mergeplan + +type byLiveSizeDescending []Segment + +func (a byLiveSizeDescending) Len() int { return len(a) } + +func (a byLiveSizeDescending) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +func (a byLiveSizeDescending) Less(i, j int) bool { + if a[i].LiveSize() != a[j].LiveSize() { + return a[i].LiveSize() > a[j].LiveSize() + } + return a[i].Id() < a[j].Id() +} diff --git a/index/scorch/optimize.go b/index/scorch/optimize.go new file mode 100644 index 0000000..7ad28a6 --- /dev/null +++ b/index/scorch/optimize.go @@ -0,0 +1,391 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + "sync/atomic" + + "github.com/RoaringBitmap/roaring/v2" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +var OptimizeConjunction = true +var OptimizeConjunctionUnadorned = true +var OptimizeDisjunctionUnadorned = true + +func (s *IndexSnapshotTermFieldReader) Optimize(kind string, + octx index.OptimizableContext) (index.OptimizableContext, error) { + if OptimizeConjunction && kind == "conjunction" { + return s.optimizeConjunction(octx) + } + + if OptimizeConjunctionUnadorned && kind == "conjunction:unadorned" { + return s.optimizeConjunctionUnadorned(octx) + } + + if OptimizeDisjunctionUnadorned && kind == "disjunction:unadorned" { + return s.optimizeDisjunctionUnadorned(octx) + } + + return nil, nil +} + +var OptimizeDisjunctionUnadornedMinChildCardinality = uint64(256) + +// ---------------------------------------------------------------- + +func (s *IndexSnapshotTermFieldReader) optimizeConjunction( + octx index.OptimizableContext) (index.OptimizableContext, error) { + if octx == nil { + octx = &OptimizeTFRConjunction{snapshot: s.snapshot} + } + + o, ok := octx.(*OptimizeTFRConjunction) + if !ok { + return octx, nil + } + + if o.snapshot != s.snapshot { + return nil, fmt.Errorf("tried to optimize conjunction across different snapshots") + } + + o.tfrs = append(o.tfrs, s) + + return o, nil +} + +type OptimizeTFRConjunction struct { + snapshot *IndexSnapshot + + tfrs []*IndexSnapshotTermFieldReader +} + +func (o *OptimizeTFRConjunction) Finish() (index.Optimized, error) { + if len(o.tfrs) <= 1 { + return nil, nil + } + + for i := range o.snapshot.segment { + itr0, ok := o.tfrs[0].iterators[i].(segment.OptimizablePostingsIterator) + if !ok || itr0.ActualBitmap() == nil { + continue + } + + itr1, ok := o.tfrs[1].iterators[i].(segment.OptimizablePostingsIterator) + if !ok || itr1.ActualBitmap() == nil { + continue + } + + bm := roaring.And(itr0.ActualBitmap(), itr1.ActualBitmap()) + + for _, tfr := range o.tfrs[2:] { + itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator) + if !ok || itr.ActualBitmap() == nil { + continue + } + + bm.And(itr.ActualBitmap()) + } + + // in this conjunction optimization, the postings iterators + // will all share the same AND'ed together actual bitmap. The + // regular conjunction searcher machinery will still be used, + // but the underlying bitmap will be smaller. + for _, tfr := range o.tfrs { + itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator) + if ok && itr.ActualBitmap() != nil { + itr.ReplaceActual(bm) + } + } + } + + return nil, nil +} + +// ---------------------------------------------------------------- + +// An "unadorned" conjunction optimization is appropriate when +// additional or subsidiary information like freq-norm's and +// term-vectors are not required, and instead only the internal-id's +// are needed. +func (s *IndexSnapshotTermFieldReader) optimizeConjunctionUnadorned( + octx index.OptimizableContext) (index.OptimizableContext, error) { + if octx == nil { + octx = &OptimizeTFRConjunctionUnadorned{snapshot: s.snapshot} + } + + o, ok := octx.(*OptimizeTFRConjunctionUnadorned) + if !ok { + return nil, nil + } + + if o.snapshot != s.snapshot { + return nil, fmt.Errorf("tried to optimize unadorned conjunction across different snapshots") + } + + o.tfrs = append(o.tfrs, s) + + return o, nil +} + +type OptimizeTFRConjunctionUnadorned struct { + snapshot *IndexSnapshot + + tfrs []*IndexSnapshotTermFieldReader +} + +var OptimizeTFRConjunctionUnadornedTerm = []byte("") +var OptimizeTFRConjunctionUnadornedField = "*" + +// Finish of an unadorned conjunction optimization will compute a +// termFieldReader with an "actual" bitmap that represents the +// constituent bitmaps AND'ed together. This termFieldReader cannot +// provide any freq-norm or termVector associated information. +func (o *OptimizeTFRConjunctionUnadorned) Finish() (rv index.Optimized, err error) { + if len(o.tfrs) <= 1 { + return nil, nil + } + + // We use an artificial term and field because the optimized + // termFieldReader can represent multiple terms and fields. + oTFR := o.snapshot.unadornedTermFieldReader( + OptimizeTFRConjunctionUnadornedTerm, OptimizeTFRConjunctionUnadornedField) + + var actualBMs []*roaring.Bitmap // Collected from regular posting lists. + +OUTER: + for i := range o.snapshot.segment { + actualBMs = actualBMs[:0] + + var docNum1HitLast uint64 + var docNum1HitLastOk bool + + for _, tfr := range o.tfrs { + if _, ok := tfr.iterators[i].(*emptyPostingsIterator); ok { + // An empty postings iterator means the entire AND is empty. + oTFR.iterators[i] = anEmptyPostingsIterator + continue OUTER + } + + itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator) + if !ok { + // We only optimize postings iterators that support this operation. + return nil, nil + } + + // If the postings iterator is "1-hit" optimized, then we + // can perform several optimizations up-front here. + docNum1Hit, ok := itr.DocNum1Hit() + if ok { + if docNum1HitLastOk && docNum1HitLast != docNum1Hit { + // The docNum1Hit doesn't match the previous + // docNum1HitLast, so the entire AND is empty. + oTFR.iterators[i] = anEmptyPostingsIterator + continue OUTER + } + + docNum1HitLast = docNum1Hit + docNum1HitLastOk = true + + continue + } + + if itr.ActualBitmap() == nil { + // An empty actual bitmap means the entire AND is empty. + oTFR.iterators[i] = anEmptyPostingsIterator + continue OUTER + } + + // Collect the actual bitmap for more processing later. + actualBMs = append(actualBMs, itr.ActualBitmap()) + } + + if docNum1HitLastOk { + // We reach here if all the 1-hit optimized posting + // iterators had the same 1-hit docNum, so we can check if + // our collected actual bitmaps also have that docNum. + for _, bm := range actualBMs { + if !bm.Contains(uint32(docNum1HitLast)) { + // The docNum1Hit isn't in one of our actual + // bitmaps, so the entire AND is empty. + oTFR.iterators[i] = anEmptyPostingsIterator + continue OUTER + } + } + + // The actual bitmaps and docNum1Hits all contain or have + // the same 1-hit docNum, so that's our AND'ed result. + oTFR.iterators[i] = newUnadornedPostingsIteratorFrom1Hit(docNum1HitLast) + + continue OUTER + } + + if len(actualBMs) == 0 { + // If we've collected no actual bitmaps at this point, + // then the entire AND is empty. + oTFR.iterators[i] = anEmptyPostingsIterator + continue OUTER + } + + if len(actualBMs) == 1 { + // If we've only 1 actual bitmap, then that's our result. + oTFR.iterators[i] = newUnadornedPostingsIteratorFromBitmap(actualBMs[0]) + + continue OUTER + } + + // Else, AND together our collected bitmaps as our result. + bm := roaring.And(actualBMs[0], actualBMs[1]) + + for _, actualBM := range actualBMs[2:] { + bm.And(actualBM) + } + + oTFR.iterators[i] = newUnadornedPostingsIteratorFromBitmap(bm) + } + + atomic.AddUint64(&o.snapshot.parent.stats.TotTermSearchersStarted, uint64(1)) + return oTFR, nil +} + +// ---------------------------------------------------------------- + +// An "unadorned" disjunction optimization is appropriate when +// additional or subsidiary information like freq-norm's and +// term-vectors are not required, and instead only the internal-id's +// are needed. +func (s *IndexSnapshotTermFieldReader) optimizeDisjunctionUnadorned( + octx index.OptimizableContext) (index.OptimizableContext, error) { + if octx == nil { + octx = &OptimizeTFRDisjunctionUnadorned{ + snapshot: s.snapshot, + } + } + + o, ok := octx.(*OptimizeTFRDisjunctionUnadorned) + if !ok { + return nil, nil + } + + if o.snapshot != s.snapshot { + return nil, fmt.Errorf("tried to optimize unadorned disjunction across different snapshots") + } + + o.tfrs = append(o.tfrs, s) + + return o, nil +} + +type OptimizeTFRDisjunctionUnadorned struct { + snapshot *IndexSnapshot + + tfrs []*IndexSnapshotTermFieldReader +} + +var OptimizeTFRDisjunctionUnadornedTerm = []byte("") +var OptimizeTFRDisjunctionUnadornedField = "*" + +// Finish of an unadorned disjunction optimization will compute a +// termFieldReader with an "actual" bitmap that represents the +// constituent bitmaps OR'ed together. This termFieldReader cannot +// provide any freq-norm or termVector associated information. +func (o *OptimizeTFRDisjunctionUnadorned) Finish() (rv index.Optimized, err error) { + if len(o.tfrs) <= 1 { + return nil, nil + } + + // We use an artificial term and field because the optimized + // termFieldReader can represent multiple terms and fields. + oTFR := o.snapshot.unadornedTermFieldReader( + OptimizeTFRDisjunctionUnadornedTerm, OptimizeTFRDisjunctionUnadornedField) + + var docNums []uint32 // Collected docNum's from 1-hit posting lists. + var actualBMs []*roaring.Bitmap // Collected from regular posting lists. + + for i := range o.snapshot.segment { + docNums = docNums[:0] + actualBMs = actualBMs[:0] + + for _, tfr := range o.tfrs { + itr, ok := tfr.iterators[i].(segment.OptimizablePostingsIterator) + if !ok { + return nil, nil + } + + docNum, ok := itr.DocNum1Hit() + if ok { + docNums = append(docNums, uint32(docNum)) + continue + } + + if itr.ActualBitmap() != nil { + actualBMs = append(actualBMs, itr.ActualBitmap()) + } + } + + var bm *roaring.Bitmap + if len(actualBMs) > 2 { + bm = roaring.HeapOr(actualBMs...) + } else if len(actualBMs) == 2 { + bm = roaring.Or(actualBMs[0], actualBMs[1]) + } else if len(actualBMs) == 1 { + bm = actualBMs[0].Clone() + } else { + if len(docNums) == 0 { + // no hits, reuse the zero-alloc empty sentinel + oTFR.iterators[i] = anEmptyPostingsIterator + continue + } else if len(docNums) == 1 { + // 1-hit optimized + oTFR.iterators[i] = newUnadornedPostingsIteratorFrom1Hit(uint64(docNums[0])) + continue + } else { + bm = roaring.New() + } + } + + bm.AddMany(docNums) + + oTFR.iterators[i] = newUnadornedPostingsIteratorFromBitmap(bm) + } + + atomic.AddUint64(&o.snapshot.parent.stats.TotTermSearchersStarted, uint64(1)) + return oTFR, nil +} + +// ---------------------------------------------------------------- + +func (i *IndexSnapshot) unadornedTermFieldReader( + term []byte, field string) *IndexSnapshotTermFieldReader { + // This IndexSnapshotTermFieldReader will not be recycled, more + // conversation here: https://github.com/blevesearch/bleve/pull/1438 + return &IndexSnapshotTermFieldReader{ + term: term, + field: field, + snapshot: i, + iterators: make([]segment.PostingsIterator, len(i.segment)), + segmentOffset: 0, + includeFreq: false, + includeNorm: false, + includeTermVectors: false, + recycle: false, + // signal downstream that this is a special unadorned termFieldReader + unadorned: true, + // unadorned TFRs do not require bytes read tracking + updateBytesRead: false, + } +} diff --git a/index/scorch/optimize_knn.go b/index/scorch/optimize_knn.go new file mode 100644 index 0000000..9d2665b --- /dev/null +++ b/index/scorch/optimize_knn.go @@ -0,0 +1,207 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package scorch + +import ( + "context" + "fmt" + "sync" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" + segment_api "github.com/blevesearch/scorch_segment_api/v2" +) + +type OptimizeVR struct { + ctx context.Context + snapshot *IndexSnapshot + totalCost uint64 + // maps field to vector readers + vrs map[string][]*IndexSnapshotVectorReader +} + +func (o *OptimizeVR) invokeSearcherEndCallback() { + if o.ctx != nil { + if cb := o.ctx.Value(search.SearcherEndCallbackKey); cb != nil { + if cbF, ok := cb.(search.SearcherEndCallbackFn); ok { + if o.totalCost > 0 { + // notify the callback that the searcher creation etc. is finished + // and report back the total cost for it to track and take actions + // appropriately. + _ = cbF(o.totalCost) + } + } + } + } +} + +// search runs the configured kNN searches for every vector +// reader against a single segment, populating the per-segment postings and +// iterators on each reader. +func (o *OptimizeVR) search(segID int) error { + snapshot := o.snapshot.segment[segID] + vecSeg, ok := snapshot.segment.(segment_api.VectorSegment) + if !ok { + return nil + } + meta := snapshot.cachedMeta + // for each field, get the vector index --> invoke the zap func. + // for each VR, populate postings list and iterators + // by passing the obtained vector index and getting similar vectors. + for field, vrs := range o.vrs { + // Early exit if the field is supposed to be completely deleted or + // if it's index data has been deleted + if info, ok := o.snapshot.updatedFields[field]; ok && (info.Deleted || info.Index) { + continue + } + vecIndex, err := vecSeg.InterpretVectorIndex(field, snapshot.deleted) + if err != nil { + return err + } + // update the vector index size as a meta value in the segment snapshot + // if not already present + if !meta.contains(field) { + meta.store(field, vecIndex.Size()) + } + for _, vr := range vrs { + var pl segment_api.VecPostingsList + var err error + // for each VR, populate postings list and iterators + // by passing the obtained vector index and getting similar vectors. + + // check if the vector reader is configured to use a pre-filter + // to filter out ineligible documents before performing + // kNN search. + if vr.eligibleSelector != nil { + pl, err = vecIndex.SearchWithFilter( + vr.vector, + vr.k, + vr.eligibleSelector.SegmentEligibleDocuments(segID), + vr.searchParams, + ) + } else { + pl, err = vecIndex.Search( + vr.vector, + vr.k, + vr.searchParams, + ) + } + if err != nil { + vecIndex.Close() + return err + } + // postings and iterators are already alloc'ed when + // IndexSnapshotVectorReader is created, here we are just + // populating them with the obtained postings list. + vr.postings[segID] = pl + vr.iterators[segID] = pl.Iterator(vr.iterators[segID]) + } + go vecIndex.Close() + } + return nil +} + +func (o *OptimizeVR) Finish() error { + defer o.invokeSearcherEndCallback() + numSegments := len(o.snapshot.segment) + + var wg sync.WaitGroup + wg.Add(numSegments) + var errM sync.Mutex + var searchErr error + // launch goroutines to search the vector index for each segment + for i := 0; i < numSegments; i++ { + go func(segID int) { + defer wg.Done() + if err := o.search(segID); err != nil { + errM.Lock() + searchErr = err + errM.Unlock() + } + }(i) + } + // wait until all the launched goroutines finish and collect errors if any + wg.Wait() + return searchErr +} + +func (s *IndexSnapshotVectorReader) VectorOptimize(ctx context.Context, + octx index.VectorOptimizableContext, +) (index.VectorOptimizableContext, error) { + if s.snapshot.parent.segPlugin.Version() < VectorSearchSupportedSegmentVersion { + return nil, fmt.Errorf("vector search not supported for this index, "+ + "index's segment version %v, supported segment version for vector search %v", + s.snapshot.parent.segPlugin.Version(), VectorSearchSupportedSegmentVersion) + } + + if octx == nil { + octx = &OptimizeVR{ + snapshot: s.snapshot, + vrs: make(map[string][]*IndexSnapshotVectorReader), + } + } + + o, ok := octx.(*OptimizeVR) + if !ok { + return octx, nil + } + o.ctx = ctx + + if o.snapshot != s.snapshot { + o.invokeSearcherEndCallback() + return nil, fmt.Errorf("tried to optimize KNN across different snapshots") + } + + // for every searcher creation, consult the segment snapshot to see + // what's the vector index size and since you're anyways going + // to use this vector index to perform the search etc. as part of the Finish() + // perform a check as to whether we allow the searcher creation (the downstream) + // Finish() logic to even occur or not. + var sumVectorIndexSize uint64 + for _, seg := range o.snapshot.segment { + if vecIndexSize, ok := seg.cachedMeta.load(s.field); ok { + sumVectorIndexSize += vecIndexSize.(uint64) + } + } + + if o.ctx != nil { + if cb := o.ctx.Value(search.SearcherStartCallbackKey); cb != nil { + if cbF, ok := cb.(search.SearcherStartCallbackFn); ok { + err := cbF(sumVectorIndexSize) + if err != nil { + // it's important to invoke the end callback at this point since + // if the earlier searchers of this optimize struct were successful + // the cost corresponding to it would be incremented and if the + // current searcher fails the check then we end up erroring out + // the overall optimized searcher creation, the cost needs to be + // handled appropriately. + o.invokeSearcherEndCallback() + return nil, err + } + } + } + } + + // total cost is essentially the sum of the vector indexes' size across all the + // searchers - all of them end up reading and maintaining a vector index. + // misacconting this value would end up calling the "end" callback with a value + // not equal to the value passed to "start" callback. + o.totalCost += sumVectorIndexSize + o.vrs[s.field] = append(o.vrs[s.field], s) + return o, nil +} diff --git a/index/scorch/persister.go b/index/scorch/persister.go new file mode 100644 index 0000000..182a701 --- /dev/null +++ b/index/scorch/persister.go @@ -0,0 +1,1681 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "log" + "math" + "os" + "path/filepath" + "slices" + "sort" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/RoaringBitmap/roaring/v2" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + bolt "go.etcd.io/bbolt" +) + +const persister = "persister" + +var ( + // DefaultPersisterNapTimeMSec is kept to zero as this helps in direct + // persistence of segments with the default safe batch option. + // If the default safe batch option results in high number of + // files on disk, then users may initialise this configuration parameter + // with higher values so that the persister will nap a bit within it's + // work loop to favour better in-memory merging of segments to result + // in fewer segment files on disk. But that may come with an indexing + // performance overhead. + // Unsafe batch users are advised to override this to higher value + // for better performance especially with high data density. + DefaultPersisterNapTimeMSec int = 0 // ms + + // DefaultPersisterNapUnderNumFiles helps in controlling the pace of + // persister. At times of a slow merger progress with heavy file merging + // operations, its better to pace down the persister for letting the merger + // to catch up within a range defined by this parameter. + // Fewer files on disk (as per the merge plan) would result in keeping the + // file handle usage under limit, faster disk merger and a healthier index. + // Its been observed that such a loosely sync'ed introducer-persister-merger + // trio results in better overall performance. + DefaultPersisterNapUnderNumFiles int = 1000 + + // MemoryPressurePauseThreshold lets persister to have a better leeway + // for prudently performing the memory merge of segments on a memory + // pressure situation. Here the config value is an upper threshold + // for the number of paused application threads. The default value would + // be a very high number to always favour the merging of memory segments. + DefaultMemoryPressurePauseThreshold uint64 = math.MaxUint64 + + // DefaultMinSegmentsForInMemoryMerge represents the default number of + // in-memory zap segments that persistSnapshotMaybeMerge() needs to + // see in an IndexSnapshot before it decides to merge and persist + // those segments + DefaultMinSegmentsForInMemoryMerge int = 2 + + // number workers which parallely perform an in-memory merge of the segments + // followed by a flush operation. + DefaultNumPersisterWorkers int = 1 + + // maximum size of data that a single worker is allowed to perform the in-memory + // merge operation. + DefaultMaxSizeInMemoryMergePerWorker int = 0 + + // NumSnapshotsToKeep represents how many recent, old snapshots to + // keep around per Scorch instance. Useful for apps that require + // rollback'ability. + NumSnapshotsToKeep int = 1 + + // RollbackSamplingInterval controls how far back we are looking + // in the history to get the rollback points. + // If we have to keep N snapshots ( = NumSnapshotsToKeep ), then upon + // including the very latest snapshot there should be N snapshots. + // If the timestamp of the latest snapshot is T, and the + // RollbackSamplingInterval is D, then the very last one would be T - (N - 1) * D + // A value of 0 means we will not sample, and will keep the very latest N snapshots. + // Example: + // - For N = 3, and the below timing diagram: + // a b c d + // |-----|-----|-----| --> time + // | D | D | D | + // X Y Z T + // - The rollback points would be b, c, and d. a would be too old. + RollbackSamplingInterval time.Duration = 0 + + // Controls what portion of the earlier rollback points to retain during + // a infrequent/sparse mutation scenario + RollbackRetentionFactor float64 = 0.5 +) + +type persisterOptions struct { + // PersisterNapTimeMSec controls the wait/delay injected into + // persistence workloop to improve the chances for + // a healthier and heavier in-memory merging + PersisterNapTimeMSec int + + // PersisterNapTimeMSec > 0, and the number of files is less than + // PersisterNapUnderNumFiles, then the persister will sleep + // PersisterNapTimeMSec amount of time to improve the chances for + // a healthier and heavier in-memory merging + PersisterNapUnderNumFiles int + + // MemoryPressurePauseThreshold lets persister to have a better leeway + // for prudently performing the memory merge of segments on a memory + // pressure situation. Here the config value is an upper threshold + // for the number of paused application threads. + MemoryPressurePauseThreshold uint64 + + // NumPersisterWorkers decides the number of parallel workers that will + // perform the in-memory merge of segments followed by a flush operation. + NumPersisterWorkers int + + // MaxSizeInMemoryMerge is the maximum size of data that a single persister + // worker is allowed to work on + MaxSizeInMemoryMergePerWorker int +} + +type notificationChan chan struct{} + +func (s *Scorch) persisterLoop() { + defer func() { + if r := recover(); r != nil { + s.fireAsyncError(NewScorchError( + persister, + fmt.Sprintf("panic: %v, path: %s", r, s.path), + ErrAsyncPanic, + )) + } + + s.asyncTasks.Done() + }() + + var persistWatchers []*epochWatcher + var lastPersistedEpoch, lastMergedEpoch uint64 + var ew *epochWatcher + + var unpersistedCallbacks []index.BatchCallback + +OUTER: + for { + atomic.AddUint64(&s.stats.TotPersistLoopBeg, 1) + + select { + case <-s.closeCh: + break OUTER + case ew = <-s.persisterNotifier: + persistWatchers = append(persistWatchers, ew) + default: + } + if ew != nil && ew.epoch > lastMergedEpoch { + lastMergedEpoch = ew.epoch + } + lastMergedEpoch, persistWatchers = s.pausePersisterForMergerCatchUp(lastPersistedEpoch, + lastMergedEpoch, persistWatchers, s.persisterOptions) + + var ourSnapshot *IndexSnapshot + var ourPersisted []chan error + var ourPersistedCallbacks []index.BatchCallback + + // check to see if there is a new snapshot to persist + s.rootLock.Lock() + if s.root != nil && s.root.epoch > lastPersistedEpoch { + ourSnapshot = s.root + ourSnapshot.AddRef() + ourPersisted = s.rootPersisted + s.rootPersisted = nil + ourPersistedCallbacks = s.persistedCallbacks + s.persistedCallbacks = nil + atomic.StoreUint64(&s.iStats.persistSnapshotSize, uint64(ourSnapshot.Size())) + atomic.StoreUint64(&s.iStats.persistEpoch, ourSnapshot.epoch) + } + s.rootLock.Unlock() + + if ourSnapshot != nil { + startTime := time.Now() + + err := s.persistSnapshot(ourSnapshot, s.persisterOptions) + for _, ch := range ourPersisted { + if err != nil { + ch <- err + } + close(ch) + } + if err != nil { + atomic.StoreUint64(&s.iStats.persistEpoch, 0) + if err == segment.ErrClosed { + // index has been closed + _ = ourSnapshot.DecRef() + break OUTER + } + + // save this current snapshot's persistedCallbacks, to invoke during + // the retry attempt + unpersistedCallbacks = append(unpersistedCallbacks, ourPersistedCallbacks...) + + s.fireAsyncError(NewScorchError( + persister, + fmt.Sprintf("got err persisting snapshot: %v", err), + ErrPersist, + )) + _ = ourSnapshot.DecRef() + atomic.AddUint64(&s.stats.TotPersistLoopErr, 1) + continue OUTER + } + + if unpersistedCallbacks != nil { + // in the event of this being a retry attempt for persisting a snapshot + // that had earlier failed, prepend the persistedCallbacks associated + // with earlier segment(s) to the latest persistedCallbacks + ourPersistedCallbacks = append(unpersistedCallbacks, ourPersistedCallbacks...) + unpersistedCallbacks = nil + } + + for i := range ourPersistedCallbacks { + ourPersistedCallbacks[i](err) + } + + atomic.StoreUint64(&s.stats.LastPersistedEpoch, ourSnapshot.epoch) + + lastPersistedEpoch = ourSnapshot.epoch + for _, ew := range persistWatchers { + close(ew.notifyCh) + } + + persistWatchers = nil + _ = ourSnapshot.DecRef() + + changed := false + s.rootLock.RLock() + if s.root != nil && s.root.epoch != lastPersistedEpoch { + changed = true + } + s.rootLock.RUnlock() + + s.fireEvent(EventKindPersisterProgress, time.Since(startTime)) + + if changed { + atomic.AddUint64(&s.stats.TotPersistLoopProgress, 1) + continue OUTER + } + } + + // tell the introducer we're waiting for changes + w := &epochWatcher{ + epoch: lastPersistedEpoch, + notifyCh: make(notificationChan, 1), + } + + select { + case <-s.closeCh: + break OUTER + case s.introducerNotifier <- w: + } + + if ok := s.fireEvent(EventKindPurgerCheck, 0); ok { + s.removeOldData() // might as well cleanup while waiting + } + + atomic.AddUint64(&s.stats.TotPersistLoopWait, 1) + + select { + case <-s.closeCh: + break OUTER + case <-w.notifyCh: + // woken up, next loop should pick up work + atomic.AddUint64(&s.stats.TotPersistLoopWaitNotified, 1) + case ew = <-s.persisterNotifier: + // if the watchers are already caught up then let them wait, + // else let them continue to do the catch up + persistWatchers = append(persistWatchers, ew) + } + + atomic.AddUint64(&s.stats.TotPersistLoopEnd, 1) + } +} + +func notifyMergeWatchers(lastPersistedEpoch uint64, + persistWatchers []*epochWatcher, +) []*epochWatcher { + var watchersNext []*epochWatcher + for _, w := range persistWatchers { + if w.epoch < lastPersistedEpoch { + close(w.notifyCh) + } else { + watchersNext = append(watchersNext, w) + } + } + return watchersNext +} + +func (s *Scorch) pausePersisterForMergerCatchUp(lastPersistedEpoch uint64, + lastMergedEpoch uint64, persistWatchers []*epochWatcher, + po *persisterOptions, +) (uint64, []*epochWatcher) { + // First, let the watchers proceed if they lag behind + persistWatchers = notifyMergeWatchers(lastPersistedEpoch, persistWatchers) + + // Check the merger lag by counting the segment files on disk, + numFilesOnDisk, _, _ := s.diskFileStats(nil) + + // On finding fewer files on disk, persister takes a short pause + // for sufficient in-memory segments to pile up for the next + // memory merge cum persist loop. + if numFilesOnDisk < uint64(po.PersisterNapUnderNumFiles) && + po.PersisterNapTimeMSec > 0 && s.NumEventsBlocking() == 0 { + select { + case <-s.closeCh: + case <-time.After(time.Millisecond * time.Duration(po.PersisterNapTimeMSec)): + atomic.AddUint64(&s.stats.TotPersisterNapPauseCompleted, 1) + + case ew := <-s.persisterNotifier: + // unblock the merger in meantime + persistWatchers = append(persistWatchers, ew) + lastMergedEpoch = ew.epoch + persistWatchers = notifyMergeWatchers(lastPersistedEpoch, persistWatchers) + atomic.AddUint64(&s.stats.TotPersisterMergerNapBreak, 1) + } + return lastMergedEpoch, persistWatchers + } + + // Finding too many files on disk could be due to two reasons. + // 1. Too many older snapshots awaiting the clean up. + // 2. The merger could be lagging behind on merging the disk files. + if numFilesOnDisk > uint64(po.PersisterNapUnderNumFiles) { + if ok := s.fireEvent(EventKindPurgerCheck, 0); ok { + s.removeOldData() + } + numFilesOnDisk, _, _ = s.diskFileStats(nil) + } + + // Persister pause until the merger catches up to reduce the segment + // file count under the threshold. + // But if there is memory pressure, then skip this sleep maneuvers. +OUTER: + for po.PersisterNapUnderNumFiles > 0 && + numFilesOnDisk >= uint64(po.PersisterNapUnderNumFiles) && + lastMergedEpoch < lastPersistedEpoch { + atomic.AddUint64(&s.stats.TotPersisterSlowMergerPause, 1) + + select { + case <-s.closeCh: + break OUTER + case ew := <-s.persisterNotifier: + persistWatchers = append(persistWatchers, ew) + lastMergedEpoch = ew.epoch + } + + atomic.AddUint64(&s.stats.TotPersisterSlowMergerResume, 1) + + // let the watchers proceed if they lag behind + persistWatchers = notifyMergeWatchers(lastPersistedEpoch, persistWatchers) + + numFilesOnDisk, _, _ = s.diskFileStats(nil) + } + + return lastMergedEpoch, persistWatchers +} + +func (s *Scorch) parsePersisterOptions() (*persisterOptions, error) { + po := persisterOptions{ + PersisterNapTimeMSec: DefaultPersisterNapTimeMSec, + PersisterNapUnderNumFiles: DefaultPersisterNapUnderNumFiles, + MemoryPressurePauseThreshold: DefaultMemoryPressurePauseThreshold, + NumPersisterWorkers: DefaultNumPersisterWorkers, + MaxSizeInMemoryMergePerWorker: DefaultMaxSizeInMemoryMergePerWorker, + } + if v, ok := s.config["scorchPersisterOptions"]; ok { + b, err := util.MarshalJSON(v) + if err != nil { + return &po, err + } + + err = util.UnmarshalJSON(b, &po) + if err != nil { + return &po, err + } + } + if err := validatePersisterOptions(&po); err != nil { + return nil, err + } + return &po, nil +} + +// validatePersisterOptions validates the persister options +func validatePersisterOptions(options *persisterOptions) error { + // P < 1 is an invalid case + if options.NumPersisterWorkers <= 0 { + return fmt.Errorf("NumPersisterWorkers must be greater than 0") + } + // P = 1 and M > 0 is a valid case where one worker will serially flush all batches + // P = 1 and M = 0 is a special case which preserves the legacy one-shot in-memory merge + flush behaviour + // P > 1 and M > 0 is a valid case where multiple workers will flush batches in parallel + // P > 1 and M = 0 is an invalid case + if options.NumPersisterWorkers > 1 && options.MaxSizeInMemoryMergePerWorker == 0 { + return fmt.Errorf("MaxSizeInMemoryMergePerWorker must be greater than 0 when NumPersisterWorkers > 1") + } + return nil +} + +func (s *Scorch) persistSnapshot(snapshot *IndexSnapshot, po *persisterOptions) error { + // Perform in-memory segment merging only when the memory pressure is + // below the configured threshold, else the persister performs the + // direct persistence of segments. + if s.NumEventsBlocking() < po.MemoryPressurePauseThreshold { + persisted, err := s.persistSnapshotMaybeMerge(snapshot, po) + if err != nil { + return err + } + if persisted { + return nil + } + } + + return s.persistSnapshotDirect(snapshot) +} + +type flushable struct { + sbsBatch []segment.Segment + sbsBatchDrops []*roaring.Bitmap + sbsBatchSnapshots []*SegmentSnapshot +} + +func legacyFlushBehaviour(maxSizeInMemoryMergePerWorker, numPersisterWorkers int) bool { + // DefaultMaxSizeInMemoryMergePerWorker = 0 is a special value to preserve the legacy + // one-shot in-memory merge + flush behaviour. + return maxSizeInMemoryMergePerWorker == 0 && numPersisterWorkers == 1 +} + +// persistSnapshotMaybeMerge examines the snapshot and might merge and +// persist the in-memory zap segments if there are enough of them +func (s *Scorch) persistSnapshotMaybeMerge(snapshot *IndexSnapshot, po *persisterOptions) ( + bool, error) { + // split our segment snapshots into unpersisted and persisted segments + var persistedSegments []*SegmentSnapshot + var unpersistedSegments []*SegmentSnapshot + numSegments := len(snapshot.segment) + + // collect details of the unpersisted segments + sbs := make([]segment.Segment, 0, numSegments) + sbsDrops := make([]*roaring.Bitmap, 0, numSegments) + sbsSizes := make([]int, 0, numSegments) + for _, segmentSnapshot := range snapshot.segment { + if _, ok := segmentSnapshot.segment.(segment.PersistedSegment); ok { + persistedSegments = append(persistedSegments, segmentSnapshot) + } else { + unpersistedSegments = append(unpersistedSegments, segmentSnapshot) + sbs = append(sbs, segmentSnapshot.segment) + sbsDrops = append(sbsDrops, segmentSnapshot.deleted) + sbsSizes = append(sbsSizes, segmentSnapshot.Size()) + } + } + numPersistedSegments := len(persistedSegments) + numUnpersistedSegments := len(unpersistedSegments) + + // if we don't have enough segments to persist, then just return + if numUnpersistedSegments < DefaultMinSegmentsForInMemoryMerge { + return false, nil + } + + // create our flushSet, which will be an array of flushable objects, + // each of which will be a batch of segments to merge and persist. + // We are guaranteed that at least one flushable object will be created, + // since we have already ensured that at least one unpersisted segment is present. + var flushSet []*flushable + if legacyFlushBehaviour(po.MaxSizeInMemoryMergePerWorker, po.NumPersisterWorkers) { + val := &flushable{ + sbsBatch: slices.Clone(sbs), + sbsBatchDrops: slices.Clone(sbsDrops), + sbsBatchSnapshots: slices.Clone(unpersistedSegments), + } + flushSet = append(flushSet, val) + } else { + sbsBatch := make([]segment.Segment, 0, numUnpersistedSegments) + sbsBatchDrops := make([]*roaring.Bitmap, 0, numUnpersistedSegments) + sbsBatchSnapshots := make([]*SegmentSnapshot, 0, numUnpersistedSegments) + runningSize := 0 + for i := 0; i < numUnpersistedSegments; i++ { + sbsBatch = append(sbsBatch, sbs[i]) + sbsBatchDrops = append(sbsBatchDrops, sbsDrops[i]) + sbsBatchSnapshots = append(sbsBatchSnapshots, unpersistedSegments[i]) + runningSize += sbsSizes[i] + if runningSize >= po.MaxSizeInMemoryMergePerWorker && len(sbsBatch) >= DefaultMinSegmentsForInMemoryMerge { + flushSet = append(flushSet, &flushable{ + sbsBatch: slices.Clone(sbsBatch), + sbsBatchDrops: slices.Clone(sbsBatchDrops), + sbsBatchSnapshots: slices.Clone(sbsBatchSnapshots), + }) + sbsBatch, sbsBatchDrops, sbsBatchSnapshots = sbsBatch[:0], sbsBatchDrops[:0], sbsBatchSnapshots[:0] + runningSize = 0 + } + } + if len(sbsBatch) > 0 { + flushSet = append(flushSet, &flushable{ + sbsBatch: slices.Clone(sbsBatch), + sbsBatchDrops: slices.Clone(sbsBatchDrops), + sbsBatchSnapshots: slices.Clone(sbsBatchSnapshots), + }) + } + } + + // now merge each batch into a new segment, and persist all of them to disk, + // and construct a new snapshot with the merged segments + newSnapshot, newSegmentIDs, err := s.mergeAndPersistInMemorySegments(flushSet, po) + if err != nil { + return false, err + } + if newSnapshot == nil { + return false, nil + } + defer func() { + _ = newSnapshot.DecRef() + }() + + // construct a snapshot that's logically equivalent to the input + // snapshot, but with merged segments replaced by the new segment + equiv := &IndexSnapshot{ + parent: snapshot.parent, + segment: make([]*SegmentSnapshot, 0, numPersistedSegments+len(newSegmentIDs)), + internal: snapshot.internal, + epoch: snapshot.epoch, + creator: "persistSnapshotMaybeMerge", + } + + // first add all the segments that were already persisted + // and did not participate in the merge + for _, segment := range persistedSegments { + equiv.segment = append(equiv.segment, segment) + } + + // next, add all the segments that were unpersisted and hence + // participated in the merge, from the merged snapshot + for _, segment := range newSnapshot.segment { + if _, ok := newSegmentIDs[segment.id]; ok { + equiv.segment = append(equiv.segment, &SegmentSnapshot{ + id: segment.id, + segment: segment.segment, + deleted: nil, // nil since merging handled deletions + stats: segment.stats, + }) + } + } + + // finally, persist the equivalent snapshot to disk + err = s.persistSnapshotDirect(equiv) + if err != nil { + return false, err + } + + return true, nil +} + +func copyToDirectory(srcPath string, d index.Directory) (int64, error) { + if d == nil { + return 0, nil + } + + dest, err := d.GetWriter(filepath.Join("store", filepath.Base(srcPath))) + if err != nil { + return 0, fmt.Errorf("GetWriter err: %v", err) + } + + // skip + if dest == nil { + return 0, nil + } + + sourceFileStat, err := os.Stat(srcPath) + if err != nil { + return 0, err + } + + if !sourceFileStat.Mode().IsRegular() { + return 0, fmt.Errorf("%s is not a regular file", srcPath) + } + + source, err := os.Open(srcPath) + if err != nil { + return 0, err + } + defer source.Close() + defer dest.Close() + return io.Copy(dest, source) +} + +func persistToDirectory(seg segment.UnpersistedSegment, d index.Directory, + path string, +) error { + if d == nil { + return seg.Persist(path) + } + + sg, ok := seg.(io.WriterTo) + if !ok { + return fmt.Errorf("no io.WriterTo segment implementation found") + } + + w, err := d.GetWriter(filepath.Join("store", filepath.Base(path))) + if err != nil { + return err + } + + _, err = sg.WriteTo(w) + w.Close() + + return err +} + +func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *util.BoltTxImpl, path string, segPlugin SegmentPlugin, d index.Directory) ([]string, map[uint64]string, error) { + snapshotsBucket, err := tx.CreateBucketIfNotExists(util.BoltSnapshotsBucket) + if err != nil { + return nil, nil, err + } + newSnapshotKey := encodeUvarintAscending(nil, snapshot.epoch) + snapshotBucket, err := snapshotsBucket.CreateBucketIfNotExists(newSnapshotKey) + if err != nil { + return nil, nil, err + } + + // persist meta values + metaBucket, err := snapshotBucket.CreateBucketIfNotExists(util.BoltMetaDataKey) + if err != nil { + return nil, nil, err + } + err = metaBucket.Put(util.BoltMetaDataSegmentTypeKey, []byte(segPlugin.Type()), nil) + if err != nil { + return nil, nil, err + } + buf := make([]byte, binary.MaxVarintLen32) + binary.BigEndian.PutUint32(buf, segPlugin.Version()) + err = metaBucket.Put(util.BoltMetaDataSegmentVersionKey, buf, nil) + if err != nil { + return nil, nil, err + } + // always obtain the path from the parent snapshot if available + // since that is the primary source of truth for context + if snapshot.parent != nil { + path = snapshot.parent.path + } + writer, err := util.NewFileWriter( + []byte(path + string(os.PathSeparator) + "root.bolt")) + if err != nil { + return nil, nil, err + } + + // persist the writer ID used for the bolt snapshot + err = metaBucket.Put(util.BoltMetaDataFileWriterIDKey, []byte(writer.Id()), writer) + if err != nil { + return nil, nil, err + } + + // Storing the timestamp at which the current indexSnapshot + // was persisted, useful when you want to spread the + // numSnapshotsToKeep reasonably better than consecutive + // epochs. + currTimeStamp := time.Now() + timeStampBinary, err := currTimeStamp.MarshalText() + if err != nil { + return nil, nil, err + } + err = metaBucket.Put(util.BoltMetaDataTimeStamp, timeStampBinary, writer) + if err != nil { + return nil, nil, err + } + + // persist internal values + internalBucket, err := snapshotBucket.CreateBucketIfNotExists(util.BoltInternalKey) + if err != nil { + return nil, nil, err + } + for k, v := range snapshot.internal { + err = internalBucket.Put([]byte(k), v, writer) + if err != nil { + return nil, nil, err + } + } + + if snapshot.parent != nil { + val := make([]byte, 8) + bytesWritten := atomic.LoadUint64(&snapshot.parent.stats.TotBytesWrittenAtIndexTime) + binary.LittleEndian.PutUint64(val, bytesWritten) + err = internalBucket.Put(util.TotBytesWrittenKey, val, writer) + if err != nil { + return nil, nil, err + } + } + + filenames := make([]string, 0, len(snapshot.segment)) + newSegmentPaths := make(map[uint64]string, len(snapshot.segment)) + + // first ensure that each segment in this snapshot has been persisted + for _, segmentSnapshot := range snapshot.segment { + snapshotSegmentKey := encodeUvarintAscending(nil, segmentSnapshot.id) + snapshotSegmentBucket, err := snapshotBucket.CreateBucketIfNotExists(snapshotSegmentKey) + if err != nil { + return nil, nil, err + } + switch seg := segmentSnapshot.segment.(type) { + case segment.PersistedSegment: + segPath := seg.Path() + _, err = copyToDirectory(segPath, d) + if err != nil { + return nil, nil, fmt.Errorf("segment: %s copy err: %v", segPath, err) + } + filename := filepath.Base(segPath) + err = snapshotSegmentBucket.Put(util.BoltPathKey, []byte(filename), writer) + if err != nil { + return nil, nil, err + } + filenames = append(filenames, filename) + case segment.UnpersistedSegment: + // need to persist this to disk + filename := zapFileName(segmentSnapshot.id) + path := filepath.Join(path, filename) + err := persistToDirectory(seg, d, path) + if err != nil { + return nil, nil, fmt.Errorf("segment: %s persist err: %v", path, err) + } + newSegmentPaths[segmentSnapshot.id] = path + err = snapshotSegmentBucket.Put(util.BoltPathKey, []byte(filename), nil) + if err != nil { + return nil, nil, err + } + filenames = append(filenames, filename) + default: + return nil, nil, fmt.Errorf("unknown segment type: %T", seg) + } + + // store current deleted bits + var roaringBuf bytes.Buffer + if segmentSnapshot.deleted != nil { + _, err = segmentSnapshot.deleted.WriteTo(&roaringBuf) + if err != nil { + return nil, nil, fmt.Errorf("error persisting roaring bytes: %v", err) + } + err = snapshotSegmentBucket.Put(util.BoltDeletedKey, roaringBuf.Bytes(), writer) + if err != nil { + return nil, nil, err + } + } + + // store segment stats + if segmentSnapshot.stats != nil { + statsBytes, err := json.Marshal(segmentSnapshot.stats.Fetch()) + if err != nil { + return nil, nil, err + } + err = snapshotSegmentBucket.Put(util.BoltStatsKey, statsBytes, writer) + if err != nil { + return nil, nil, err + } + } + + // store updated field info + if segmentSnapshot.updatedFields != nil { + updatedFieldsBytes, err := json.Marshal(segmentSnapshot.updatedFields) + if err != nil { + return nil, nil, err + } + err = snapshotSegmentBucket.Put( + util.BoltUpdatedFieldsKey, updatedFieldsBytes, writer) + if err != nil { + return nil, nil, err + } + } + } + + return filenames, newSegmentPaths, nil +} + +func (s *Scorch) persistSnapshotDirect(snapshot *IndexSnapshot) (err error) { + // start a write transaction + tx, err := s.rootBolt.Begin(true) + if err != nil { + return err + } + // defer rollback on error + defer func() { + if err != nil { + _ = tx.Rollback() + } + }() + + filenames, newSegmentPaths, err := prepareBoltSnapshot(snapshot, tx, s.path, s.segPlugin, nil) + if err != nil { + return err + } + + // we need to swap in a new root only when we've persisted 1 or + // more segments -- whereby the new root would have 1-for-1 + // replacements of in-memory segments with file-based segments + // + // other cases like updates to internal values only, and/or when + // there are only deletions, are already covered and persisted by + // the newly populated boltdb snapshotBucket above + if len(newSegmentPaths) > 0 { + // now try to open all the new snapshots + newSegments := make(map[uint64]segment.Segment, len(newSegmentPaths)) + defer func() { + for _, s := range newSegments { + if s != nil { + // cleanup segments that were opened but not + // swapped into the new root + _ = s.Close() + } + } + }() + for segmentID, path := range newSegmentPaths { + newSegments[segmentID], err = s.segPlugin.OpenUsing(path, s.segmentConfig) + if err != nil { + return fmt.Errorf("error opening new segment at %s, %v", path, err) + } + } + + persist := &persistIntroduction{ + persisted: newSegments, + applied: make(notificationChan), + } + + select { + case <-s.closeCh: + return segment.ErrClosed + case s.persists <- persist: + } + + // blockingly wait until the persist has been applied + <-persist.applied + } + + err = tx.Commit() + if err != nil { + return err + } + + err = s.rootBolt.Sync() + if err != nil { + return err + } + + // allow files to become eligible for removal after commit, such + // as file segments from snapshots that came from the merger + s.rootLock.Lock() + for _, filename := range filenames { + delete(s.ineligibleForRemoval, filename) + } + s.rootLock.Unlock() + + return nil +} + +func zapFileName(epoch uint64) string { + return fmt.Sprintf("%012x.zap", epoch) +} + +// bolt snapshot code +func (s *Scorch) loadFromBolt() error { + err := s.rootBolt.View(func(tx *util.BoltTxImpl) error { + snapshots := tx.Bucket(util.BoltSnapshotsBucket) + if snapshots == nil { + return nil + } + foundRoot := false + c := snapshots.Cursor() + for k, _ := c.Last(); k != nil; k, _ = c.Prev() { + _, snapshotEpoch, err := decodeUvarintAscending(k) + if err != nil { + log.Printf("unable to parse segment epoch %x, continuing", k) + continue + } + if foundRoot { + s.AddEligibleForRemoval(snapshotEpoch) + continue + } + snapshot := snapshots.GetBucket(k) + if snapshot == nil { + log.Printf("snapshot key, but bucket missing %x, continuing", k) + s.AddEligibleForRemoval(snapshotEpoch) + continue + } + indexSnapshot, err := s.loadSnapshot(snapshot) + if err != nil { + log.Printf("unable to load snapshot, %v, continuing", err) + s.AddEligibleForRemoval(snapshotEpoch) + continue + } + indexSnapshot.epoch = snapshotEpoch + // set the nextSegmentID + s.nextSegmentID, err = s.maxSegmentIDOnDisk() + if err != nil { + return err + } + s.nextSegmentID++ + s.rootLock.Lock() + s.nextSnapshotEpoch = snapshotEpoch + 1 + rootPrev := s.root + s.root = indexSnapshot + s.rootLock.Unlock() + + if rootPrev != nil { + _ = rootPrev.DecRef() + } + + foundRoot = true + } + + // try init trainer and load the trained data + if trainer := initTrainer(s, s.config); trainer != nil { + s.trainer = trainer + trainerBucket := snapshots.GetBucket(util.BoltTrainerKey) + err := s.trainer.loadTrainedData(trainerBucket) + if err != nil { + return err + } + } + + return nil + }) + if err != nil { + return err + } + // we initialize the checkPoints from all the persisted snapshots, + // because if we have state to load, the previous process was already + // maintaining checkpoints, whatever survives in the bucket now is that state. + persistedSnapshots, err := s.rootBoltSnapshotMetaData() + if err != nil { + return err + } + s.checkPoints = persistedSnapshots + return nil +} + +// LoadSnapshot loads the segment with the specified epoch +// NOTE: this is currently ONLY intended to be used by the command-line tool +func (s *Scorch) LoadSnapshot(epoch uint64) (rv *IndexSnapshot, err error) { + err = s.rootBolt.View(func(tx *util.BoltTxImpl) error { + snapshots := tx.Bucket(util.BoltSnapshotsBucket) + if snapshots == nil { + return nil + } + snapshotKey := encodeUvarintAscending(nil, epoch) + snapshot := snapshots.GetBucket(snapshotKey) + if snapshot == nil { + return fmt.Errorf("snapshot with epoch: %v - doesn't exist", epoch) + } + rv, err = s.loadSnapshot(snapshot) + return err + }) + if err != nil { + return nil, err + } + return rv, nil +} + +func (s *Scorch) loadSnapshot(snapshot *util.BoltBucketImpl) (*IndexSnapshot, error) { + rv := &IndexSnapshot{ + parent: s, + internal: make(map[string][]byte), + refs: 1, + creator: "loadSnapshot", + } + // first we look for the meta-data bucket, this will tell us + // which segment type/version was used for this snapshot + // all operations for this scorch will use this type/version + metaBucket := snapshot.GetBucket(util.BoltMetaDataKey) + if metaBucket == nil { + _ = rv.DecRef() + return nil, fmt.Errorf("meta-data bucket missing") + } + segmentType, err := metaBucket.Get(util.BoltMetaDataSegmentTypeKey, nil) + if err != nil { + _ = rv.DecRef() + return nil, fmt.Errorf("segment type missing: %v", err) + } + segmentVersionBytes, err := metaBucket.Get(util.BoltMetaDataSegmentVersionKey, nil) + if err != nil { + _ = rv.DecRef() + return nil, fmt.Errorf("segment version missing: %v", err) + } + segmentVersion := binary.BigEndian.Uint32(segmentVersionBytes) + err = s.loadSegmentPlugin(string(segmentType), segmentVersion) + if err != nil { + _ = rv.DecRef() + return nil, fmt.Errorf( + "unable to load correct segment wrapper: %v", err) + } + fileWriterID, err := metaBucket.Get(util.BoltMetaDataFileWriterIDKey, nil) + if err != nil { + _ = rv.DecRef() + return nil, fmt.Errorf("file writer id missing: %v", err) + } + reader, err := util.NewFileReader( + string(fileWriterID), []byte(s.path+string(os.PathSeparator)+"root.bolt")) + if err != nil { + _ = rv.DecRef() + return nil, fmt.Errorf("unable to load correct reader: %v", err) + } + + var running uint64 + c := snapshot.Cursor() + for k, _ := c.First(); k != nil; k, _ = c.Next() { + if k[0] == util.BoltInternalKey[0] { + internalBucket := snapshot.GetBucket(k) + if internalBucket == nil { + _ = rv.DecRef() + return nil, fmt.Errorf("internal bucket missing") + } + err := internalBucket.ForEach(func(key []byte, val []byte) error { + rv.internal[string(key)] = val + return nil + }, reader) + if err != nil { + _ = rv.DecRef() + return nil, err + } + } else if k[0] != util.BoltMetaDataKey[0] { + segmentBucket := snapshot.GetBucket(k) + if segmentBucket == nil { + _ = rv.DecRef() + return nil, fmt.Errorf("segment key, but bucket missing %x", k) + } + segmentSnapshot, err := s.loadSegment(segmentBucket, reader) + if err != nil { + _ = rv.DecRef() + return nil, fmt.Errorf("failed to load segment: %v", err) + } + _, segmentSnapshot.id, err = decodeUvarintAscending(k) + if err != nil { + _ = rv.DecRef() + return nil, fmt.Errorf("failed to decode segment id: %v", err) + } + rv.segment = append(rv.segment, segmentSnapshot) + rv.offsets = append(rv.offsets, running) + // Merge all segment level updated field info for use during queries + if segmentSnapshot.updatedFields != nil { + rv.MergeUpdateFieldsInfo(segmentSnapshot.updatedFields) + } + running += segmentSnapshot.segment.Count() + } + } + return rv, nil +} + +func (s *Scorch) loadSegment(segmentBucket *util.BoltBucketImpl, reader util.FileReader) ( + *SegmentSnapshot, error) { + pathBytes, err := segmentBucket.Get(util.BoltPathKey, nil) + if pathBytes == nil { + return nil, fmt.Errorf("segment path missing") + } + segmentPath := s.path + string(os.PathSeparator) + string(pathBytes) + seg, err := s.segPlugin.OpenUsing(segmentPath, s.segmentConfig) + if err != nil { + return nil, fmt.Errorf("error opening bolt segment: %v", err) + } + + rv := &SegmentSnapshot{ + segment: seg, + cachedDocs: &cachedDocs{cache: nil}, + cachedMeta: newCachedMeta(), + } + deletedBytes, err := segmentBucket.Get(util.BoltDeletedKey, reader) + if err != nil { + _ = seg.Close() + return nil, fmt.Errorf("error getting deleted bytes: %v", err) + } + if deletedBytes != nil { + deletedBitmap := roaring.NewBitmap() + r := bytes.NewReader(deletedBytes) + _, err := deletedBitmap.ReadFrom(r) + if err != nil { + _ = seg.Close() + return nil, fmt.Errorf("error reading deleted bytes: %v", err) + } + if !deletedBitmap.IsEmpty() { + rv.deleted = deletedBitmap + } + } + statBytes, err := segmentBucket.Get(util.BoltStatsKey, reader) + if err != nil { + _ = seg.Close() + return nil, fmt.Errorf("error getting stat bytes: %v", err) + } + if statBytes != nil { + var statsMap map[string]map[string]uint64 + err := json.Unmarshal(statBytes, &statsMap) + if err != nil { + _ = seg.Close() + return nil, fmt.Errorf("error reading stat bytes: %v", err) + } + rv.stats = &fieldStats{statMap: statsMap} + } + updatedFieldBytes, err := segmentBucket.Get(util.BoltUpdatedFieldsKey, reader) + if err != nil { + _ = seg.Close() + return nil, fmt.Errorf("error getting updated field bytes: %v", err) + } + if updatedFieldBytes != nil { + var updatedFields map[string]*index.UpdateFieldInfo + err = json.Unmarshal(updatedFieldBytes, &updatedFields) + if err != nil { + _ = seg.Close() + return nil, fmt.Errorf("error reading updated field bytes: %v", err) + } + rv.updatedFields = updatedFields + // Set the value within the segment base for use during merge + rv.UpdateFieldsInfo(rv.updatedFields) + } + + return rv, nil +} + +// identify all the file callback writer ids that are in use by boltdb +func (s *Scorch) boltFileWriterIDsInUse() (map[string]struct{}, error) { + idMap := make(map[string]struct{}) + err := s.rootBolt.View(func(tx *util.BoltTxImpl) error { + snapshots := tx.Bucket(util.BoltSnapshotsBucket) + if snapshots == nil { + return nil + } + c := snapshots.Cursor() + for k, _ := c.First(); k != nil; k, _ = c.Next() { + snapshot := snapshots.GetBucket(k) + if snapshot == nil { + continue + } + metaBucket := snapshot.GetBucket(util.BoltMetaDataKey) + if metaBucket == nil { + continue + } + id, err := metaBucket.Get(util.BoltMetaDataFileWriterIDKey, nil) + if err != nil { + return err + } + idMap[string(id)] = struct{}{} + } + return nil + }) + if err != nil { + return nil, err + } + + return idMap, nil +} + +// remove all content in boltdb associated with the file callback +// writer ids and process the data using the latest file writer +func (s *Scorch) removeBoltFileWriterIDs(ids map[string]struct{}) error { + filePath := s.path + string(os.PathSeparator) + "root.bolt" + writer, err := util.NewFileWriter([]byte(filePath)) + if err != nil { + return err + } + + err = s.rootBolt.Update(func(tx *util.BoltTxImpl) error { + snapshots := tx.Bucket(util.BoltSnapshotsBucket) + if snapshots == nil { + return nil + } + c := snapshots.Cursor() + for k, _ := c.First(); k != nil; k, _ = c.Next() { + snapshot := snapshots.GetBucket(k) + if snapshot == nil { + continue + } + metaBucket := snapshot.GetBucket(util.BoltMetaDataKey) + if metaBucket == nil { + continue + } + fileWriterIDBytes, err := metaBucket.Get(util.BoltMetaDataFileWriterIDKey, nil) + if err != nil { + return err + } + fileWriterID := string(fileWriterIDBytes) + if _, ok := ids[fileWriterID]; ok { + reader, err := util.NewFileReader(fileWriterID, []byte(filePath)) + if err != nil { + return fmt.Errorf("unable to load correct reader: %v", err) + } + + cc := snapshot.Cursor() + for kk, _ := cc.First(); kk != nil; kk, _ = cc.Next() { + if kk[0] == util.BoltInternalKey[0] { + internalBucket := snapshot.GetBucket(kk) + if internalBucket == nil { + continue + } + // process all of the internal values and replace them with new values + internalBucketVals := make(map[string][]byte) + err := internalBucket.ForEach(func(key []byte, val []byte) error { + internalBucketVals[string(key)] = val + return nil + }, reader) + if err != nil { + return err + } + for key, val := range internalBucketVals { + err = internalBucket.Put([]byte(key), val, writer) + if err != nil { + return err + } + } + } else if kk[0] != util.BoltMetaDataKey[0] { + segmentBucket := snapshot.GetBucket(kk) + if segmentBucket == nil { + continue + } + // process the updated field key + updatedFieldBytes, err := segmentBucket.Get(util.BoltUpdatedFieldsKey, reader) + if err != nil { + return fmt.Errorf("error getting updated field bytes: %v", err) + } + if updatedFieldBytes != nil { + err = segmentBucket.Put(util.BoltUpdatedFieldsKey, updatedFieldBytes, writer) + if err != nil { + return err + } + } + + // process the deleted key + deletedBytes, err := segmentBucket.Get(util.BoltDeletedKey, reader) + if err != nil { + return fmt.Errorf("error getting deleted bytes: %v", err) + } + if deletedBytes != nil { + err = segmentBucket.Put(util.BoltDeletedKey, deletedBytes, writer) + if err != nil { + return err + } + } + // process the stats key + statsBytes, err := segmentBucket.Get(util.BoltStatsKey, reader) + if err != nil { + return fmt.Errorf("error getting stats bytes: %v", err) + } + if statsBytes != nil { + err = segmentBucket.Put(util.BoltStatsKey, statsBytes, writer) + if err != nil { + return err + } + } + } + } + err = metaBucket.Put(util.BoltMetaDataFileWriterIDKey, + []byte(writer.Id()), writer) + if err != nil { + return err + } + } + } + return nil + }) + if err != nil { + return err + } + return nil +} + +func (s *Scorch) removeOldData() { + removed, err := s.removeOldBoltSnapshots() + if err != nil { + s.fireAsyncError(NewScorchError( + persister, + fmt.Sprintf("got err removing old bolt snapshots: %v", err), + ErrCleanup, + )) + } + atomic.AddUint64(&s.stats.TotSnapshotsRemovedFromMetaStore, uint64(removed)) + + err = s.removeOldZapFiles() + if err != nil { + s.fireAsyncError(NewScorchError( + persister, + fmt.Sprintf("got err removing old zap files: %v", err), + ErrCleanup, + )) + } +} + +func getTimeSeriesSnapshots(maxDataPoints int, interval time.Duration, + snapshots []*snapshotMetaData) map[uint64]time.Time { + if interval == 0 || len(snapshots) == 0 || maxDataPoints <= 0 { + return map[uint64]time.Time{} + } + // the map containing the time series snapshots, i.e the timeseries of snapshots + // each of which is separated by rollbackSamplingInterval + rv := make(map[uint64]time.Time, maxDataPoints) + // the last point in the "time series", i.e. the timeseries of snapshots + // each of which is separated by rollbackSamplingInterval + ptr := len(snapshots) - 1 + rv[snapshots[ptr].epoch] = snapshots[ptr].timeStamp + numSnapshotsProtected := 1 + // traverse the list in reverse order, older timestamps to newer ones. + for i := ptr - 1; i >= 0 && numSnapshotsProtected < maxDataPoints; i-- { + sinceLast := snapshots[i].timeStamp.Sub(snapshots[ptr].timeStamp) + if sinceLast >= interval { + // capture the snapshot at the interval boundary: the exact match + // if there is one, otherwise the older neighbour (i+1), which was + // the last snapshot seen before the interval was crossed + idx := i + if sinceLast > interval { + idx = i + 1 + } + if _, ok := rv[snapshots[idx].epoch]; !ok { + rv[snapshots[idx].epoch] = snapshots[idx].timeStamp + ptr = idx + numSnapshotsProtected++ + } + } + } + return rv +} + +// getProtectedSnapshots aims to fetch the epochs keep based on a timestamp basis. +// It tries to get NumSnapshotsToKeep snapshots, each of which are separated +// by a time duration of RollbackSamplingInterval. +func (s *Scorch) getProtectedSnapshots(liveSnapshots []*snapshotMetaData) map[uint64]time.Time { + // keep numSnapshotsToKeep - 1 worth of time series snapshots, because we always + // must preserve the very latest snapshot in bolt as well to avoid accidental + // deletes of bolt entries and cleanups by the purger code. + protectedEpochs := getTimeSeriesSnapshots(s.numSnapshotsToKeep-1, + s.rollbackSamplingInterval, liveSnapshots) + numProtected := len(protectedEpochs) + // always protect the latest snapshot + latestSnapshot := liveSnapshots[0] + if _, ok := protectedEpochs[latestSnapshot.epoch]; !ok { + protectedEpochs[latestSnapshot.epoch] = latestSnapshot.timeStamp + numProtected++ + } + // if we still have not protected enough snapshots, then protect the next most recent snapshots + for i := 1; i < len(liveSnapshots) && numProtected < s.numSnapshotsToKeep; i++ { + if _, ok := protectedEpochs[liveSnapshots[i].epoch]; !ok { + protectedEpochs[liveSnapshots[i].epoch] = liveSnapshots[i].timeStamp + numProtected++ + } + } + return protectedEpochs +} + +func newCheckPoints(snapshots map[uint64]time.Time) []*snapshotMetaData { + rv := make([]*snapshotMetaData, 0, len(snapshots)) + + keys := make([]uint64, 0, len(snapshots)) + for k := range snapshots { + keys = append(keys, k) + } + + sort.SliceStable(keys, func(i, j int) bool { + return snapshots[keys[i]].After(snapshots[keys[j]]) + }) + + for _, key := range keys { + rv = append(rv, &snapshotMetaData{ + epoch: key, + timeStamp: snapshots[key], + }) + } + + return rv +} + +func (s *Scorch) removeOldBoltSnapshots() (numRemoved int, err error) { + // first get the set of live snapshots + liveSnapshots, err := s.getLiveSnapshots() + if err != nil { + return 0, err + } + + // if no live snapshots, then nothing to do + if len(liveSnapshots) == 0 { + return 0, nil + } + + // then get the set of protected snapshots, which are the ones we want to keep + protectedSnapshots := s.getProtectedSnapshots(liveSnapshots) + + var epochsToRemove []uint64 + var newEligible []uint64 + s.rootLock.Lock() + for _, epoch := range s.eligibleForRemoval { + if _, ok := protectedSnapshots[epoch]; ok { + // protected + newEligible = append(newEligible, epoch) + } else { + epochsToRemove = append(epochsToRemove, epoch) + } + } + s.eligibleForRemoval = newEligible + s.rootLock.Unlock() + s.checkPoints = newCheckPoints(protectedSnapshots) + + if len(epochsToRemove) == 0 { + return 0, nil + } + + tx, err := s.rootBolt.Begin(true) + if err != nil { + return 0, err + } + defer func() { + if err == nil { + err = tx.Commit() + } else { + _ = tx.Rollback() + } + if err == nil { + err = s.rootBolt.Sync() + } + }() + + snapshots := tx.Bucket(util.BoltSnapshotsBucket) + if snapshots == nil { + return 0, nil + } + + for _, epochToRemove := range epochsToRemove { + k := encodeUvarintAscending(nil, epochToRemove) + err = snapshots.DeleteBucket(k) + if err == bolt.ErrBucketNotFound { + err = nil + } + if err == nil { + numRemoved++ + } + } + + return numRemoved, err +} + +func (s *Scorch) maxSegmentIDOnDisk() (uint64, error) { + files, err := os.ReadDir(s.path) + if err != nil { + return 0, err + } + + var rv uint64 + for _, f := range files { + fname := f.Name() + if filepath.Ext(fname) == ".zap" { + prefix := strings.TrimSuffix(fname, ".zap") + id, err2 := strconv.ParseUint(prefix, 16, 64) + if err2 != nil { + return 0, err2 + } + if id > rv { + rv = id + } + } + } + return rv, err +} + +// Removes any *.zap files which aren't listed in the rootBolt. +func (s *Scorch) removeOldZapFiles() error { + liveFileNames, err := s.loadZapFileNames() + if err != nil { + return err + } + + files, err := os.ReadDir(s.path) + if err != nil { + return err + } + + s.rootLock.RLock() + + for _, f := range files { + fname := f.Name() + if filepath.Ext(fname) == ".zap" { + if _, exists := liveFileNames[fname]; !exists && !s.ineligibleForRemoval[fname] && (s.copyScheduled[fname] <= 0) { + err := os.Remove(s.path + string(os.PathSeparator) + fname) + if err != nil { + log.Printf("got err removing file: %s, err: %v", fname, err) + } + } + } + } + + s.rootLock.RUnlock() + + return nil +} + +func (s *Scorch) getBoundaryCheckPoint(timeStamp time.Time) time.Time { + numCheckpoints := float64(len(s.checkPoints)) + if numCheckpoints == 0 { + return timeStamp + } + checkpointIdx := int(math.Floor(numCheckpoints * s.rollbackRetentionFactor)) + if checkpointIdx >= len(s.checkPoints) { + checkpointIdx = len(s.checkPoints) - 1 + } + boundary := s.checkPoints[checkpointIdx] + if boundary.timeStamp.Before(timeStamp) { + return boundary.timeStamp + } + return timeStamp +} + +type snapshotMetaData struct { + epoch uint64 + timeStamp time.Time +} + +// returns all the snapshots that are currently persisted +// in the rootBolt, sorted by latest to oldest epoch. +func (s *Scorch) rootBoltSnapshotMetaData() ([]*snapshotMetaData, error) { + var rv []*snapshotMetaData + err := s.rootBolt.View(func(tx *util.BoltTxImpl) error { + snapshots := tx.Bucket(util.BoltSnapshotsBucket) + if snapshots == nil { + return nil + } + sc := snapshots.Cursor() + for sk, _ := sc.Last(); sk != nil; sk, _ = sc.Prev() { + _, snapshotEpoch, err := decodeUvarintAscending(sk) + if err != nil { + continue + } + snapshot := snapshots.GetBucket(sk) + if snapshot == nil { + continue + } + metaBucket := snapshot.GetBucket(util.BoltMetaDataKey) + if metaBucket == nil { + continue + } + timeStampBytes, err := metaBucket.Get(util.BoltMetaDataTimeStamp, nil) + if err != nil { + continue + } + var timeStamp time.Time + err = timeStamp.UnmarshalText(timeStampBytes) + if err != nil { + continue + } + meta := &snapshotMetaData{ + epoch: snapshotEpoch, + timeStamp: timeStamp, + } + rv = append(rv, meta) + } + return nil + }) + if err != nil { + return nil, err + } + return rv, nil +} + +func (s *Scorch) getLiveSnapshots() ([]*snapshotMetaData, error) { + // get all the snapshots that are currently persisted + meta, err := s.rootBoltSnapshotMetaData() + if err != nil { + return nil, err + } + // if none persisted, then nothing to do + if len(meta) == 0 { + return nil, nil + } + // check if we have a rollback sampling interval, if not, + // then we will just return the latest numSnapshotsToKeep snapshots + if s.rollbackSamplingInterval <= 0 { + if len(meta) <= s.numSnapshotsToKeep { + return meta, nil + } + return meta[:s.numSnapshotsToKeep], nil + } + // we consider a snapshot to be live if: + // 1. it is the latest snapshot + // 2. it is within our expiration duration + var liveSnapshots []*snapshotMetaData + // always keep the latest snapshot + liveSnapshots = append(liveSnapshots, meta[0]) + extraSnapshots := s.numSnapshotsToKeep - 1 + if extraSnapshots <= 0 { + return liveSnapshots, nil + } + // if we need extra snapshots, compute an expiration duration + // beyond which we will not consider snapshots to be live + currTime := time.Now() + expirationDuration := time.Duration(extraSnapshots) * s.rollbackSamplingInterval + cutoffTime := currTime.Add(-expirationDuration) + // if we have previous checkpoints, then we can extend + // the cutoff time based on the boundary checkpoint + for _, snapshot := range meta[1:] { + if snapshot.timeStamp.Before(cutoffTime) { + boundary := s.getBoundaryCheckPoint(snapshot.timeStamp) + if boundary.Before(snapshot.timeStamp) { + cutoffTime = boundary + } + break + } + } + // add all snapshots that are newer than the cutoff time + for _, snapshot := range meta[1:] { + if !snapshot.timeStamp.Before(cutoffTime) { + liveSnapshots = append(liveSnapshots, snapshot) + } + } + return liveSnapshots, nil +} + +func (s *Scorch) RootBoltSnapshotEpochs() ([]uint64, error) { + var rv []uint64 + err := s.rootBolt.View(func(tx *util.BoltTxImpl) error { + snapshots := tx.Bucket(util.BoltSnapshotsBucket) + if snapshots == nil { + return nil + } + sc := snapshots.Cursor() + for sk, _ := sc.Last(); sk != nil; sk, _ = sc.Prev() { + _, snapshotEpoch, err := decodeUvarintAscending(sk) + if err != nil { + continue + } + rv = append(rv, snapshotEpoch) + } + return nil + }) + return rv, err +} + +// Returns the *.zap file names that are listed in the rootBolt. +func (s *Scorch) loadZapFileNames() (map[string]struct{}, error) { + rv := map[string]struct{}{} + err := s.rootBolt.View(func(tx *util.BoltTxImpl) error { + snapshots := tx.Bucket(util.BoltSnapshotsBucket) + if snapshots == nil { + return nil + } + sc := snapshots.Cursor() + for sk, _ := sc.First(); sk != nil; sk, _ = sc.Next() { + snapshot := snapshots.GetBucket(sk) + if snapshot == nil { + continue + } + segc := snapshot.Cursor() + for segk, _ := segc.First(); segk != nil; segk, _ = segc.Next() { + if segk[0] == util.BoltInternalKey[0] { + continue + } + segmentBucket := snapshot.GetBucket(segk) + if segmentBucket == nil { + continue + } + pathBytes, err := segmentBucket.Get(util.BoltPathKey, nil) + if err != nil { + continue + } + if pathBytes == nil { + continue + } + pathString := string(pathBytes) + rv[string(pathString)] = struct{}{} + } + } + return nil + }) + + return rv, err +} diff --git a/index/scorch/reader_test.go b/index/scorch/reader_test.go new file mode 100644 index 0000000..a523334 --- /dev/null +++ b/index/scorch/reader_test.go @@ -0,0 +1,697 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "context" + "encoding/binary" + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/document" + index "github.com/blevesearch/bleve_index_api" +) + +func TestIndexReader(t *testing.T) { + cfg := CreateConfig("TestIndexReader") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + var expectedCount uint64 + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + doc = document.NewDocument("2") + doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test test test"), testAnalyzer)) + doc.AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("eat more rice"), index.IndexField|index.IncludeTermVectors, testAnalyzer)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + // first look for a term that doesn't exist + reader, err := indexReader.TermFieldReader(context.TODO(), []byte("nope"), "name", true, true, true) + if err != nil { + t.Errorf("Error accessing term field reader: %v", err) + } + count := reader.Count() + if count != 0 { + t.Errorf("Expected doc count to be: %d got: %d", 0, count) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + reader, err = indexReader.TermFieldReader(context.TODO(), []byte("test"), "name", true, true, true) + if err != nil { + t.Errorf("Error accessing term field reader: %v", err) + } + + count = reader.Count() + if count != expectedCount { + t.Errorf("Expected doc count to be: %d got: %d", expectedCount, count) + } + + var match *index.TermFieldDoc + var actualCount uint64 + match, err = reader.Next(nil) + for err == nil && match != nil { + match, err = reader.Next(nil) + if err != nil { + t.Errorf("unexpected error reading next") + } + actualCount++ + } + if actualCount != count { + t.Errorf("count was 2, but only saw %d", actualCount) + } + + internalIDBogus, err := indexReader.InternalID("a-bogus-docId") + if err != nil { + t.Fatal(err) + } + if internalIDBogus != nil { + t.Errorf("expected bogus docId to have nil InternalID") + } + + internalID2, err := indexReader.InternalID("2") + if err != nil { + t.Fatal(err) + } + expectedMatch := &index.TermFieldDoc{ + ID: internalID2, + Freq: 1, + Norm: 0.5773502588272095, + Vectors: []*index.TermFieldVector{ + { + Field: "desc", + Pos: 3, + Start: 9, + End: 13, + }, + }, + } + tfr, err := indexReader.TermFieldReader(context.TODO(), []byte("rice"), "desc", true, true, true) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + match, err = tfr.Next(nil) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if !reflect.DeepEqual(expectedMatch, match) { + t.Errorf("got %#v, expected %#v", match, expectedMatch) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + // now test usage of advance + reader, err = indexReader.TermFieldReader(context.TODO(), []byte("test"), "name", true, true, true) + if err != nil { + t.Errorf("Error accessing term field reader: %v", err) + } + + match, err = reader.Advance(internalID2, nil) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if match == nil { + t.Fatalf("Expected match, got nil") + } + if !match.ID.Equals(internalID2) { + t.Errorf("Expected ID '2', got '%s'", match.ID) + } + // have to manually construct bogus id, because it doesn't exist + internalID3 := make([]byte, 8) + binary.BigEndian.PutUint64(internalID3, 3) + match, err = reader.Advance(index.IndexInternalID(internalID3), nil) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if match != nil { + t.Errorf("expected nil, got %v", match) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + // now test creating a reader for a field that doesn't exist + reader, err = indexReader.TermFieldReader(context.TODO(), []byte("water"), "doesnotexist", true, true, true) + if err != nil { + t.Errorf("Error accessing term field reader: %v", err) + } + count = reader.Count() + if count != 0 { + t.Errorf("expected count 0 for reader of non-existent field") + } + match, err = reader.Next(nil) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if match != nil { + t.Errorf("expected nil, got %v", match) + } + match, err = reader.Advance(index.IndexInternalID("anywhere"), nil) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if match != nil { + t.Errorf("expected nil, got %v", match) + } +} + +func TestIndexDocIdReader(t *testing.T) { + cfg := CreateConfig("TestIndexDocIdReader") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + var expectedCount uint64 + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + doc = document.NewDocument("2") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test test test"))) + doc.AddField(document.NewTextFieldWithIndexingOptions("desc", []uint64{}, []byte("eat more rice"), index.IndexField|index.IncludeTermVectors)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Error(err) + } + }() + + // first get all doc ids + reader, err := indexReader.DocIDReaderAll() + if err != nil { + t.Errorf("Error accessing doc id reader: %v", err) + } + defer func() { + err := reader.Close() + if err != nil { + t.Fatal(err) + } + }() + + id, err := reader.Next() + if err != nil { + t.Fatal(err) + } + + count := uint64(0) + for id != nil { + count++ + id, err = reader.Next() + if err != nil { + t.Fatal(err) + } + } + if count != expectedCount { + t.Errorf("expected %d, got %d", expectedCount, count) + } + + // try it again, but jump to the second doc this time + reader2, err := indexReader.DocIDReaderAll() + if err != nil { + t.Errorf("Error accessing doc id reader: %v", err) + } + defer func() { + err := reader2.Close() + if err != nil { + t.Error(err) + } + }() + + internalID2, err := indexReader.InternalID("2") + if err != nil { + t.Fatal(err) + } + + id, err = reader2.Advance(internalID2) + if err != nil { + t.Error(err) + } + if !id.Equals(internalID2) { + t.Errorf("expected to find id '2', got '%s'", id) + } + + // again 3 doesn't exist cannot use internal id for 3 as there is none + // the important aspect is that this id doesn't exist, so its ok + id, err = reader2.Advance(index.IndexInternalID("3")) + if err != nil { + t.Error(err) + } + if id != nil { + t.Errorf("expected to find id '', got '%s'", id) + } +} + +func TestIndexDocIdOnlyReader(t *testing.T) { + cfg := CreateConfig("TestIndexDocIdOnlyReader") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + doc := document.NewDocument("1") + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + doc = document.NewDocument("3") + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + doc = document.NewDocument("5") + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + doc = document.NewDocument("7") + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + doc = document.NewDocument("9") + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Error(err) + } + }() + + onlyIds := []string{"1", "5", "9"} + reader, err := indexReader.DocIDReaderOnly(onlyIds) + if err != nil { + t.Errorf("Error accessing doc id reader: %v", err) + } + defer func() { + err := reader.Close() + if err != nil { + t.Fatal(err) + } + }() + + id, err := reader.Next() + if err != nil { + t.Fatal(err) + } + + count := uint64(0) + for id != nil { + count++ + id, err = reader.Next() + if err != nil { + t.Fatal(err) + } + } + if count != 3 { + t.Errorf("expected 3, got %d", count) + } + + // commented out because advance works with internal ids + // this test presumes we see items in external doc id order + // which is no longer the case, so simply converting external ids + // to internal ones is not logically correct + // not removing though because we need some way to test Advance() + + // // try it again, but jump + // reader2, err := indexReader.DocIDReaderOnly(onlyIds) + // if err != nil { + // t.Errorf("Error accessing doc id reader: %v", err) + // } + // defer func() { + // err := reader2.Close() + // if err != nil { + // t.Error(err) + // } + // }() + // + // id, err = reader2.Advance(index.IndexInternalID("5")) + // if err != nil { + // t.Error(err) + // } + // if !id.Equals(index.IndexInternalID("5")) { + // t.Errorf("expected to find id '5', got '%s'", id) + // } + // + // id, err = reader2.Advance(index.IndexInternalID("a")) + // if err != nil { + // t.Error(err) + // } + // if id != nil { + // t.Errorf("expected to find id '', got '%s'", id) + // } + + // some keys aren't actually there + onlyIds = []string{"0", "2", "4", "5", "6", "8", "a"} + reader3, err := indexReader.DocIDReaderOnly(onlyIds) + if err != nil { + t.Errorf("Error accessing doc id reader: %v", err) + } + defer func() { + err := reader3.Close() + if err != nil { + t.Error(err) + } + }() + + id, err = reader3.Next() + if err != nil { + t.Fatal(err) + } + + count = uint64(0) + for id != nil { + count++ + id, err = reader3.Next() + if err != nil { + t.Fatal(err) + } + } + if count != 1 { + t.Errorf("expected 1, got %d", count) + } + + // commented out because advance works with internal ids + // this test presumes we see items in external doc id order + // which is no longer the case, so simply converting external ids + // to internal ones is not logically correct + // not removing though because we need some way to test Advance() + + // // mix advance and next + // onlyIds = []string{"0", "1", "3", "5", "6", "9"} + // reader4, err := indexReader.DocIDReaderOnly(onlyIds) + // if err != nil { + // t.Errorf("Error accessing doc id reader: %v", err) + // } + // defer func() { + // err := reader4.Close() + // if err != nil { + // t.Error(err) + // } + // }() + // + // // first key is "1" + // id, err = reader4.Next() + // if err != nil { + // t.Error(err) + // } + // if !id.Equals(index.IndexInternalID("1")) { + // t.Errorf("expected to find id '1', got '%s'", id) + // } + // + // // advancing to key we dont have gives next + // id, err = reader4.Advance(index.IndexInternalID("2")) + // if err != nil { + // t.Error(err) + // } + // if !id.Equals(index.IndexInternalID("3")) { + // t.Errorf("expected to find id '3', got '%s'", id) + // } + // + // // next after advance works + // id, err = reader4.Next() + // if err != nil { + // t.Error(err) + // } + // if !id.Equals(index.IndexInternalID("5")) { + // t.Errorf("expected to find id '5', got '%s'", id) + // } + // + // // advancing to key we do have works + // id, err = reader4.Advance(index.IndexInternalID("9")) + // if err != nil { + // t.Error(err) + // } + // if !id.Equals(index.IndexInternalID("9")) { + // t.Errorf("expected to find id '9', got '%s'", id) + // } + // + // // advance backwards at end + // id, err = reader4.Advance(index.IndexInternalID("4")) + // if err != nil { + // t.Error(err) + // } + // if !id.Equals(index.IndexInternalID("5")) { + // t.Errorf("expected to find id '5', got '%s'", id) + // } + // + // // next after advance works + // id, err = reader4.Next() + // if err != nil { + // t.Error(err) + // } + // if !id.Equals(index.IndexInternalID("9")) { + // t.Errorf("expected to find id '9', got '%s'", id) + // } + // + // // advance backwards to key that exists, but not in only set + // id, err = reader4.Advance(index.IndexInternalID("7")) + // if err != nil { + // t.Error(err) + // } + // if !id.Equals(index.IndexInternalID("9")) { + // t.Errorf("expected to find id '9', got '%s'", id) + // } +} + +func TestSegmentIndexAndLocalDocNumFromGlobal(t *testing.T) { + tests := []struct { + offsets []uint64 + globalDocNum uint64 + segmentIndex int + localDocNum uint64 + }{ + // just 1 segment + { + offsets: []uint64{0}, + globalDocNum: 0, + segmentIndex: 0, + localDocNum: 0, + }, + { + offsets: []uint64{0}, + globalDocNum: 1, + segmentIndex: 0, + localDocNum: 1, + }, + { + offsets: []uint64{0}, + globalDocNum: 25, + segmentIndex: 0, + localDocNum: 25, + }, + // now 2 segments, 30 docs in first + { + offsets: []uint64{0, 30}, + globalDocNum: 0, + segmentIndex: 0, + localDocNum: 0, + }, + { + offsets: []uint64{0, 30}, + globalDocNum: 1, + segmentIndex: 0, + localDocNum: 1, + }, + { + offsets: []uint64{0, 30}, + globalDocNum: 25, + segmentIndex: 0, + localDocNum: 25, + }, + { + offsets: []uint64{0, 30}, + globalDocNum: 30, + segmentIndex: 1, + localDocNum: 0, + }, + { + offsets: []uint64{0, 30}, + globalDocNum: 35, + segmentIndex: 1, + localDocNum: 5, + }, + // lots of segments + { + offsets: []uint64{0, 30, 40, 70, 99, 172, 800, 25000}, + globalDocNum: 0, + segmentIndex: 0, + localDocNum: 0, + }, + { + offsets: []uint64{0, 30, 40, 70, 99, 172, 800, 25000}, + globalDocNum: 25, + segmentIndex: 0, + localDocNum: 25, + }, + { + offsets: []uint64{0, 30, 40, 70, 99, 172, 800, 25000}, + globalDocNum: 35, + segmentIndex: 1, + localDocNum: 5, + }, + { + offsets: []uint64{0, 30, 40, 70, 99, 172, 800, 25000}, + globalDocNum: 100, + segmentIndex: 4, + localDocNum: 1, + }, + { + offsets: []uint64{0, 30, 40, 70, 99, 172, 800, 25000}, + globalDocNum: 825, + segmentIndex: 6, + localDocNum: 25, + }, + } + + for _, test := range tests { + i := &IndexSnapshot{ + offsets: test.offsets, + refs: 1, + } + gotSegmentIndex, gotLocalDocNum := i.segmentIndexAndLocalDocNumFromGlobal(test.globalDocNum) + if gotSegmentIndex != test.segmentIndex { + t.Errorf("got segment index %d expected %d for offsets %v globalDocNum %d", gotSegmentIndex, test.segmentIndex, test.offsets, test.globalDocNum) + } + if gotLocalDocNum != test.localDocNum { + t.Errorf("got localDocNum %d expected %d for offsets %v globalDocNum %d", gotLocalDocNum, test.localDocNum, test.offsets, test.globalDocNum) + } + err := i.DecRef() + if err != nil { + t.Errorf("expected no err, got: %v", err) + } + } +} diff --git a/index/scorch/regexp.go b/index/scorch/regexp.go new file mode 100644 index 0000000..5a3584f --- /dev/null +++ b/index/scorch/regexp.go @@ -0,0 +1,63 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "regexp/syntax" + + "github.com/blevesearch/vellum/regexp" +) + +func parseRegexp(pattern string) (a *regexp.Regexp, prefixBeg, prefixEnd []byte, err error) { + // TODO: potential optimization where syntax.Regexp supports a Simplify() API? + + parsed, err := syntax.Parse(pattern, syntax.Perl) + if err != nil { + return nil, nil, nil, err + } + + re, err := regexp.NewParsedWithLimit(pattern, parsed, regexp.DefaultLimit) + if err != nil { + return nil, nil, nil, err + } + + prefix := literalPrefix(parsed) + if prefix != "" { + prefixBeg := []byte(prefix) + prefixEnd := calculateExclusiveEndFromPrefix(prefixBeg) + return re, prefixBeg, prefixEnd, nil + } + + return re, nil, nil, nil +} + +// Returns the literal prefix given the parse tree for a regexp +func literalPrefix(s *syntax.Regexp) string { + // traverse the left-most branch in the parse tree as long as the + // node represents a concatenation + for s != nil && s.Op == syntax.OpConcat { + if len(s.Sub) < 1 { + return "" + } + + s = s.Sub[0] + } + + if s.Op == syntax.OpLiteral && (s.Flags&syntax.FoldCase == 0) { + return string(s.Rune) + } + + return "" // no literal prefix +} diff --git a/index/scorch/regexp_test.go b/index/scorch/regexp_test.go new file mode 100644 index 0000000..305b20c --- /dev/null +++ b/index/scorch/regexp_test.go @@ -0,0 +1,57 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "regexp/syntax" + "testing" +) + +func TestLiteralPrefix(t *testing.T) { + tests := []struct { + input, expected string + }{ + {"", ""}, + {"hello", "hello"}, + {"hello.?", "hello"}, + {"hello$", "hello"}, + {`[h][e][l][l][o].*world`, "hello"}, + {`[h-h][e-e][l-l][l-l][o-o].*world`, "hello"}, + {".*", ""}, + {"h.*", "h"}, + {"h.?", "h"}, + {"h[a-z]", "h"}, + {`h\s`, "h"}, + {`(hello)world`, ""}, + {`日本語`, "日本語"}, + {`日本語\w`, "日本語"}, + {`^hello`, ""}, + {`^`, ""}, + {`$`, ""}, + {`(?i)mArTy`, ""}, + } + + for i, test := range tests { + s, err := syntax.Parse(test.input, syntax.Perl) + if err != nil { + t.Fatalf("expected no syntax.Parse error, got: %v", err) + } + + got := literalPrefix(s) + if test.expected != got { + t.Fatalf("test: %d, %+v, got: %s", i, test, got) + } + } +} diff --git a/index/scorch/rollback.go b/index/scorch/rollback.go new file mode 100644 index 0000000..008b364 --- /dev/null +++ b/index/scorch/rollback.go @@ -0,0 +1,230 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + "log" + "os" + + "github.com/blevesearch/bleve/v2/util" + bolt "go.etcd.io/bbolt" +) + +type RollbackPoint struct { + epoch uint64 + meta map[string][]byte +} + +func (r *RollbackPoint) GetInternal(key []byte) []byte { + return r.meta[string(key)] +} + +// RollbackPoints returns an array of rollback points available for +// the application to rollback to, with more recent rollback points +// (higher epochs) coming first. +func RollbackPoints(path string) ([]*RollbackPoint, error) { + if len(path) == 0 { + return nil, fmt.Errorf("RollbackPoints: invalid path") + } + + rootBoltPath := path + string(os.PathSeparator) + "root.bolt" + rootBoltOpt := &bolt.Options{ + ReadOnly: true, + } + rootBolt, err := util.OpenBolt(rootBoltPath, 0600, rootBoltOpt) + if err != nil || rootBolt == nil { + return nil, err + } + + // start a read-only bolt transaction + tx, err := rootBolt.Begin(false) + if err != nil { + return nil, fmt.Errorf("RollbackPoints: failed to start" + + " read-only transaction") + } + + // read-only bolt transactions to be rolled back + defer func() { + _ = tx.Rollback() + _ = rootBolt.Close() + }() + + snapshots := tx.Bucket(util.BoltSnapshotsBucket) + if snapshots == nil { + return nil, nil + } + + rollbackPoints := []*RollbackPoint{} + + c1 := snapshots.Cursor() + for k, _ := c1.Last(); k != nil; k, _ = c1.Prev() { + _, snapshotEpoch, err := decodeUvarintAscending(k) + if err != nil { + log.Printf("RollbackPoints:"+ + " unable to parse segment epoch %x, continuing", k) + continue + } + + snapshot := snapshots.GetBucket(k) + if snapshot == nil { + log.Printf("RollbackPoints:"+ + " snapshot key, but bucket missing %x, continuing", k) + continue + } + + metaBucket := snapshot.GetBucket(util.BoltMetaDataKey) + if metaBucket == nil { + return nil, fmt.Errorf("meta-data bucket missing") + } + + fileWriterID, err := metaBucket.Get(util.BoltMetaDataFileWriterIDKey, nil) + if err != nil { + return nil, fmt.Errorf("unable to load file writer ID: %v", err) + } + reader, err := util.NewFileReader(string(fileWriterID), []byte(rootBoltPath)) + if err != nil { + return nil, fmt.Errorf("unable to load correct reader: %v", err) + } + + meta := map[string][]byte{} + c2 := snapshot.Cursor() + for j, _ := c2.First(); j != nil; j, _ = c2.Next() { + if j[0] == util.BoltInternalKey[0] { + internalBucket := snapshot.GetBucket(j) + if internalBucket == nil { + err = fmt.Errorf("internal bucket missing") + break + } + err = internalBucket.ForEach(func(key []byte, val []byte) error { + meta[string(key)] = val + return nil + }, reader) + if err != nil { + break + } + } + } + + if err != nil { + log.Printf("RollbackPoints:"+ + " failed in fetching internal data: %v", err) + continue + } + + rollbackPoints = append(rollbackPoints, &RollbackPoint{ + epoch: snapshotEpoch, + meta: meta, + }) + } + + return rollbackPoints, nil +} + +// Rollback atomically and durably brings the store back to the point +// in time as represented by the RollbackPoint. +// Rollback() should only be passed a RollbackPoint that came from the +// same store using the RollbackPoints() API along with the index path. +func Rollback(path string, to *RollbackPoint) error { + if to == nil { + return fmt.Errorf("Rollback: RollbackPoint is nil") + } + if len(path) == 0 { + return fmt.Errorf("Rollback: index path is empty") + } + + rootBoltPath := path + string(os.PathSeparator) + "root.bolt" + rootBoltOpt := &bolt.Options{ + ReadOnly: false, + } + rootBolt, err := util.OpenBolt(rootBoltPath, 0600, rootBoltOpt) + if err != nil || rootBolt == nil { + return err + } + defer func() { + err1 := rootBolt.Close() + if err1 != nil && err == nil { + err = err1 + } + }() + + // pick all the younger persisted epochs in bolt store + // including the target one. + var found bool + var eligibleEpochs []uint64 + err = rootBolt.View(func(tx *util.BoltTxImpl) error { + snapshots := tx.Bucket(util.BoltSnapshotsBucket) + if snapshots == nil { + return nil + } + sc := snapshots.Cursor() + for sk, _ := sc.Last(); sk != nil && !found; sk, _ = sc.Prev() { + _, snapshotEpoch, err := decodeUvarintAscending(sk) + if err != nil { + continue + } + if snapshotEpoch == to.epoch { + found = true + } + eligibleEpochs = append(eligibleEpochs, snapshotEpoch) + } + return nil + }) + + if len(eligibleEpochs) == 0 { + return fmt.Errorf("Rollback: no persisted epochs found in bolt") + } + if !found { + return fmt.Errorf("Rollback: target epoch %d not found in bolt", to.epoch) + } + + // start a write transaction + tx, err := rootBolt.Begin(true) + if err != nil { + return err + } + + defer func() { + if err == nil { + err = tx.Commit() + } else { + _ = tx.Rollback() + } + if err == nil { + err = rootBolt.Sync() + } + }() + + snapshots := tx.Bucket(util.BoltSnapshotsBucket) + if snapshots == nil { + return nil + } + for _, epoch := range eligibleEpochs { + k := encodeUvarintAscending(nil, epoch) + if err != nil { + continue + } + if epoch == to.epoch { + // return here as it already processed until the given epoch + return nil + } + err = snapshots.DeleteBucket(k) + if err == bolt.ErrBucketNotFound { + err = nil + } + } + + return err +} diff --git a/index/scorch/rollback_test.go b/index/scorch/rollback_test.go new file mode 100644 index 0000000..a39e718 --- /dev/null +++ b/index/scorch/rollback_test.go @@ -0,0 +1,1010 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "io" + "math/rand" + "os" + "path/filepath" + "strconv" + "sync/atomic" + "testing" + "time" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +func TestIndexRollback(t *testing.T) { + cfg := CreateConfig("TestIndexRollback") + numSnapshotsToKeepOrig := NumSnapshotsToKeep + NumSnapshotsToKeep = 1000 + + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + NumSnapshotsToKeep = numSnapshotsToKeepOrig + + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + + _, ok := idx.(*Scorch) + if !ok { + t.Fatalf("Not a scorch index?") + } + + indexPath, _ := cfg["path"].(string) + // should have no rollback points initially + rollbackPoints, err := RollbackPoints(indexPath) + if err == nil { + t.Fatalf("expected no err, got: %v, %d", err, len(rollbackPoints)) + } + if len(rollbackPoints) != 0 { + t.Fatalf("expected no rollbackPoints, got %d", len(rollbackPoints)) + } + + err = idx.Open() + if err != nil { + t.Fatal(err) + } + // create a batch, insert 2 new documents + batch := index.NewBatch() + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test1"))) + batch.Update(doc) + doc = document.NewDocument("2") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test2"))) + batch.Update(doc) + + err = idx.Batch(batch) + if err != nil { + t.Fatal(err) + } + + readerSlow, err := idx.Reader() // keep snapshot around so it's not cleaned up + if err != nil { + t.Fatal(err) + } + defer func() { + _ = readerSlow.Close() + }() + + err = idx.Close() + if err != nil { + t.Fatal(err) + } + + // fetch rollback points after first batch + rollbackPoints, err = RollbackPoints(indexPath) + if err != nil { + t.Fatalf("expected no err, got: %v, %d", err, len(rollbackPoints)) + } + if len(rollbackPoints) == 0 { + t.Fatalf("expected some rollbackPoints, got none") + } + + // set this as a rollback point for the future + rollbackPoint := rollbackPoints[0] + + err = idx.Open() + if err != nil { + t.Fatal(err) + } + // create another batch, insert 2 new documents, and delete an existing one + batch = index.NewBatch() + doc = document.NewDocument("3") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test3"))) + batch.Update(doc) + doc = document.NewDocument("4") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test4"))) + batch.Update(doc) + batch.Delete("1") + + err = idx.Batch(batch) + if err != nil { + t.Fatal(err) + } + + err = idx.Close() + if err != nil { + t.Fatal(err) + } + + rollbackPointsB, err := RollbackPoints(indexPath) + if err != nil || len(rollbackPointsB) <= len(rollbackPoints) { + t.Fatalf("expected no err, got: %v, %d", err, len(rollbackPointsB)) + } + + found := false + for _, p := range rollbackPointsB { + if rollbackPoint.epoch == p.epoch { + found = true + } + } + if !found { + t.Fatalf("expected rollbackPoint epoch to still be available") + } + + err = idx.Open() + if err != nil { + t.Fatal(err) + } + + reader, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + + docCount, err := reader.DocCount() + if err != nil { + t.Fatal(err) + } + + // expect docs 2, 3, 4 + if docCount != 3 { + t.Fatalf("unexpected doc count: %v", docCount) + } + ret, err := reader.Document("1") + if err != nil || ret != nil { + t.Fatal(ret, err) + } + ret, err = reader.Document("2") + if err != nil || ret == nil { + t.Fatal(ret, err) + } + ret, err = reader.Document("3") + if err != nil || ret == nil { + t.Fatal(ret, err) + } + ret, err = reader.Document("4") + if err != nil || ret == nil { + t.Fatal(ret, err) + } + + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + err = idx.Close() + if err != nil { + t.Fatal(err) + } + + // rollback to a non existing rollback point + err = Rollback(indexPath, &RollbackPoint{epoch: 100}) + if err == nil { + t.Fatalf("expected err: Rollback: target epoch 100 not found in bolt") + } + + // rollback to the selected rollback point + err = Rollback(indexPath, rollbackPoint) + if err != nil { + t.Fatal(err) + } + + err = idx.Open() + if err != nil { + t.Fatal(err) + } + + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + + docCount, err = reader.DocCount() + if err != nil { + t.Fatal(err) + } + + // expect only docs 1, 2 + if docCount != 2 { + t.Fatalf("unexpected doc count: %v", docCount) + } + ret, err = reader.Document("1") + if err != nil || ret == nil { + t.Fatal(ret, err) + } + ret, err = reader.Document("2") + if err != nil || ret == nil { + t.Fatal(ret, err) + } + ret, err = reader.Document("3") + if err != nil || ret != nil { + t.Fatal(ret, err) + } + ret, err = reader.Document("4") + if err != nil || ret != nil { + t.Fatal(ret, err) + } + + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + err = idx.Close() + if err != nil { + t.Fatal(err) + } +} + +func openTestBolt(t *testing.T, name string) *util.RootBoltImpl { + t.Helper() + cfg := CreateConfig(name) + if err := InitTest(cfg); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := DestroyTest(cfg); err != nil { + t.Error(err) + } + }) + path := cfg["path"].(string) + if err := os.MkdirAll(path, 0o700); err != nil { + t.Fatal(err) + } + rootBolt, err := util.OpenBolt(filepath.Join(path, "root.bolt"), 0o600, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := rootBolt.Close(); err != nil { + t.Error(err) + } + }) + return rootBolt +} + +func TestGetProtectedSnapshots(t *testing.T) { + rootBolt := openTestBolt(t, "TestGetProtectedSnapshots") + interval := 10 * time.Minute + s := &Scorch{ + rootBolt: rootBolt, + rollbackSamplingInterval: interval, + } + currentTimeStamp := time.Now() + tests := []struct { + title string + metaData []*snapshotMetaData + numSnapshotsToKeep int + expCount int + expEpochs []uint64 + }{ + { + title: "epochs that have exact timestamps as per expectation for protecting", + metaData: []*snapshotMetaData{ + {epoch: 100, timeStamp: currentTimeStamp}, + {epoch: 99, timeStamp: currentTimeStamp.Add(-(interval / 12))}, + {epoch: 88, timeStamp: currentTimeStamp.Add(-(interval / 6))}, + {epoch: 50, timeStamp: currentTimeStamp.Add(-(interval))}, + {epoch: 35, timeStamp: currentTimeStamp.Add(-(6 * interval / 5))}, + {epoch: 10, timeStamp: currentTimeStamp.Add(-(2 * interval))}, + }, + numSnapshotsToKeep: 3, + expCount: 3, + expEpochs: []uint64{100, 50, 10}, + }, + { + title: "epochs that have exact timestamps as per expectation for protecting", + metaData: []*snapshotMetaData{ + {epoch: 100, timeStamp: currentTimeStamp}, + {epoch: 99, timeStamp: currentTimeStamp.Add(-(interval / 12))}, + {epoch: 88, timeStamp: currentTimeStamp.Add(-(interval / 6))}, + {epoch: 50, timeStamp: currentTimeStamp.Add(-(interval))}, + }, + numSnapshotsToKeep: 2, + expCount: 2, + expEpochs: []uint64{100, 50}, + }, + { + title: "epochs that have timestamps approximated to the expected value, " + + "always retain the latest one", + metaData: []*snapshotMetaData{ + {epoch: 100, timeStamp: currentTimeStamp}, + {epoch: 99, timeStamp: currentTimeStamp.Add(-(interval / 12))}, + {epoch: 88, timeStamp: currentTimeStamp.Add(-(interval / 6))}, + {epoch: 50, timeStamp: currentTimeStamp.Add(-(3 * interval / 4))}, + {epoch: 35, timeStamp: currentTimeStamp.Add(-(6 * interval / 5))}, + {epoch: 10, timeStamp: currentTimeStamp.Add(-(2 * interval))}, + }, + numSnapshotsToKeep: 3, + expCount: 3, + expEpochs: []uint64{100, 35, 10}, + }, + { + title: "protecting epochs when we don't have enough snapshots with rollbackSamplingInterval" + + " separated timestamps", + metaData: []*snapshotMetaData{ + {epoch: 100, timeStamp: currentTimeStamp}, + {epoch: 99, timeStamp: currentTimeStamp.Add(-(interval / 12))}, + {epoch: 88, timeStamp: currentTimeStamp.Add(-(interval / 6))}, + {epoch: 50, timeStamp: currentTimeStamp.Add(-(3 * interval / 4))}, + {epoch: 35, timeStamp: currentTimeStamp.Add(-(5 * interval / 6))}, + {epoch: 10, timeStamp: currentTimeStamp.Add(-(7 * interval / 8))}, + }, + numSnapshotsToKeep: 4, + expCount: 4, + expEpochs: []uint64{100, 99, 88, 10}, + }, + { + title: "epochs of which some are approximated to the expected timestamps, and" + + " we don't have enough snapshots with rollbackSamplingInterval separated timestamps", + metaData: []*snapshotMetaData{ + {epoch: 100, timeStamp: currentTimeStamp}, + {epoch: 99, timeStamp: currentTimeStamp.Add(-(interval / 12))}, + {epoch: 88, timeStamp: currentTimeStamp.Add(-(interval / 6))}, + {epoch: 50, timeStamp: currentTimeStamp.Add(-(3 * interval / 4))}, + {epoch: 35, timeStamp: currentTimeStamp.Add(-(8 * interval / 7))}, + {epoch: 10, timeStamp: currentTimeStamp.Add(-(6 * interval / 5))}, + }, + numSnapshotsToKeep: 3, + expCount: 3, + expEpochs: []uint64{100, 50, 10}, + }, + { + title: "sparse time series with a recent snapshot must still retain " + + "numSnapshotsToKeep by filling the skipped middle snapshots", + metaData: []*snapshotMetaData{ + {epoch: 100, timeStamp: currentTimeStamp}, + {epoch: 99, timeStamp: currentTimeStamp.Add(-(6 * interval / 10))}, + {epoch: 88, timeStamp: currentTimeStamp.Add(-(7 * interval / 10))}, + {epoch: 77, timeStamp: currentTimeStamp.Add(-(8 * interval / 10))}, + {epoch: 10, timeStamp: currentTimeStamp.Add(-(15 * interval / 10))}, + }, + numSnapshotsToKeep: 4, + expCount: 4, + expEpochs: []uint64{100, 99, 88, 10}, + }, + { + title: "latest snapshot exactly one interval from the oldest must " + + "still retain numSnapshotsToKeep", + metaData: []*snapshotMetaData{ + {epoch: 100, timeStamp: currentTimeStamp}, + {epoch: 99, timeStamp: currentTimeStamp.Add(-(interval / 2))}, + {epoch: 10, timeStamp: currentTimeStamp.Add(-(interval))}, + }, + numSnapshotsToKeep: 3, + expCount: 3, + expEpochs: []uint64{100, 99, 10}, + }, + { + title: "numSnapshotsToKeep=2 with a boundary-spaced older snapshot must " + + "protect exactly the latest and the oldest", + metaData: []*snapshotMetaData{ + {epoch: 100, timeStamp: currentTimeStamp}, + {epoch: 99, timeStamp: currentTimeStamp.Add(-(interval / 12))}, + {epoch: 50, timeStamp: currentTimeStamp.Add(-(2 * interval))}, + {epoch: 10, timeStamp: currentTimeStamp.Add(-(3 * interval))}, + }, + numSnapshotsToKeep: 2, + expCount: 2, + expEpochs: []uint64{100, 10}, + }, + } + for i, test := range tests { + s.numSnapshotsToKeep = test.numSnapshotsToKeep + protectedEpochs := s.getProtectedSnapshots(test.metaData) + if len(protectedEpochs) != test.expCount { + t.Errorf("test %d: %s, getProtectedSnapshots expected to return %d "+ + "snapshots, but got: %d", i, test.title, test.expCount, len(protectedEpochs)) + } + for _, e := range test.expEpochs { + if _, found := protectedEpochs[e]; !found { + t.Errorf("test %d: %s, %d epoch expected to be protected, "+ + "but missing from protected list: %v", i, test.title, e, protectedEpochs) + } + } + } +} + +// updateRoot creates a new snapshot, updates the root +// to point to it, and returns the new snapshot. +// if empty is true, then the new snapshot will be a delete-only +// snapshot, otherwise it will contain a single document. +func updateRoot(t *testing.T, s *Scorch, empty bool) *IndexSnapshot { + t.Helper() + var seg segment.Segment + var ids []string + var err error + if !empty { + doc := genDoc(t) + docs := []index.Document{doc} + ids = []string{doc.ID()} + seg, _, err = s.segPlugin.New(docs) + if err != nil { + t.Fatalf("failed to create segment: %v", err) + } + } else { + seg = nil + ids = []string{strconv.Itoa(rand.Intn(1000000))} + } + intro := &segmentIntroduction{ + id: atomic.AddUint64(&s.nextSegmentID, 1), + data: seg, + ids: ids, + applied: make(chan error), + } + go func(intro *segmentIntroduction) { + s.introduceSegment(intro) + }(intro) + err = <-intro.applied + if err != nil { + t.Fatalf("failed to apply segment introduction: %v", err) + } + return s.root +} + +// persistToBolt persists the given snapshot to bolt, +// sets the timestamp for the snapshot in the bolt metadata and +// persists any unpersisted segments to the directory. +func persistToBolt(t *testing.T, snapshot *IndexSnapshot, s *Scorch, timeStamp time.Time) error { + t.Helper() + tx, err := s.rootBolt.Begin(true) + if err != nil { + return err + } + _, _, err = prepareBoltSnapshot(snapshot, tx, s.path, s.segPlugin, nil) + if err != nil { + _ = tx.Rollback() + return err + } + snapshotsBucket, err := tx.CreateBucketIfNotExists(util.BoltSnapshotsBucket) + if err != nil { + _ = tx.Rollback() + return err + } + newSnapshotKey := encodeUvarintAscending(nil, snapshot.epoch) + snapshotBucket, err := snapshotsBucket.CreateBucketIfNotExists(newSnapshotKey) + if err != nil { + _ = tx.Rollback() + return err + } + metaBucket, err := snapshotBucket.CreateBucketIfNotExists(util.BoltMetaDataKey) + if err != nil { + _ = tx.Rollback() + return err + } + timeStampBinary, err := timeStamp.MarshalText() + if err != nil { + _ = tx.Rollback() + return err + } + err = metaBucket.Put(util.BoltMetaDataTimeStamp, timeStampBinary, nil) + if err != nil { + _ = tx.Rollback() + return err + } + for _, segmentSnapshot := range snapshot.segment { + if seg, ok := segmentSnapshot.segment.(segment.UnpersistedSegment); ok { + filename := zapFileName(segmentSnapshot.id) + path := filepath.Join(s.path, filename) + err := persistToDirectory(seg, nil, path) + if err != nil { + _ = tx.Rollback() + return err + } + persistedSeg, err := s.segPlugin.OpenUsing(path, s.segmentConfig) + if err != nil { + _ = tx.Rollback() + return err + } + segmentSnapshot.segment = persistedSeg + } + } + err = tx.Commit() + if err != nil { + _ = tx.Rollback() + return err + } + err = s.rootBolt.Sync() + if err != nil { + _ = tx.Rollback() + return err + } + return nil +} + +func TestLatestSnapshotProtected(t *testing.T) { + cfg := CreateConfig("TestLatestSnapshotProtected") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { _ = DestroyTest(cfg) }() + dirPath := cfg["path"].(string) + if err = os.Mkdir(dirPath, 0o755); err != nil { + t.Fatal(err) + } + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("failed to create Scorch: %v", err) + } + s, ok := idx.(*Scorch) + if !ok { + t.Fatalf("expected *Scorch, got %T", s) + } + s.numSnapshotsToKeep = 3 + s.rollbackSamplingInterval = 10 * time.Second + s.path = dirPath + err = s.openBolt() + if err != nil { + t.Fatalf("failed to open bolt: %v", err) + } + // replicate the following scenario of persistence of snapshots + // tc - 9d/5, tc - 6d/5, tc - 3d/4, tc - d/6, tc - d/12, tc + // approximate timestamps where there's a chance that the latest snapshot + // might not fit into the time-series and get purged along with its segment + // files. + const ( + numSnapshots = 6 + ) + curTime := time.Now() + d := s.rollbackSamplingInterval + // old --> new + timeStamps := []time.Time{ + curTime.Add(-(9 * d / 5)), + curTime.Add(-(6 * d / 5)), + curTime.Add(-(3 * d / 4)), + curTime.Add(-(d / 6)), + curTime.Add(-(d / 12)), + curTime, + } + snapshots := make([]*IndexSnapshot, numSnapshots) + for i := 0; i < numSnapshots; i++ { + snapshots[i] = updateRoot(t, s, false) + err = persistToBolt(t, snapshots[i], s, timeStamps[i]) + if err != nil { + t.Fatalf("failed to persist snapshot %d: %v", i, err) + } + } + meta, err := s.getLiveSnapshots() + if err != nil { + t.Fatalf("failed to get live snapshots: %v", err) + } + if len(meta) != numSnapshots { + t.Fatalf("expected %d live snapshots, got %d", numSnapshots, len(meta)) + } + for i, m := range meta { + expectIdx := numSnapshots - i - 1 + expectedEpoch := snapshots[expectIdx].epoch + if m.epoch != expectedEpoch { + t.Fatalf("expected epoch %d, got %d", expectedEpoch, m.epoch) + } + expectedTimeStamp := timeStamps[expectIdx] + if !m.timeStamp.Equal(expectedTimeStamp) { + t.Fatalf("expected timestamp %v, got %v", expectedTimeStamp, m.timeStamp) + } + } + protectedSnapshots := s.getProtectedSnapshots(meta) + if len(protectedSnapshots) != s.numSnapshotsToKeep { + t.Fatalf("expected %d protected snapshots, got %d", s.numSnapshotsToKeep, len(protectedSnapshots)) + } + expectedProtectedEpochs := []uint64{6, 2, 1} + for _, expectedEpoch := range expectedProtectedEpochs { + if _, found := protectedSnapshots[expectedEpoch]; !found { + t.Fatalf("expected epoch %d to be protected, but not found", expectedEpoch) + } + } + err = s.Close() + if err != nil { + t.Fatalf("failed to close Scorch: %v", err) + } +} + +// testFSDirectory implements index.Directory by writing files under a +// directory on the local filesystem, mirroring bleve.FileSystemDirectory. It +// is used as the backup target for CopyTo. +type testFSDirectory string + +func (d testFSDirectory) GetWriter(filePath string) (io.WriteCloser, error) { + dir, file := filepath.Split(filePath) + if dir != "" { + if err := os.MkdirAll(filepath.Join(string(d), dir), os.ModePerm); err != nil { + return nil, err + } + } + return os.OpenFile(filepath.Join(string(d), dir, file), + os.O_RDWR|os.O_CREATE, 0o600) +} + +func TestBackupRacingWithPurge(t *testing.T) { + cfg := CreateConfig("TestBackupRacingWithPurge") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { _ = DestroyTest(cfg) }() + dirPath := cfg["path"].(string) + if err = os.Mkdir(dirPath, 0o755); err != nil { + t.Fatal(err) + } + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("failed to create Scorch: %v", err) + } + s, ok := idx.(*Scorch) + if !ok { + t.Fatalf("expected *Scorch, got %T", s) + } + s.numSnapshotsToKeep = 3 + s.rollbackSamplingInterval = 10 * time.Second + s.path = dirPath + err = s.openBolt() + if err != nil { + t.Fatalf("failed to open bolt: %v", err) + } + // replicate the following scenario of persistence of snapshots + // tc - 9d/5, tc - 6d/5, tc - 3d/4, tc - d/6, tc - d/12, tc + // approximate timestamps where there's a chance that the latest snapshot + // might not fit into the time-series and get purged along with its segment + // files. + const ( + numSnapshots = 6 + ) + curTime := time.Now() + d := s.rollbackSamplingInterval + // old --> new + timeStamps := []time.Time{ + curTime.Add(-(9 * d / 5)), + curTime.Add(-(6 * d / 5)), + curTime.Add(-(3 * d / 4)), + curTime.Add(-(d / 6)), + curTime.Add(-(d / 12)), + curTime, + } + for i := 0; i < numSnapshots; i++ { + snapshot := updateRoot(t, s, false) + if err = persistToBolt(t, snapshot, s, timeStamps[i]); err != nil { + t.Fatalf("failed to persist snapshot %d: %v", i, err) + } + } + // now update the root again, but with an empty snapshot, + // so that the latest root now references a segment that + // belongs to the snapshot which was just persisted to bolt. + updateRoot(t, s, true) + // now if the purge code is invoked, there's a possibility of the latest snapshot + // being removed from bolt and the corresponding file segment getting cleaned up. + s.removeOldData() + // acquire the copy reader which will now refer to + // the latest in-memory snapshot which was the + // empty snapshot we just introduced. + copyReader := s.CopyReader() + defer func() { copyReader.CloseCopyReader() }() + + backupidxConfig := CreateConfig("backup-directory") + err = InitTest(backupidxConfig) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(backupidxConfig) + if err != nil { + t.Log(err) + } + }() + // if the latest snapshot was purged, the following will return error + err = copyReader.CopyTo(testFSDirectory(backupidxConfig["path"].(string))) + if err != nil { + t.Fatalf("error copying the index: %v", err) + } + err = s.Close() + if err != nil { + t.Fatalf("failed to close Scorch: %v", err) + } +} + +func TestSparseMutationCheckpointing(t *testing.T) { + cfg := CreateConfig("TestSparseMutationCheckpointing") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { _ = DestroyTest(cfg) }() + dirPath := cfg["path"].(string) + if err = os.Mkdir(dirPath, 0o755); err != nil { + t.Fatal(err) + } + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("failed to create Scorch: %v", err) + } + s, ok := idx.(*Scorch) + if !ok { + t.Fatalf("expected *Scorch, got %T", s) + } + s.numSnapshotsToKeep = 3 + s.rollbackSamplingInterval = 2 * time.Second + s.rollbackRetentionFactor = 0.5 + s.path = dirPath + err = s.openBolt() + if err != nil { + t.Fatalf("failed to open bolt: %v", err) + } + const ( + numSnapshots = 5 + ) + curTime := time.Now() + d := s.rollbackSamplingInterval + // old --> new + timeStamps := []time.Time{ + curTime.Add(-(6 * d)), + curTime.Add(-(5 * d)), + curTime.Add(-(4 * d)), + curTime.Add(-(3 * d)), + curTime, + } + for i := 0; i < numSnapshots; i++ { + snapshot := updateRoot(t, s, false) + if err = persistToBolt(t, snapshot, s, timeStamps[i]); err != nil { + t.Fatalf("failed to persist snapshot %d: %v", i, err) + } + } + s.checkPoints = newCheckPoints(map[uint64]time.Time{ + 2: timeStamps[1], + 3: timeStamps[2], + 4: timeStamps[3], + }) + meta, err := s.getLiveSnapshots() + if err != nil { + t.Fatal(err) + } + if len(meta) != s.numSnapshotsToKeep { + t.Fatalf("expected %d live snapshots, got %d", s.numSnapshotsToKeep, len(meta)) + } + for i, m := range meta { + expectIdx := numSnapshots - i - 1 + expectedEpoch := uint64(expectIdx + 1) + if m.epoch != expectedEpoch { + t.Fatalf("expected epoch %d, got %d", expectedEpoch, m.epoch) + } + expectedTimeStamp := timeStamps[expectIdx] + if !m.timeStamp.Equal(expectedTimeStamp) { + t.Fatalf("expected timestamp %v, got %v", expectedTimeStamp, m.timeStamp) + } + } + protectedSnapshots := s.getProtectedSnapshots(meta) + if len(protectedSnapshots) != s.numSnapshotsToKeep { + t.Fatalf("expected %d protected snapshots, got %d", s.numSnapshotsToKeep, len(protectedSnapshots)) + } + expectedProtectedEpochs := []uint64{5, 4, 3} + for _, expectedEpoch := range expectedProtectedEpochs { + if _, found := protectedSnapshots[expectedEpoch]; !found { + t.Fatalf("expected epoch %d to be protected, but not found", expectedEpoch) + } + } + err = s.Close() + if err != nil { + t.Fatalf("failed to close Scorch: %v", err) + } +} + +func TestRollbackCheckpointsOnRestart(t *testing.T) { + cfg := CreateConfig("TestRollbackCheckpointsOnRestart") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { _ = DestroyTest(cfg) }() + dirPath := cfg["path"].(string) + if err = os.Mkdir(dirPath, 0o755); err != nil { + t.Fatal(err) + } + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("failed to create Scorch: %v", err) + } + s, ok := idx.(*Scorch) + if !ok { + t.Fatalf("expected *Scorch, got %T", s) + } + s.numSnapshotsToKeep = 3 + s.rollbackSamplingInterval = 2 * time.Second + s.rollbackRetentionFactor = 0.5 + s.path = dirPath + err = s.openBolt() + if err != nil { + t.Fatalf("failed to open bolt: %v", err) + } + const ( + numSnapshots = 6 + ) + curTime := time.Now() + d := s.rollbackSamplingInterval + // old --> new + timeStamps := []time.Time{ + curTime.Add(-(10 * d)), + curTime.Add(-(9 * d)), + curTime.Add(-(8 * d)), + + curTime.Add(-(2 * d)), + curTime.Add(-(1 * d)), + curTime, + } + for i := 0; i < 3; i++ { + snapshot := updateRoot(t, s, false) + if err = persistToBolt(t, snapshot, s, timeStamps[i]); err != nil { + t.Fatalf("failed to persist snapshot %d: %v", i, err) + } + } + err = s.Close() + if err != nil { + t.Fatalf("failed to close Scorch: %v", err) + } + idx, err = NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("failed to create Scorch: %v", err) + } + s, ok = idx.(*Scorch) + if !ok { + t.Fatalf("expected *Scorch, got %T", s) + } + s.numSnapshotsToKeep = 3 + s.rollbackSamplingInterval = 2 * time.Second + s.rollbackRetentionFactor = 0.5 + s.path = dirPath + err = s.openBolt() + if err != nil { + t.Fatalf("failed to open bolt: %v", err) + } + for i := 3; i < numSnapshots; i++ { + snapshot := updateRoot(t, s, false) + if err = persistToBolt(t, snapshot, s, timeStamps[i]); err != nil { + t.Fatalf("failed to persist snapshot %d: %v", i, err) + } + } + meta, err := s.getLiveSnapshots() + if err != nil { + t.Fatal(err) + } + if len(meta) != 5 { + t.Fatalf("expected %d live snapshots, got %d", 5, len(meta)) + } + for i, m := range meta { + expectIdx := numSnapshots - i - 1 + expectedEpoch := uint64(expectIdx + 1) + if m.epoch != expectedEpoch { + t.Fatalf("expected epoch %d, got %d", expectedEpoch, m.epoch) + } + expectedTimeStamp := timeStamps[expectIdx] + if !m.timeStamp.Equal(expectedTimeStamp) { + t.Fatalf("expected timestamp %v, got %v", expectedTimeStamp, m.timeStamp) + } + } + protectedSnapshots := s.getProtectedSnapshots(meta) + if len(protectedSnapshots) != s.numSnapshotsToKeep { + t.Fatalf("expected %d protected snapshots, got %d", s.numSnapshotsToKeep, len(protectedSnapshots)) + } + expectedProtectedEpochs := []uint64{6, 3, 2} + for _, expectedEpoch := range expectedProtectedEpochs { + if _, found := protectedSnapshots[expectedEpoch]; !found { + t.Fatalf("expected epoch %d to be protected, but not found", expectedEpoch) + } + } + err = s.Close() + if err != nil { + t.Fatalf("failed to close Scorch: %v", err) + } +} + +func TestRollbackOptions(t *testing.T) { + cfg := CreateConfig("TestRollbackOptions") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { _ = DestroyTest(cfg) }() + dirPath := cfg["path"].(string) + if err = os.Mkdir(dirPath, 0o755); err != nil { + t.Fatal(err) + } + type testCase struct { + cfg map[string]interface{} + expectErr bool + } + tests := []testCase{ + { + cfg: map[string]interface{}{ + "numSnapshotsToKeep": 3, + "rollbackSamplingInterval": "10m", + "rollbackRetentionFactor": 0.5, + }, + expectErr: false, + }, + { + cfg: map[string]interface{}{ + "numSnapshotsToKeep": 0.5, + "rollbackSamplingInterval": "10a", + "rollbackRetentionFactor": 1, + }, + expectErr: true, + }, + { + cfg: map[string]interface{}{ + "numSnapshotsToKeep": 3.0, + "rollbackSamplingInterval": 10, + "rollbackRetentionFactor": 1.5, + }, + expectErr: true, + }, + { + cfg: map[string]interface{}{ + "numSnapshotsToKeep": 3.0, + "rollbackSamplingInterval": "10s", + "rollbackRetentionFactor": 0, + }, + expectErr: false, + }, + { + cfg: map[string]interface{}{ + "numSnapshotsToKeep": "3.0", + "rollbackSamplingInterval": "10s", + "rollbackRetentionFactor": 2, + }, + expectErr: true, + }, + { + cfg: map[string]interface{}{ + "numSnapshotsToKeep": 3.0, + "rollbackSamplingInterval": "10s", + "rollbackRetentionFactor": "2", + }, + expectErr: true, + }, + { + cfg: map[string]interface{}{ + "numSnapshotsToKeep": 3.0, + "rollbackSamplingInterval": "10s", + "rollbackRetentionFactor": 2, + }, + expectErr: true, + }, + } + for i, test := range tests { + test.cfg["path"] = dirPath + s, err := NewScorch(Name, test.cfg, index.NewAnalysisQueue(1)) + gotErr := err != nil + wantErr := test.expectErr + if gotErr != wantErr { + t.Fatalf("test %d: expected error: %v, got error: %v", i, wantErr, gotErr) + } + if err == nil { + err = s.Close() + if err != nil { + t.Fatalf("test %d: failed to close Scorch: %v", i, err) + } + } + } +} diff --git a/index/scorch/scorch.go b/index/scorch/scorch.go new file mode 100644 index 0000000..af65f30 --- /dev/null +++ b/index/scorch/scorch.go @@ -0,0 +1,1528 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/RoaringBitmap/roaring/v2" + "github.com/blevesearch/bleve/v2/index/scorch/mergeplan" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + bolt "go.etcd.io/bbolt" +) + +const Name = "scorch" + +const Version uint8 = 2 + +var ErrClosed = fmt.Errorf("scorch closed") + +type Scorch struct { + nextSegmentID uint64 + stats Stats + zapStats segment.Stats + iStats internalStats + + readOnly bool + version uint8 + config map[string]interface{} + segmentConfig map[string]interface{} + analysisQueue *index.AnalysisQueue + path string + + unsafeBatch bool + + rootLock sync.RWMutex + + root *IndexSnapshot // holds 1 ref-count on the root + rootPersisted []chan error // closed when root is persisted + persistedCallbacks []index.BatchCallback + nextSnapshotEpoch uint64 + eligibleForRemoval []uint64 // Index snapshot epochs that are safe to GC. + ineligibleForRemoval map[string]bool // Filenames that should not be GC'ed yet. + + // keeps track of segments scheduled for online copy/backup operation. Each segment's filename maps to + // the count of copy schedules. Segments with non-zero counts are protected from removal by the cleanup + // operation. Counts decrement upon successful copy, allowing removal of segments with zero or absent counts. + // must be accessed within the rootLock as it is accessed by the asynchronous cleanup routine. + copyScheduled map[string]int + + persisterOptions *persisterOptions + mergePlannerOptions *mergeplan.MergePlanOptions + + numSnapshotsToKeep int + rollbackRetentionFactor float64 + checkPoints []*snapshotMetaData + rollbackSamplingInterval time.Duration + closeCh chan struct{} + introductions chan *segmentIntroduction + persists chan *persistIntroduction + merges chan *segmentMerge + introducerNotifier chan *epochWatcher + persisterNotifier chan *epochWatcher + rootBolt *util.RootBoltImpl + asyncTasks sync.WaitGroup + + trainer trainer + + onEvent func(event Event) bool + onAsyncError func(err error, path string) + + forceMergeRequestCh chan *mergerCtrl + + segPlugin SegmentPlugin + + spatialPlugin index.SpatialAnalyzerPlugin +} + +// trainer interface is used for training an index that has the concept +// of "learning". Naturally, a vector index is one such thing that would +// implement this interface. There can be multiple implementations of the +// training itself even for the same index type. +// +// this component is not supposed to interact with the other master routines +// of scorch and will be used only for training the index before the actual data +// ingestion starts. The routine should also be released once the +// training is marked as complete - which can be done using the BoltTrainCompleteKey +// key and a bool value. However the struct is still maintained for the pointer to +// the instance so that we can use in the later stages of the index lifecycle. +type trainer interface { + // ephemeral + trainLoop() + // for the training state and the ingestion of the samples + train(batch *index.Batch) error + + // to load the metadata from the bolt under the BoltTrainerKey + loadTrainedData(*util.BoltBucketImpl) error + // to fetch the internal data from the component + getInternal(key []byte) ([]byte, error) + + // trainer specific file transfer operations + copyFileLOCKED(file string, d index.IndexDirectory) error + updateBolt(snapshotsBucket *util.BoltBucketImpl, key []byte, value []byte) error + + dropFileWriterIDs(ids map[string]struct{}) error + fileWriterIDsInUse() (map[string]struct{}, error) +} + +type ScorchErrorType string + +func (t ScorchErrorType) Error() string { + return string(t) +} + +// ErrType values for ScorchError +const ( + ErrAsyncPanic = ScorchErrorType("async panic error") + ErrPersist = ScorchErrorType("persist error") + ErrCleanup = ScorchErrorType("cleanup error") +) + +// ScorchError is passed to onAsyncError when errors are +// fired from scorch background processes +type ScorchError struct { + Source string + ErrMsg string + ErrType ScorchErrorType +} + +func (e *ScorchError) Error() string { + return fmt.Sprintf("source: %s, %v: %s", e.Source, e.ErrType, e.ErrMsg) +} + +// Lets the onAsyncError function verify what type of +// error is fired using errors.Is(...). This lets the function +// handle errors differently. +func (e *ScorchError) Unwrap() error { + return e.ErrType +} + +func NewScorchError(source, errMsg string, errType ScorchErrorType) error { + return &ScorchError{ + Source: source, + ErrMsg: errMsg, + ErrType: errType, + } +} + +type internalStats struct { + persistEpoch uint64 + persistSnapshotSize uint64 + mergeEpoch uint64 + mergeSnapshotSize uint64 + newSegBufBytesAdded uint64 + newSegBufBytesRemoved uint64 + analysisBytesAdded uint64 + analysisBytesRemoved uint64 +} + +func NewScorch(storeName string, + config map[string]interface{}, + analysisQueue *index.AnalysisQueue, +) (index.Index, error) { + rv := &Scorch{ + version: Version, + config: config, + analysisQueue: analysisQueue, + nextSnapshotEpoch: 1, + closeCh: make(chan struct{}), + ineligibleForRemoval: map[string]bool{}, + forceMergeRequestCh: make(chan *mergerCtrl, 1), + segPlugin: defaultSegmentPlugin, + copyScheduled: map[string]int{}, + segmentConfig: make(map[string]interface{}), + } + + forcedSegmentType, forcedSegmentVersion, err := configForceSegmentTypeVersion(config) + if err != nil { + return nil, err + } + if forcedSegmentType != "" && forcedSegmentVersion != 0 { + err := rv.loadSegmentPlugin(forcedSegmentType, + uint32(forcedSegmentVersion)) + if err != nil { + return nil, err + } + } + + segConfig, ok := config["segmentConfig"].(map[string]interface{}) + if ok { + rv.segmentConfig = segConfig + } + + // NOTE: always register the stats handler in the config map to track zap layer stats + rv.segmentConfig[segment.StatsKey] = &rv.zapStats + + typ, ok := config["spatialPlugin"].(string) + if ok { + if err := rv.loadSpatialAnalyzerPlugin(typ); err != nil { + return nil, err + } + } + + rv.root = &IndexSnapshot{parent: rv, refs: 1, creator: "NewScorch"} + ro, ok := config["read_only"].(bool) + if ok { + rv.readOnly = ro + } + ub, ok := config["unsafe_batch"].(bool) + if ok { + rv.unsafeBatch = ub + } + ecbName, ok := config["eventCallbackName"].(string) + if ok { + rv.onEvent = RegistryEventCallbacks[ecbName] + } + aecbName, ok := config["asyncErrorCallbackName"].(string) + if ok { + rv.onAsyncError = RegistryAsyncErrorCallbacks[aecbName] + } + + rv.numSnapshotsToKeep = NumSnapshotsToKeep + if v, ok := rv.config["numSnapshotsToKeep"]; ok { + var n int + var err error + if n, err = parseToInteger(v); err != nil { + return nil, fmt.Errorf("numSnapshotsToKeep parse err: %v", err) + } + if n > 0 { + rv.numSnapshotsToKeep = n + } + } + + rv.rollbackSamplingInterval = RollbackSamplingInterval + if v, ok := rv.config["rollbackSamplingInterval"]; ok { + var d time.Duration + var err error + if d, err = parseToTimeDuration(v); err != nil { + return nil, fmt.Errorf("rollbackSamplingInterval parse err: %v", err) + } + rv.rollbackSamplingInterval = d + } + + rv.rollbackRetentionFactor = RollbackRetentionFactor + if v, ok := rv.config["rollbackRetentionFactor"]; ok { + var r float64 + var err error + if r, err = parseToFloat(v); err != nil { + return nil, fmt.Errorf("rollbackRetentionFactor parse err: %v", err) + } + if r < 0 || r > 1 { + return nil, fmt.Errorf("rollbackRetentionFactor must be between 0 and 1") + } + rv.rollbackRetentionFactor = r + } + + po, err := rv.parsePersisterOptions() + if err != nil { + return nil, err + } + rv.persisterOptions = po + + mpo, err := rv.parseMergePlannerOptions(po) + if err != nil { + return nil, err + } + rv.mergePlannerOptions = mpo + + if trainer := initTrainer(rv, config); trainer != nil { + rv.trainer = trainer + } + + return rv, nil +} + +// configForceSegmentTypeVersion checks if the caller has requested a +// specific segment type/version +func configForceSegmentTypeVersion(config map[string]interface{}) (string, uint32, error) { + forcedSegmentVersion, err := parseToInteger(config["forceSegmentVersion"]) + if err != nil { + return "", 0, nil + } + + forcedSegmentType, ok := config["forceSegmentType"].(string) + if !ok { + return "", 0, fmt.Errorf( + "forceSegmentVersion set to %d, must also specify forceSegmentType", forcedSegmentVersion) + } + + return forcedSegmentType, uint32(forcedSegmentVersion), nil +} + +func (s *Scorch) NumEventsBlocking() uint64 { + eventsCompleted := atomic.LoadUint64(&s.stats.TotEventTriggerCompleted) + eventsStarted := atomic.LoadUint64(&s.stats.TotEventTriggerStarted) + return eventsStarted - eventsCompleted +} + +func (s *Scorch) fireEvent(kind EventKind, dur time.Duration) bool { + res := true + if s.onEvent != nil { + atomic.AddUint64(&s.stats.TotEventTriggerStarted, 1) + res = s.onEvent(Event{Kind: kind, Scorch: s, Duration: dur}) + atomic.AddUint64(&s.stats.TotEventTriggerCompleted, 1) + } + return res +} + +func (s *Scorch) fireAsyncError(err error) { + if s.onAsyncError != nil { + s.onAsyncError(err, s.path) + } + atomic.AddUint64(&s.stats.TotOnErrors, 1) +} + +func (s *Scorch) Open() error { + if s.rootBolt == nil { + err := s.openBolt() + if err != nil { + return err + } + } + + s.asyncTasks.Add(1) + go s.introducerLoop() + + if s.trainer != nil { + s.asyncTasks.Add(1) + go s.trainer.trainLoop() + } + + if !s.readOnly && s.path != "" { + s.asyncTasks.Add(1) + go s.persisterLoop() + s.asyncTasks.Add(1) + go s.mergerLoop() + } + + return nil +} + +func (s *Scorch) openBolt() error { + var ok bool + s.path, ok = s.config["path"].(string) + if !ok { + return fmt.Errorf("must specify path") + } + if s.path == "" { + s.unsafeBatch = true + } + + // bolt options + rootBoltOpt := *bolt.DefaultOptions + if s.readOnly { + rootBoltOpt.ReadOnly = true + rootBoltOpt.OpenFile = func(path string, flag int, mode os.FileMode) (*os.File, error) { + // Bolt appends an O_CREATE flag regardless. + // See - https://github.com/etcd-io/bbolt/blob/v1.3.5/db.go#L210 + // Use os.O_RDONLY only if path exists (#1623) + if _, err := os.Stat(path); os.IsNotExist(err) { + return os.OpenFile(path, flag, mode) + } + return os.OpenFile(path, os.O_RDONLY, mode) + } + } else { + if s.path != "" { + err := os.MkdirAll(s.path, 0o700) + if err != nil { + return err + } + } + } + + if boltTimeoutStr, ok := s.config["bolt_timeout"].(string); ok { + var err error + boltTimeout, err := time.ParseDuration(boltTimeoutStr) + if err != nil { + return fmt.Errorf("invalid duration specified for bolt_timeout: %v", err) + } + rootBoltOpt.Timeout = boltTimeout + } + + rootBoltPath := s.path + string(os.PathSeparator) + "root.bolt" + var err error + if s.path != "" { + s.rootBolt, err = util.OpenBolt(rootBoltPath, 0o600, &rootBoltOpt) + if err != nil { + return err + } + + // now see if there is any existing state to load + err = s.loadFromBolt() + if err != nil { + _ = s.Close() + return err + } + } + + atomic.StoreUint64(&s.stats.TotFileSegmentsAtRoot, uint64(len(s.root.segment))) + + s.introductions = make(chan *segmentIntroduction) + s.persists = make(chan *persistIntroduction) + s.merges = make(chan *segmentMerge) + s.introducerNotifier = make(chan *epochWatcher, 1) + s.persisterNotifier = make(chan *epochWatcher, 1) + s.closeCh = make(chan struct{}) + s.forceMergeRequestCh = make(chan *mergerCtrl, 1) + + if !s.readOnly && s.path != "" { + err := s.removeOldZapFiles() // Before persister or merger create any new files. + if err != nil { + _ = s.Close() + return err + } + } + + return nil +} + +func (s *Scorch) Close() (err error) { + startTime := time.Now() + defer func() { + s.fireEvent(EventKindClose, time.Since(startTime)) + }() + + s.fireEvent(EventKindCloseStart, 0) + + // signal to async tasks we want to close + close(s.closeCh) + // wait for them to close + s.asyncTasks.Wait() + // now close the root bolt + if s.rootBolt != nil { + err = s.rootBolt.Close() + s.rootLock.Lock() + if s.root != nil { + err2 := s.root.DecRef() + if err == nil { + err = err2 + } + } + s.root = nil + s.rootBolt = nil + s.rootLock.Unlock() + } + + return +} + +func (s *Scorch) Update(doc index.Document) error { + b := index.NewBatch() + b.Update(doc) + return s.Batch(b) +} + +func (s *Scorch) Delete(id string) error { + b := index.NewBatch() + b.Delete(id) + return s.Batch(b) +} + +func (s *Scorch) isTrained(batch *index.Batch) (bool, error) { + trained := true + if len(batch.IndexOps) > 0 && s.trainer != nil { + val, err := s.getInternal(util.BoltTrainCompleteKey) + if err != nil { + return false, err + } + + if val != nil { + trained, err = strconv.ParseBool(string(val)) + if err != nil { + return false, err + } + } + } + return trained, nil +} + +// Batch applices a batch of changes to the index atomically +func (s *Scorch) Batch(batch *index.Batch) (err error) { + start := time.Now() + + // notify handlers that we're about to index a batch of data + s.fireEvent(EventKindBatchIntroductionStart, 0) + defer func() { + s.fireEvent(EventKindBatchIntroduction, time.Since(start)) + }() + + trained, err := s.isTrained(batch) + if err != nil { + return err + } + + if !trained { + return fmt.Errorf("index is not trained yet") + } + + resultChan := make(chan index.Document, len(batch.IndexOps)) + + var numUpdates uint64 + var numDeletes uint64 + var numPlainTextBytes uint64 + var ids []string + for docID, doc := range batch.IndexOps { + if doc != nil { + // insert _id field + doc.AddIDField() + numUpdates++ + numPlainTextBytes += doc.NumPlainTextBytes() + } else { + numDeletes++ + } + ids = append(ids, docID) + } + + // FIXME could sort ids list concurrent with analysis? + + if numUpdates > 0 { + go func() { + for k := range batch.IndexOps { + doc := batch.IndexOps[k] + if doc != nil { + // put the work on the queue + s.analysisQueue.Queue(func() { + analyze(doc, s.setSpatialAnalyzerPlugin) + resultChan <- doc + }) + } + } + }() + } + + // wait for analysis result + analysisResults := make([]index.Document, int(numUpdates)) + var itemsDeQueued uint64 + var totalAnalysisSize int + for itemsDeQueued < numUpdates { + result := <-resultChan + resultSize := result.Size() + // check if the document is searchable by the index + if result.Indexed() { + atomic.AddUint64(&s.stats.TotMutationsFiltered, 1) + } + atomic.AddUint64(&s.iStats.analysisBytesAdded, uint64(resultSize)) + totalAnalysisSize += resultSize + analysisResults[itemsDeQueued] = result + itemsDeQueued++ + } + close(resultChan) + defer atomic.AddUint64(&s.iStats.analysisBytesRemoved, uint64(totalAnalysisSize)) + + atomic.AddUint64(&s.stats.TotAnalysisTime, uint64(time.Since(start))) + + indexStart := time.Now() + + var newSegment segment.Segment + var bufBytes uint64 + + if len(analysisResults) > 0 { + newSegment, bufBytes, err = s.segPlugin.NewUsing(analysisResults, s.segmentConfig) + if err != nil { + return err + } + if segB, ok := newSegment.(segment.DiskStatsReporter); ok { + atomic.AddUint64(&s.stats.TotBytesWrittenAtIndexTime, + segB.BytesWritten()) + } + atomic.AddUint64(&s.iStats.newSegBufBytesAdded, bufBytes) + } else { + atomic.AddUint64(&s.stats.TotBatchesEmpty, 1) + } + + err = s.prepareSegment(newSegment, ids, batch.InternalOps, batch.PersistedCallback()) + if err != nil { + if newSegment != nil { + _ = newSegment.Close() + } + atomic.AddUint64(&s.stats.TotOnErrors, 1) + } else { + atomic.AddUint64(&s.stats.TotUpdates, numUpdates) + atomic.AddUint64(&s.stats.TotDeletes, numDeletes) + atomic.AddUint64(&s.stats.TotBatches, 1) + atomic.AddUint64(&s.stats.TotIndexedPlainTextBytes, numPlainTextBytes) + } + + atomic.AddUint64(&s.iStats.newSegBufBytesRemoved, bufBytes) + atomic.AddUint64(&s.stats.TotIndexTime, uint64(time.Since(indexStart))) + + return err +} + +func (s *Scorch) getInternal(key []byte) ([]byte, error) { + s.rootLock.RLock() + defer s.rootLock.RUnlock() + + switch string(key) { + case string(util.BoltTrainCompleteKey): + if s.trainer != nil { + return s.trainer.getInternal(key) + } else { + return nil, fmt.Errorf("get on BoltTrainCompleteKey is not supported" + + " with this build") + } + } + return nil, nil +} + +func (s *Scorch) Train(batch *index.Batch) error { + if s.trainer != nil { + return s.trainer.train(batch) + } + return fmt.Errorf("training is not supported with this build") +} + +func (s *Scorch) prepareSegment(newSegment segment.Segment, ids []string, + internalOps map[string][]byte, persistedCallback index.BatchCallback) error { + // new introduction + introduction := &segmentIntroduction{ + id: atomic.AddUint64(&s.nextSegmentID, 1), + data: newSegment, + ids: ids, + internal: internalOps, + applied: make(chan error), + persistedCallback: persistedCallback, + } + + if !s.unsafeBatch { + introduction.persisted = make(chan error, 1) + } + + // optimistically prepare obsoletes outside of rootLock + s.rootLock.RLock() + root := s.root + root.AddRef() + s.rootLock.RUnlock() + + defer func() { _ = root.DecRef() }() + + introduction.obsoletes = make(map[uint64]*roaring.Bitmap, len(root.segment)) + + for _, seg := range root.segment { + delta, err := seg.segment.DocNumbers(ids) + if err != nil { + return err + } + introduction.obsoletes[seg.id] = delta + } + + introStartTime := time.Now() + + s.introductions <- introduction + + // block until this segment is applied + err := <-introduction.applied + if err != nil { + return err + } + + if introduction.persisted != nil { + err = <-introduction.persisted + } + + introTime := uint64(time.Since(introStartTime)) + atomic.AddUint64(&s.stats.TotBatchIntroTime, introTime) + if atomic.LoadUint64(&s.stats.MaxBatchIntroTime) < introTime { + atomic.StoreUint64(&s.stats.MaxBatchIntroTime, introTime) + } + + return err +} + +func (s *Scorch) SetInternal(key, val []byte) error { + b := index.NewBatch() + b.SetInternal(key, val) + return s.Batch(b) +} + +func (s *Scorch) DeleteInternal(key []byte) error { + b := index.NewBatch() + b.DeleteInternal(key) + return s.Batch(b) +} + +// Reader returns a low-level accessor on the index data. Close it to +// release associated resources. +func (s *Scorch) Reader() (index.IndexReader, error) { + return s.currentSnapshot(), nil +} + +func (s *Scorch) currentSnapshot() *IndexSnapshot { + s.rootLock.RLock() + rv := s.root + if rv != nil { + rv.AddRef() + } + s.rootLock.RUnlock() + return rv +} + +func (s *Scorch) Stats() json.Marshaler { + return &s.stats +} + +func (s *Scorch) BytesReadQueryTime() uint64 { + return s.stats.TotBytesReadAtQueryTime +} + +func (s *Scorch) diskFileStats(rootSegmentPaths map[string]struct{}) (uint64, + uint64, uint64, +) { + var numFilesOnDisk, numBytesUsedDisk, numBytesOnDiskByRoot uint64 + if s.path != "" { + files, err := os.ReadDir(s.path) + if err == nil { + for _, f := range files { + if !f.IsDir() { + if finfo, err := f.Info(); err == nil { + numBytesUsedDisk += uint64(finfo.Size()) + numFilesOnDisk++ + if rootSegmentPaths != nil { + fname := s.path + string(os.PathSeparator) + finfo.Name() + if _, fileAtRoot := rootSegmentPaths[fname]; fileAtRoot { + numBytesOnDiskByRoot += uint64(finfo.Size()) + } + } + } + } + } + } + } + // if no root files path given, then consider all disk files. + if rootSegmentPaths == nil { + return numFilesOnDisk, numBytesUsedDisk, numBytesUsedDisk + } + + return numFilesOnDisk, numBytesUsedDisk, numBytesOnDiskByRoot +} + +func (s *Scorch) StatsMap() map[string]interface{} { + m := s.stats.ToMap() + + indexSnapshot := s.currentSnapshot() + if indexSnapshot == nil { + return nil + } + + defer func() { + _ = indexSnapshot.Close() + }() + + rootSegPaths := indexSnapshot.diskSegmentsPaths() + + s.rootLock.RLock() + m["CurFilesIneligibleForRemoval"] = uint64(len(s.ineligibleForRemoval)) + s.rootLock.RUnlock() + + numFilesOnDisk, numBytesUsedDisk, numBytesOnDiskByRoot := s.diskFileStats(rootSegPaths) + + m["CurOnDiskBytes"] = numBytesUsedDisk + m["CurOnDiskFiles"] = numFilesOnDisk + + // TODO: consider one day removing these backwards compatible + // names for apps using the old names + m["updates"] = m["TotUpdates"] + m["deletes"] = m["TotDeletes"] + m["batches"] = m["TotBatches"] + m["errors"] = m["TotOnErrors"] + m["analysis_time"] = m["TotAnalysisTime"] + m["index_time"] = m["TotIndexTime"] + m["term_searchers_started"] = m["TotTermSearchersStarted"] + m["term_searchers_finished"] = m["TotTermSearchersFinished"] + m["knn_searches"] = m["TotKNNSearches"] + m["synonym_searches"] = m["TotSynonymSearches"] + m["total_mutations_filtered"] = m["TotMutationsFiltered"] + + m["num_bytes_read_at_query_time"] = m["TotBytesReadAtQueryTime"] + m["num_plain_text_bytes_indexed"] = m["TotIndexedPlainTextBytes"] + m["num_bytes_written_at_index_time"] = m["TotBytesWrittenAtIndexTime"] + m["num_items_introduced"] = m["TotIntroducedItems"] + m["num_items_persisted"] = m["TotPersistedItems"] + m["num_recs_to_persist"] = m["TotItemsToPersist"] + // total disk bytes found in index directory inclusive of older snapshots + m["num_bytes_used_disk"] = numBytesUsedDisk + // total disk bytes by the latest root index, exclusive of older snapshots + m["num_bytes_used_disk_by_root"] = numBytesOnDiskByRoot + // num_bytes_used_disk_by_root_reclaimable is an approximation about the + // reclaimable disk space in an index. (eg: from a full compaction) + m["num_bytes_used_disk_by_root_reclaimable"] = uint64(float64(numBytesOnDiskByRoot) * + indexSnapshot.reClaimableDocsRatio()) + m["num_files_on_disk"] = numFilesOnDisk + m["num_root_memorysegments"] = m["TotMemorySegmentsAtRoot"] + m["num_root_filesegments"] = m["TotFileSegmentsAtRoot"] + m["num_persister_nap_pause_completed"] = m["TotPersisterNapPauseCompleted"] + m["num_persister_nap_merger_break"] = m["TotPersisterMergerNapBreak"] + m["total_compaction_written_bytes"] = m["TotFileMergeWrittenBytes"] + + // the bool stat `index_bgthreads_active` indicates whether the background routines + // (which are responsible for the index to attain a steady state) are still + // doing some work. + if rootEpoch, ok := m["CurRootEpoch"].(uint64); ok { + if lastMergedEpoch, ok := m["LastMergedEpoch"].(uint64); ok { + if lastPersistedEpoch, ok := m["LastPersistedEpoch"].(uint64); ok { + m["index_bgthreads_active"] = !(lastMergedEpoch == rootEpoch && lastPersistedEpoch == rootEpoch) + } + } + } + + // calculate the aggregate of all the segment's field stats + aggFieldStats := newFieldStats() + for _, segmentSnapshot := range indexSnapshot.Segments() { + if segmentSnapshot.stats != nil { + aggFieldStats.Aggregate(segmentSnapshot.stats) + } + } + + aggFieldStatsMap := aggFieldStats.Fetch() + for statName, stats := range aggFieldStatsMap { + for fieldName, val := range stats { + m["field:"+fieldName+":"+statName] = val + } + } + + aggVectorStats := newFieldStats() + for _, segmentSnapshot := range indexSnapshot.Segments() { + if vsr, ok := segmentSnapshot.Segment().(segment.VectorFieldStatsReporter); ok { + segStats := newFieldStats() + vsr.UpdateVectorFieldStats(segStats) + aggVectorStats.Aggregate(segStats) + } + } + for statName, stats := range aggVectorStats.Fetch() { + for fieldName, val := range stats { + m["field:"+fieldName+":"+statName] = val + } + } + + // zap layer stats that are continously updated throughout the index lifecycle + m[segment.StatsKey] = s.zapStats.StatsMap() + return m +} + +func (s *Scorch) Analyze(d index.Document) { + analyze(d, s.setSpatialAnalyzerPlugin) +} + +type customAnalyzerPluginInitFunc func(field index.Field) + +func (s *Scorch) setSpatialAnalyzerPlugin(f index.Field) { + if s.segPlugin != nil { + // check whether the current field is a custom tokenizable + // spatial field then set the spatial analyser plugin for + // overriding the tokenisation during the analysis stage. + if sf, ok := f.(index.TokenizableSpatialField); ok { + sf.SetSpatialAnalyzerPlugin(s.spatialPlugin) + } + } +} + +func analyze(d index.Document, fn customAnalyzerPluginInitFunc) { + d.VisitFields(func(field index.Field) { + if field.Options().IsIndexed() { + if fn != nil { + fn(field) + } + + field.Analyze() + + if d.HasComposite() && field.Name() != "_id" { + // see if any of the composite fields need this + d.VisitComposite(func(cf index.CompositeField) { + cf.Compose(field.Name(), field.AnalyzedLength(), field.AnalyzedTokenFrequencies()) + }) + // Since the encoded geoShape is only necessary within the doc values + // of the geoShapeField, it has been removed from the field's term dictionary. + // However, '_all' field uses its term dictionary as its docValues, so it + // becomes necessary to add the geoShape into the '_all' field's term dictionary + if f, ok := field.(index.GeoShapeField); ok { + d.VisitComposite(func(cf index.CompositeField) { + geoshape := f.EncodedShape() + cf.Compose(field.Name(), 1, index.TokenFrequencies{ + string(geoshape): &index.TokenFreq{ + Term: geoshape, + Locations: []*index.TokenLocation{ + { + Start: 0, + End: len(geoshape), + Position: 1, + }, + }, + }, + }) + }) + } + } + } + }) + if nd, ok := d.(index.NestedDocument); ok { + nd.VisitNestedDocuments(func(doc index.Document) { + doc.AddIDField() + analyze(doc, fn) + }) + } +} + +func (s *Scorch) AddEligibleForRemoval(epoch uint64) { + s.rootLock.Lock() + if s.root == nil || s.root.epoch != epoch { + s.eligibleForRemoval = append(s.eligibleForRemoval, epoch) + } + s.rootLock.Unlock() +} + +func (s *Scorch) MemoryUsed() (memUsed uint64) { + indexSnapshot := s.currentSnapshot() + if indexSnapshot == nil { + return + } + + defer func() { + _ = indexSnapshot.Close() + }() + + // Account for current root snapshot overhead + memUsed += uint64(indexSnapshot.Size()) + + // Account for snapshot that the persister may be working on + persistEpoch := atomic.LoadUint64(&s.iStats.persistEpoch) + persistSnapshotSize := atomic.LoadUint64(&s.iStats.persistSnapshotSize) + if persistEpoch != 0 && indexSnapshot.epoch > persistEpoch { + // the snapshot that the persister is working on isn't the same as + // the current snapshot + memUsed += persistSnapshotSize + } + + // Account for snapshot that the merger may be working on + mergeEpoch := atomic.LoadUint64(&s.iStats.mergeEpoch) + mergeSnapshotSize := atomic.LoadUint64(&s.iStats.mergeSnapshotSize) + if mergeEpoch != 0 && indexSnapshot.epoch > mergeEpoch { + // the snapshot that the merger is working on isn't the same as + // the current snapshot + memUsed += mergeSnapshotSize + } + + memUsed += (atomic.LoadUint64(&s.iStats.newSegBufBytesAdded) - + atomic.LoadUint64(&s.iStats.newSegBufBytesRemoved)) + + memUsed += (atomic.LoadUint64(&s.iStats.analysisBytesAdded) - + atomic.LoadUint64(&s.iStats.analysisBytesRemoved)) + + return memUsed +} + +func (s *Scorch) markIneligibleForRemoval(filename string) { + s.rootLock.Lock() + s.ineligibleForRemoval[filename] = true + s.rootLock.Unlock() +} + +func (s *Scorch) unmarkIneligibleForRemoval(filename string) { + s.rootLock.Lock() + delete(s.ineligibleForRemoval, filename) + s.rootLock.Unlock() +} + +func init() { + err := registry.RegisterIndexType(Name, NewScorch) + if err != nil { + panic(err) + } +} + +func parseToTimeDuration(i interface{}) (time.Duration, error) { + switch v := i.(type) { + case string: + return time.ParseDuration(v) + + default: + return 0, fmt.Errorf("expects a duration string") + } +} + +func parseToInteger(i interface{}) (int, error) { + switch v := i.(type) { + case float64: + return int(v), nil + case int: + return v, nil + + default: + return 0, fmt.Errorf("expects int or float64 value") + } +} + +func parseToFloat(i interface{}) (float64, error) { + switch v := i.(type) { + case float64: + return v, nil + case float32: + return float64(v), nil + case int: + return float64(v), nil + + default: + return 0, fmt.Errorf("expects float64, float32 or int value") + } +} + +// Holds Zap's field level stats at a segment level +type fieldStats struct { + // StatName -> FieldName -> value + statMap map[string]map[string]uint64 +} + +// Add the data into the map after checking if the statname is valid +func (fs *fieldStats) Store(statName, fieldName string, value uint64) { + if _, exists := fs.statMap[statName]; !exists { + fs.statMap[statName] = make(map[string]uint64) + } + fs.statMap[statName][fieldName] = value +} + +// Combine the given stats map with the existing map +func (fs *fieldStats) Aggregate(stats segment.FieldStats) { + statMap := stats.Fetch() + if statMap == nil { + return + } + for statName, statMap := range statMap { + if _, exists := fs.statMap[statName]; !exists { + fs.statMap[statName] = make(map[string]uint64) + } + for fieldName, val := range statMap { + if _, exists := fs.statMap[statName][fieldName]; !exists { + fs.statMap[statName][fieldName] = 0 + } + fs.statMap[statName][fieldName] += val + } + } +} + +// Returns the stats map +func (fs *fieldStats) Fetch() map[string]map[string]uint64 { + if fs == nil { + return nil + } + + return fs.statMap +} + +// Initializes an empty stats map +func newFieldStats() *fieldStats { + rv := &fieldStats{ + statMap: map[string]map[string]uint64{}, + } + return rv +} + +// CopyReader returns a low-level accessor for index data, ensuring persisted segments +// remain on disk for backup, preventing race conditions with the persister/merger cleanup. +// Close the reader after backup to allow segment removal by the persister/merger. +func (s *Scorch) CopyReader() index.CopyReader { + s.rootLock.Lock() + rv := s.root + if rv != nil { + rv.AddRef() + var fileName string + // schedule a backup for all the segments from the root. Note that the + // both the unpersisted and persisted segments are scheduled for backup. + // because during the backup, the unpersisted segments may get persisted and + // hence we need to protect both the unpersisted and persisted segments from removal + // by the cleanup routine during the online backup + for _, seg := range rv.segment { + if perSeg, ok := seg.segment.(segment.PersistedSegment); ok { + // segment is persisted + fileName = filepath.Base(perSeg.Path()) + } else { + // segment is not persisted + // the name of the segment file that is generated if the + // the segment is persisted in the future. + fileName = zapFileName(seg.id) + } + rv.parent.copyScheduled[fileName]++ + } + } + s.rootLock.Unlock() + return rv +} + +func (s *Scorch) SetPathInBolt(key []byte, value []byte) error { + tx, err := s.rootBolt.Begin(true) + if err != nil { + return err + } + defer func() { + if err != nil { + _ = tx.Rollback() + } + }() + + snapshotsBucket, err := tx.CreateBucketIfNotExists(util.BoltSnapshotsBucket) + if err != nil { + return err + } + + if s.trainer != nil { + // currently this is specific to trained index file update + err = s.trainer.updateBolt(snapshotsBucket, key, value) + if err != nil { + return err + } + } + + err = tx.Commit() + if err != nil { + return err + } + + return s.rootBolt.Sync() +} + +// CopyFile copies a specific file to a destination directory which has an access to a bleve index +// doing a io.Copy() isn't enough because the file needs to be tracked in bolt file as well +func (s *Scorch) CopyFile(file string, d index.IndexDirectory) error { + s.rootLock.Lock() + defer s.rootLock.Unlock() + + dest, err := d.GetWriter(filepath.Join("store", file)) + if err != nil { + return err + } + + source, err := os.Open(filepath.Join(s.path, file)) + if err != nil { + return err + } + + defer source.Close() + defer dest.Close() + _, err = io.Copy(dest, source) + if err != nil { + return err + } + + // this code is currently specific to copying trained data but is future proofed for other files + // to be updated in the dest's bolt + err = s.trainer.copyFileLOCKED(file, d) + return err +} + +// external API to fire a scorch event (EventKindIndexStart) externally from bleve +func (s *Scorch) FireIndexEvent() { + s.fireEvent(EventKindIndexStart, 0) +} + +// Updates bolt db with the given field info. Existing field info already in bolt +// will be merged before persisting. The index mapping is also overwritted both +// in bolt as well as the index snapshot +func (s *Scorch) UpdateFields(fieldInfo map[string]*index.UpdateFieldInfo, mappingBytes []byte) error { + err := s.updateBolt(fieldInfo, mappingBytes) + if err != nil { + return err + } + // Pass the update field info to all snapshots and segment bases + s.root.UpdateFieldsInfo(fieldInfo) + return nil +} + +func (s *Scorch) OpenMeta() error { + if s.rootBolt == nil { + err := s.openBolt() + if err != nil { + return err + } + } + + return nil +} + +// Merge and update deleted field info and rewrite index mapping +func (s *Scorch) updateBolt(fieldInfo map[string]*index.UpdateFieldInfo, mappingBytes []byte) error { + filePath := s.path + string(os.PathSeparator) + "root.bolt" + return s.rootBolt.Update(func(tx *util.BoltTxImpl) error { + snapshots := tx.Bucket(util.BoltSnapshotsBucket) + if snapshots == nil { + return nil + } + + c := snapshots.Cursor() + for k, _ := c.Last(); k != nil; k, _ = c.Prev() { + _, _, err := decodeUvarintAscending(k) + if err != nil { + fmt.Printf("unable to parse segment epoch %x, continuing", k) + continue + } + snapshot := snapshots.GetBucket(k) + metaBucket := snapshot.GetBucket(util.BoltMetaDataKey) + if metaBucket == nil { + return fmt.Errorf("meta-data bucket missing") + } + + writer, err := util.NewFileWriter([]byte(filePath)) + if err != nil { + return fmt.Errorf("unable to load correct writer: %v", err) + } + + fileWriterID, err := metaBucket.Get(util.BoltMetaDataFileWriterIDKey, nil) + if err != nil { + return fmt.Errorf("unable to get file writer id: %v", err) + } + if fileWriterID == nil { + return fmt.Errorf("file writer id missing in meta data") + } + reader, err := util.NewFileReader(string(fileWriterID), []byte(filePath)) + if err != nil { + return fmt.Errorf("unable to load correct reader: %v", err) + } + + err = metaBucket.Put(util.BoltMetaDataFileWriterIDKey, []byte(writer.Id()), writer) + if err != nil { + return err + } + + cc := snapshot.Cursor() + for kk, _ := cc.First(); kk != nil; kk, _ = cc.Next() { + if kk[0] == util.BoltInternalKey[0] { + internalBucket := snapshot.GetBucket(kk) + if internalBucket == nil { + return fmt.Errorf("segment key, but bucket missing %x", kk) + } + + internalVals := make(map[string][]byte) + err := internalBucket.ForEach(func(key []byte, val []byte) error { + internalVals[string(key)] = val + return nil + }, reader) + if err != nil { + return err + } + + for key, val := range internalVals { + err = internalBucket.Put([]byte(key), val, writer) + if err != nil { + return err + } + } + } else if kk[0] != util.BoltMetaDataKey[0] { + segmentBucket := snapshot.GetBucket(kk) + if segmentBucket == nil { + return fmt.Errorf("segment key, but bucket missing %x", kk) + } + var updatedFields map[string]*index.UpdateFieldInfo + updatedFieldBytes, err := segmentBucket.Get(util.BoltUpdatedFieldsKey, reader) + if err != nil { + return fmt.Errorf("error getting updated field bytes: %v", err) + } + if updatedFieldBytes != nil { + err = json.Unmarshal(updatedFieldBytes, &updatedFields) + if err != nil { + return fmt.Errorf("error reading updated field bytes: %v", err) + } + for field, info := range fieldInfo { + if val, ok := updatedFields[field]; ok { + updatedFields[field] = &index.UpdateFieldInfo{ + Deleted: info.Deleted || val.Deleted, + Store: info.Store || val.Store, + DocValues: info.DocValues || val.DocValues, + Index: info.Index || val.Index, + } + } else { + updatedFields[field] = info + } + } + } else { + updatedFields = fieldInfo + } + buf, err := json.Marshal(updatedFields) + if err != nil { + return err + } + err = segmentBucket.Put(util.BoltUpdatedFieldsKey, buf, writer) + if err != nil { + return err + } + + deletedBytes, err := segmentBucket.Get(util.BoltDeletedKey, reader) + if err != nil { + return fmt.Errorf("error getting deleted bytes: %v", err) + } + if deletedBytes != nil { + err = segmentBucket.Put(util.BoltDeletedKey, deletedBytes, writer) + if err != nil { + return err + } + } + + statBytes, err := segmentBucket.Get(util.BoltStatsKey, reader) + if err != nil { + return fmt.Errorf("error getting stats bytes: %v", err) + } + if statBytes != nil { + err = segmentBucket.Put(util.BoltStatsKey, statBytes, writer) + if err != nil { + return err + } + } + } + } + } + return nil + }) +} + +// returns the set of file callback writer ids in use by all of the segments and boltdb +func (s *Scorch) FileWriterIDsInUse() (map[string]struct{}, error) { + s.rootLock.RLock() + keyMap := make(map[string]struct{}) + for _, segmentSnapShot := range s.root.segment { + if seg, ok := segmentSnapShot.segment.(segment.SegmentWithCallbacks); ok { + keyMap[seg.CallbackId()] = struct{}{} + } + } + + if s.trainer != nil { + trainerKeys, err := s.trainer.fileWriterIDsInUse() + if err != nil { + return nil, err + } + for k, _ := range trainerKeys { + keyMap[k] = struct{}{} + } + } + + boltKeys, err := s.boltFileWriterIDsInUse() + if err != nil { + return nil, err + } + + for k, _ := range boltKeys { + keyMap[k] = struct{}{} + } + s.rootLock.RUnlock() + + return keyMap, nil +} + +// removes all file callback writer ids in use from all of the segments and boltdb +// boltdb is updated with the latest callback writer while segments are force +// merged blockingly until snapshot is persisted with the latest callback writer +func (s *Scorch) DropFileWriterIDs(ids map[string]struct{}) error { + err := s.removeBoltFileWriterIDs(ids) + if err != nil { + return err + } + + if s.trainer != nil { + err := s.trainer.dropFileWriterIDs(ids) + if err != nil { + return err + } + } + + s.rootLock.Lock() + // create a done channel to ensure success of merge + ctx := context.Background() + doneCh := make(chan error) + ctx = context.WithValue(ctx, mergeDoneKey, doneCh) + + // PARTIAL ROLLBACK WILL NOT BE SUPPORTED DURING THIS OPERATION + // this is done because all of the rollback snapshots + // are likely to have the same sequence numbers and + // morever, it is not functionally correct to hold + // data with writer ids that have been removed + prevNumSnapshotsToKeep := s.numSnapshotsToKeep + s.numSnapshotsToKeep = 1 + defer func() { + // reset rollback snapshot retention + s.rootLock.Lock() + s.numSnapshotsToKeep = prevNumSnapshotsToKeep + s.rootLock.Unlock() + }() + + // track the zapx files that are expected to be removed after + // the merge so that we can block until they are removed by the persister + filePaths := make([]string, 0) + + var mergePlanner mergePlanFunc = func(ourSnapshot *IndexSnapshot) (*mergeplan.MergePlan, error) { + // Create a merge plan with the filtered segments and force a merge + // to remove the callback from the segments. + atomic.AddUint64(&s.stats.TotFileMergePlan, 1) + + // filter all segments that have callbacks that need to be removed + // and add them to the list of segments to compact + segsToCompact := make([]mergeplan.Segment, 0) + for _, ss := range ourSnapshot.segment { + // only persisted segments needs to be checked + if _, ok := ss.segment.(segment.PersistedSegment); ok { + if segWithCallbacks, ok := ss.segment.(segment.SegmentWithCallbacks); ok { + if _, ok := ids[segWithCallbacks.CallbackId()]; ok { + segsToCompact = append(segsToCompact, ss) + filePaths = append(filePaths, zapFileName(ss.id)) + } + } + } + } + + // attempt a merge plan with the default merge planner options + mergePlan, err := mergeplan.Plan(segsToCompact, s.mergePlannerOptions) + if err != nil { + atomic.AddUint64(&s.stats.TotFileMergePlanErr, 1) + return nil, fmt.Errorf("merge plan creation err: %v", err) + } + + // create a map to track segments included in the default merge plan + segDictionary := make(map[uint64]bool) + for _, seg := range segsToCompact { + segDictionary[seg.Id()] = true + } + + // create a merge plan if the default merge planner is unable + // to create one with the given segments + if mergePlan == nil { + mergePlan = &mergeplan.MergePlan{ + Tasks: make([]*mergeplan.MergeTask, 0), + } + } + + // mark all segments included in the default merge plan + for _, task := range mergePlan.Tasks { + for _, seg := range task.Segments { + segDictionary[seg.Id()] = false + } + } + + // Create additional merge tasks for segments that are unable to be merged + for _, seg := range segsToCompact { + if segDictionary[seg.Id()] { + mergePlan.Tasks = append(mergePlan.Tasks, &mergeplan.MergeTask{ + Segments: []mergeplan.Segment{seg}, + }) + } + } + + return mergePlan, nil + } + + // set the merge plan func in the context for the merger to use when it receives the merge request + // this is to ensure that the merge request is triggered with the latest snapshot, thus avoiding + // any races + ctx = context.WithValue(ctx, mergePlanFuncKey, mergePlanner) + + // trigger the merge with the force merge plan + s.forceMergeRequestCh <- &mergerCtrl{ + ctx: ctx, + } + s.rootLock.Unlock() + + // blockingly wait for merge to complete + err = <-doneCh + close(doneCh) + if err != nil { + return err + } + + // wait for files to be cleaned up by persister + err = s.waitTillFileCleanup(filePaths) + if err != nil { + return err + } + + return nil +} + +// waitTillFileCleanup blocks until the given files are cleaned up by the persister or merger or +// returns an error after a timeout. It does so by checking the index directory every 5 seconds +// for the presence of the given files, and returns once they are no longer present. +func (s *Scorch) waitTillFileCleanup(filePaths []string) error { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + timeout := time.After(5 * time.Minute) + + for { + select { + case <-ticker.C: + files, err := os.ReadDir(s.path) + if err != nil { + return err + } + for _, f := range files { + fname := f.Name() + if filepath.Ext(fname) == ".zap" { + for _, filePath := range filePaths { + if fname == filePath { + continue + } + } + } + } + return nil + case <-timeout: + return fmt.Errorf("timeout waiting for file cleanup for files: %v", filePaths) + } + } +} diff --git a/index/scorch/scorch_knn_test.go b/index/scorch/scorch_knn_test.go new file mode 100644 index 0000000..49527d5 --- /dev/null +++ b/index/scorch/scorch_knn_test.go @@ -0,0 +1,165 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package scorch + +import ( + "fmt" + "math/rand" + "os" + "sync/atomic" + "testing" + + "github.com/blevesearch/bleve/v2/document" + index "github.com/blevesearch/bleve_index_api" +) + +const ( + testVectorDims = 3 + testVectorSimilarity = index.CosineSimilarity + testVectorIndexOptimizedFor = index.IndexOptimizedForRecall +) + +func genVectorDoc(t *testing.T, fieldName string) index.Document { + t.Helper() + randNum := rand.Intn(1000000) + doc := document.NewDocument(fmt.Sprintf("vectest%d", randNum)) + vector := make([]float32, testVectorDims) + for i := range vector { + vector[i] = float32(i+1) * float32(randNum+1) + } + doc.AddField(document.NewVectorField(fieldName, nil, vector, testVectorDims, testVectorSimilarity, testVectorIndexOptimizedFor)) + doc.AddIDField() + doc.VisitFields(func(field index.Field) { + field.Analyze() + }) + return doc +} + +func TestFieldStatPersistence(t *testing.T) { + cfg := CreateConfig("TestFieldStatPersistence") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { _ = DestroyTest(cfg) }() + dirPath := cfg["path"].(string) + if err = os.Mkdir(dirPath, 0o755); err != nil { + t.Fatal(err) + } + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("failed to create Scorch: %v", err) + } + s, ok := idx.(*Scorch) + if !ok { + t.Fatalf("expected *Scorch, got %T", idx) + } + s.path = dirPath + if err = s.openBolt(); err != nil { + t.Fatalf("failed to open bolt: %v", err) + } + const ( + numSegments = 3 + fieldName = "vector" + statName = "field:" + fieldName + ":num_vectors" + ) + docs := make([]index.Document, numSegments) + ids := make([]string, numSegments) + for i := 0; i < numSegments; i++ { + docs[i] = genVectorDoc(t, fieldName) + ids[i] = docs[i].ID() + seg, _, err := s.segPlugin.New([]index.Document{docs[i]}) + if err != nil { + t.Fatalf("failed to create segment %d: %v", i, err) + } + intro := &segmentIntroduction{ + id: atomic.AddUint64(&s.nextSegmentID, 1), + data: seg, + ids: []string{ids[i]}, + applied: make(chan error), + } + if err = s.introduceSegment(intro); err != nil { + t.Fatalf("introduceSegment %d failed: %v", i, err) + } + } + + errCh := make(chan error, 1) + doneCh := make(chan struct{}) + go func() { + snapshot := s.root + snapshot.AddRef() + defer snapshot.DecRef() + err := s.persistSnapshot(snapshot, s.persisterOptions) + if err != nil { + errCh <- err + return + } + doneCh <- struct{}{} + }() + select { + case merge := <-s.merges: + s.introduceMerge(merge) + case err := <-errCh: + t.Fatalf("unexpected error during persist (merge phase): %v", err) + } + <-doneCh + snapshot := s.root + if len(snapshot.segment) != 1 { + t.Fatalf("expected 1 segment after introduce, got %d", len(snapshot.segment)) + } + if !snapshot.segment[0].HasVector() { + t.Fatalf("expected HasVector() == true after index persist") + } + scorchStats := s.StatsMap() + numVecs := scorchStats[statName].(uint64) + if numVecs != uint64(numSegments) { + t.Fatalf("expected %d vectors in stats, got %d", numSegments, numVecs) + } + if err = idx.Close(); err != nil { + t.Fatalf("failed to close index: %v", err) + } + idx, err = NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("failed to recreate Scorch: %v", err) + } + s, ok = idx.(*Scorch) + if !ok { + t.Fatalf("expected *Scorch, got %T", idx) + } + s.path = dirPath + if err = s.openBolt(); err != nil { + t.Fatalf("failed to open bolt on reopen: %v", err) + } + snapshot = s.root + if len(snapshot.segment) != 1 { + t.Fatalf("expected 1 segment after reopen, got %d", len(snapshot.segment)) + } + if !snapshot.segment[0].HasVector() { + t.Fatalf("reopened segment: expected HasVector() == true after index reopen") + } + scorchStats = s.StatsMap() + numVecs = scorchStats[statName].(uint64) + if numVecs != uint64(numSegments) { + t.Fatalf("expected %d vectors in stats, got %d", numSegments, numVecs) + } + err = idx.Close() + if err != nil { + t.Fatalf("failed to close index on reopen: %v", err) + } +} diff --git a/index/scorch/scorch_test.go b/index/scorch/scorch_test.go new file mode 100644 index 0000000..7a4d337 --- /dev/null +++ b/index/scorch/scorch_test.go @@ -0,0 +1,3963 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "context" + "encoding/binary" + "encoding/json" + "fmt" + "log" + "math/rand" + "os" + "path/filepath" + "reflect" + "regexp" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/RoaringBitmap/roaring/v2" + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" + regexpTokenizer "github.com/blevesearch/bleve/v2/analysis/tokenizer/regexp" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/index/scorch/mergeplan" + "github.com/blevesearch/bleve/v2/mapping" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +func init() { + // override for tests + DefaultPersisterNapTimeMSec = 1 +} + +func InitTest(cfg map[string]interface{}) error { + return os.RemoveAll(cfg["path"].(string)) +} + +func DestroyTest(cfg map[string]interface{}) error { + return os.RemoveAll(cfg["path"].(string)) +} + +func CreateConfig(name string) map[string]interface{} { + // TODO: Use t.Name() when Go 1.7 support terminates. + rv := make(map[string]interface{}) + rv["path"] = os.TempDir() + "/bleve-scorch-test-" + name + return rv +} + +var testAnalyzer = &analysis.DefaultAnalyzer{ + Tokenizer: regexpTokenizer.NewRegexpTokenizer(regexp.MustCompile(`\w+`)), +} + +func TestIndexOpenReopen(t *testing.T) { + cfg := CreateConfig("TestIndexOpenReopen") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + + var expectedCount uint64 + reader, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err := reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + // insert a doc + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err = reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + // now close it + err = idx.Close() + if err != nil { + t.Fatal(err) + } + + idx, err = NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + + // check the doc count again after reopening it + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err = reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + // now close it + err = idx.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestIndexOpenReopenWithInsert(t *testing.T) { + cfg := CreateConfig("TestIndexOpenReopen") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + + var expectedCount uint64 + reader, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err := reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + // insert a doc + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err = reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + // now close it + err = idx.Close() + if err != nil { + t.Fatal(err) + } + + // try to open the index and insert data + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + + // insert a doc + doc = document.NewDocument("2") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test2"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + // check the doc count again after reopening it + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err = reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + // now close it + err = idx.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestIndexInsert(t *testing.T) { + cfg := CreateConfig("TestIndexInsert") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + var expectedCount uint64 + reader, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err := reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err = reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestIndexInsertThenDelete(t *testing.T) { + cfg := CreateConfig("TestIndexInsertThenDelete") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + var expectedCount uint64 + reader, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err := reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + doc2 := document.NewDocument("2") + doc2.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc2) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err = reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + iid, err := reader.InternalID("1") + if err != nil || iid == nil { + t.Errorf("unexpected on doc id 1") + } + iid, err = reader.InternalID("2") + if err != nil || iid == nil { + t.Errorf("unexpected on doc id 2") + } + iid, err = reader.InternalID("3") + if err != nil || iid != nil { + t.Errorf("unexpected on doc id 3") + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + err = idx.Delete("1") + if err != nil { + t.Errorf("Error deleting entry from index: %v", err) + } + expectedCount-- + + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err = reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + storedDoc, err := reader.Document("1") + if err != nil { + t.Error(err) + } + if storedDoc != nil { + t.Errorf("expected nil for deleted stored doc #1, got %v", storedDoc) + } + storedDoc, err = reader.Document("2") + if err != nil { + t.Error(err) + } + if storedDoc == nil { + t.Errorf("expected stored doc for #2, got nil") + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + // now close it + err = idx.Close() + if err != nil { + t.Fatal(err) + } + + idx, err = NewScorch(Name, cfg, analysisQueue) // reopen + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error reopening index: %v", err) + } + + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err = reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + storedDoc, err = reader.Document("1") + if err != nil { + t.Error(err) + } + if storedDoc != nil { + t.Errorf("expected nil for deleted stored doc #1, got %v", storedDoc) + } + storedDoc, err = reader.Document("2") + if err != nil { + t.Error(err) + } + if storedDoc == nil { + t.Errorf("expected stored doc for #2, got nil") + } + iid, err = reader.InternalID("1") + if err != nil || iid != nil { + t.Errorf("unexpected on doc id 1") + } + iid, err = reader.InternalID("2") + if err != nil || iid == nil { + t.Errorf("unexpected on doc id 2, should exist") + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + err = idx.Delete("2") + if err != nil { + t.Errorf("Error deleting entry from index: %v", err) + } + expectedCount-- + + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err = reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + storedDoc, err = reader.Document("1") + if err != nil { + t.Error(err) + } + if storedDoc != nil { + t.Errorf("expected nil for deleted stored doc #1, got %v", storedDoc) + } + storedDoc, err = reader.Document("2") + if err != nil { + t.Error(err) + } + if storedDoc != nil { + t.Errorf("expected nil for deleted stored doc #2, got nil") + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestIndexInsertThenUpdate(t *testing.T) { + cfg := CreateConfig("TestIndexInsertThenUpdate") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + + var expectedCount uint64 + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + // this update should overwrite one term, and introduce one new one + doc = document.NewDocument("1") + doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test fail"), testAnalyzer)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error deleting entry from index: %v", err) + } + + // now do another update that should remove one of the terms + doc = document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("fail"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error deleting entry from index: %v", err) + } + + reader, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err := reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestIndexInsertMultiple(t *testing.T) { + cfg := CreateConfig("TestIndexInsertMultiple") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + var expectedCount uint64 + + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + doc = document.NewDocument("2") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + doc = document.NewDocument("3") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + reader, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err := reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestIndexInsertWithStore(t *testing.T) { + cfg := CreateConfig("TestIndexInsertWithStore") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + var expectedCount uint64 + reader, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err := reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + doc := document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), index.IndexField|index.StoreField)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err = reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + storedDocInt, err := indexReader.Document("1") + if err != nil { + t.Error(err) + } + + storedDoc := storedDocInt.(*document.Document) + + if len(storedDoc.Fields) != 1 { + t.Errorf("expected 1 stored field, got %d", len(storedDoc.Fields)) + } + for _, field := range storedDoc.Fields { + if field.Name() == "name" { + textField, ok := field.(*document.TextField) + if !ok { + t.Errorf("expected text field") + } + if string(textField.Value()) != "test" { + t.Errorf("expected field content 'test', got '%s'", string(textField.Value())) + } + } else if field.Name() == "_id" { + t.Errorf("not expecting _id field") + } + } +} + +func TestIndexInternalCRUD(t *testing.T) { + cfg := CreateConfig("TestIndexInternalCRUD") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + + if len(indexReader.(*IndexSnapshot).segment) != 0 { + t.Errorf("expected 0 segments") + } + + // get something that doesn't exist yet + val, err := indexReader.GetInternal([]byte("key")) + if err != nil { + t.Error(err) + } + if val != nil { + t.Errorf("expected nil, got %s", val) + } + + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + + // set + err = idx.SetInternal([]byte("key"), []byte("abc")) + if err != nil { + t.Error(err) + } + + indexReader2, err := idx.Reader() + if err != nil { + t.Error(err) + } + + if len(indexReader2.(*IndexSnapshot).segment) != 0 { + t.Errorf("expected 0 segments") + } + + // get + val, err = indexReader2.GetInternal([]byte("key")) + if err != nil { + t.Error(err) + } + if string(val) != "abc" { + t.Errorf("expected %s, got '%s'", "abc", val) + } + + err = indexReader2.Close() + if err != nil { + t.Fatal(err) + } + + // delete + err = idx.DeleteInternal([]byte("key")) + if err != nil { + t.Error(err) + } + + indexReader3, err := idx.Reader() + if err != nil { + t.Error(err) + } + + if len(indexReader3.(*IndexSnapshot).segment) != 0 { + t.Errorf("expected 0 segments") + } + + // get again + val, err = indexReader3.GetInternal([]byte("key")) + if err != nil { + t.Error(err) + } + if val != nil { + t.Errorf("expected nil, got %s", val) + } + + err = indexReader3.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestIndexBatch(t *testing.T) { + cfg := CreateConfig("TestIndexBatch") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + var expectedCount uint64 + + // first create 2 docs the old fashioned way + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + doc = document.NewDocument("2") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test2"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + // now create a batch which does 3 things + // insert new doc + // update existing doc + // delete existing doc + // net document count change 0 + + batch := index.NewBatch() + doc = document.NewDocument("3") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test3"))) + batch.Update(doc) + doc = document.NewDocument("2") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test2updated"))) + batch.Update(doc) + batch.Delete("1") + + err = idx.Batch(batch) + if err != nil { + t.Error(err) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + numSegments := len(indexReader.(*IndexSnapshot).segment) + if numSegments <= 0 { + t.Errorf("expected some segments, got: %d", numSegments) + } + + docCount, err := indexReader.DocCount() + if err != nil { + t.Fatal(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + + docIDReader, err := indexReader.DocIDReaderAll() + if err != nil { + t.Error(err) + } + var docIds []index.IndexInternalID + docID, err := docIDReader.Next() + for docID != nil && err == nil { + docIds = append(docIds, docID) + docID, err = docIDReader.Next() + } + if err != nil { + t.Error(err) + } + externalDocIds := map[string]struct{}{} + // convert back to external doc ids + for _, id := range docIds { + externalID, err := indexReader.ExternalID(id) + if err != nil { + t.Fatal(err) + } + externalDocIds[externalID] = struct{}{} + } + expectedDocIds := map[string]struct{}{ + "2": {}, + "3": {}, + } + if !reflect.DeepEqual(externalDocIds, expectedDocIds) { + t.Errorf("expected ids: %v, got ids: %v", expectedDocIds, externalDocIds) + } +} + +func TestIndexBatchWithCallbacks(t *testing.T) { + cfg := CreateConfig("TestIndexBatchWithCallbacks") + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + cerr := idx.Close() + if cerr != nil { + t.Fatal(cerr) + } + }() + + // Check that callback function works + var wg sync.WaitGroup + wg.Add(1) + + batch := index.NewBatch() + doc := document.NewDocument("3") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test3"))) + batch.Update(doc) + batch.SetPersistedCallback(func(e error) { + wg.Done() + }) + + err = idx.Batch(batch) + if err != nil { + t.Error(err) + } + + wg.Wait() + // test has no assertion but will timeout if callback doesn't fire +} + +func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) { + cfg := CreateConfig("TestIndexInsertUpdateDeleteWithMultipleTypesStored") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + var expectedCount uint64 + reader, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err := reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + doc := document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), index.IndexField|index.StoreField)) + doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, index.IndexField|index.StoreField)) + df, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), time.RFC3339, index.IndexField|index.StoreField) + if err != nil { + t.Error(err) + } + doc.AddField(df) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err = reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + + storedDocInt, err := indexReader.Document("1") + if err != nil { + t.Error(err) + } + + storedDoc := storedDocInt.(*document.Document) + + err = indexReader.Close() + if err != nil { + t.Error(err) + } + + if len(storedDoc.Fields) != 3 { + t.Errorf("expected 3 stored field, got %d", len(storedDoc.Fields)) + } + for _, field := range storedDoc.Fields { + if field.Name() == "name" { + textField, ok := field.(*document.TextField) + if !ok { + t.Errorf("expected text field") + } + if string(textField.Value()) != "test" { + t.Errorf("expected field content 'test', got '%s'", string(textField.Value())) + } + } else if field.Name() == "age" { + numField, ok := field.(*document.NumericField) + if !ok { + t.Errorf("expected numeric field") + } + numFieldNumer, err := numField.Number() + if err != nil { + t.Error(err) + } else { + if numFieldNumer != 35.99 { + t.Errorf("expected numeric value 35.99, got %f", numFieldNumer) + } + } + } else if field.Name() == "unixEpoch" { + dateField, ok := field.(*document.DateTimeField) + if !ok { + t.Errorf("expected date field") + } + dateFieldDate, _, err := dateField.DateTime() + if err != nil { + t.Error(err) + } else { + if dateFieldDate != time.Unix(0, 0).UTC() { + t.Errorf("expected date value unix epoch, got %v", dateFieldDate) + } + } + } else if field.Name() == "_id" { + t.Errorf("not expecting _id field") + } + } + + // now update the document, but omit one of the fields + doc = document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testup"), index.IndexField|index.StoreField)) + doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 36.99, index.IndexField|index.StoreField)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + indexReader2, err := idx.Reader() + if err != nil { + t.Error(err) + } + + // expected doc count shouldn't have changed + docCount, err = indexReader2.DocCount() + if err != nil { + t.Fatal(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + + // should only get 2 fields back now though + storedDocInt, err = indexReader2.Document("1") + if err != nil { + t.Error(err) + } + + storedDoc = storedDocInt.(*document.Document) + + err = indexReader2.Close() + if err != nil { + t.Error(err) + } + + if len(storedDoc.Fields) != 2 { + t.Errorf("expected 2 stored field, got %d", len(storedDoc.Fields)) + } + + for _, field := range storedDoc.Fields { + if field.Name() == "name" { + textField, ok := field.(*document.TextField) + if !ok { + t.Errorf("expected text field") + } + if string(textField.Value()) != "testup" { + t.Errorf("expected field content 'testup', got '%s'", string(textField.Value())) + } + } else if field.Name() == "age" { + numField, ok := field.(*document.NumericField) + if !ok { + t.Errorf("expected numeric field") + } + numFieldNumer, err := numField.Number() + if err != nil { + t.Error(err) + } else { + if numFieldNumer != 36.99 { + t.Errorf("expected numeric value 36.99, got %f", numFieldNumer) + } + } + } else if field.Name() == "_id" { + t.Errorf("not expecting _id field") + } + } + + // now delete the document + err = idx.Delete("1") + if err != nil { + t.Errorf("Error deleting entry from index: %v", err) + } + expectedCount-- + + // expected doc count shouldn't have changed + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err = reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestIndexInsertFields(t *testing.T) { + cfg := CreateConfig("TestIndexInsertFields") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + doc := document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), index.IndexField|index.StoreField)) + doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, index.IndexField|index.StoreField)) + dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), time.RFC3339, index.IndexField|index.StoreField) + if err != nil { + t.Error(err) + } + doc.AddField(dateField) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + fields, err := indexReader.Fields() + if err != nil { + t.Error(err) + } else { + fieldsMap := map[string]struct{}{} + for _, field := range fields { + fieldsMap[field] = struct{}{} + } + expectedFieldsMap := map[string]struct{}{ + "_id": {}, + "name": {}, + "age": {}, + "unixEpoch": {}, + } + if !reflect.DeepEqual(fieldsMap, expectedFieldsMap) { + t.Errorf("expected fields: %v, got %v", expectedFieldsMap, fieldsMap) + } + } +} + +func TestIndexUpdateComposites(t *testing.T) { + cfg := CreateConfig("TestIndexUpdateComposites") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + doc := document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), index.IndexField|index.StoreField)) + doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), index.IndexField|index.StoreField)) + doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, index.IndexField)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + // now lets update it + doc = document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testupdated"), index.IndexField|index.StoreField)) + doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("misterupdated"), index.IndexField|index.StoreField)) + doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, index.IndexField)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + // make sure new values are in index + storedDocInt, err := indexReader.Document("1") + if err != nil { + t.Error(err) + } + storedDoc := storedDocInt.(*document.Document) + if len(storedDoc.Fields) != 2 { + t.Errorf("expected 2 stored field, got %d", len(storedDoc.Fields)) + } + for _, field := range storedDoc.Fields { + if field.Name() == "name" { + textField, ok := field.(*document.TextField) + if !ok { + t.Errorf("expected text field") + } + if string(textField.Value()) != "testupdated" { + t.Errorf("expected field content 'test', got '%s'", string(textField.Value())) + } + } else if field.Name() == "_id" { + t.Errorf("not expecting _id field") + } + } +} + +func TestIndexTermReaderCompositeFields(t *testing.T) { + cfg := CreateConfig("TestIndexTermReaderCompositeFields") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + doc := document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), index.IndexField|index.StoreField|index.IncludeTermVectors)) + doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), index.IndexField|index.StoreField|index.IncludeTermVectors)) + doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, index.IndexField|index.IncludeTermVectors)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + termFieldReader, err := indexReader.TermFieldReader(context.TODO(), []byte("mister"), "_all", true, true, true) + if err != nil { + t.Error(err) + } + + tfd, err := termFieldReader.Next(nil) + for tfd != nil && err == nil { + externalID, err := indexReader.ExternalID(tfd.ID) + if err != nil { + t.Fatal(err) + } + + if externalID != "1" { + t.Errorf("expected to find document id 1") + } + + tfd, err = termFieldReader.Next(nil) + if err != nil { + t.Error(err) + } + } + if err != nil { + t.Error(err) + } +} + +func TestIndexDocValueReader(t *testing.T) { + cfg := CreateConfig("TestIndexDocumentVisitFieldTerms") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + doc := document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), index.IndexField|index.StoreField|index.IncludeTermVectors)) + doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), index.IndexField|index.StoreField|index.IncludeTermVectors)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + actualFieldTerms := make(fieldTerms) + + internalID, err := indexReader.InternalID("1") + if err != nil { + t.Fatal(err) + } + + dvr, err := indexReader.DocValueReader([]string{"name", "title"}) + if err != nil { + t.Error(err) + } + + err = dvr.VisitDocValues(internalID, func(field string, term []byte) { + actualFieldTerms[field] = append(actualFieldTerms[field], string(term)) + }) + if err != nil { + t.Error(err) + } + expectedFieldTerms := fieldTerms{ + "name": []string{"test"}, + "title": []string{"mister"}, + } + if !reflect.DeepEqual(actualFieldTerms, expectedFieldTerms) { + t.Errorf("expected field terms: %#v, got: %#v", expectedFieldTerms, actualFieldTerms) + } +} + +func TestDocValueReaderConcurrent(t *testing.T) { + cfg := CreateConfig("TestFieldTermsConcurrent") + + // setting path to empty string disables persistence/merging + // which ensures we have in-memory segments + // which is important for this test, to trigger the right code + // path, where fields exist, but have NOT been uninverted by + // the Segment impl (in memory segments are still SegmentBase) + cfg["path"] = "" + + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Fatal(err) + } + }() + + mp := mapping.NewIndexMapping() + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + cerr := idx.Close() + if cerr != nil { + t.Fatal(cerr) + } + }() + + // create a single bath (leading to 1 in-memory segment) + // have one field named "name" and 100 others named f0-f99 + batch := index.NewBatch() + for i := 0; i < 1000; i++ { + data := map[string]string{ + "name": fmt.Sprintf("doc-%d", i), + } + for j := 0; j < 100; j++ { + data[fmt.Sprintf("f%d", j)] = fmt.Sprintf("v%d", i) + } + doc := document.NewDocument(fmt.Sprintf("%d", i)) + err = mp.MapDocument(doc, data) + if err != nil { + t.Errorf("error mapping doc: %v", err) + } + batch.Update(doc) + } + + err = idx.Batch(batch) + if err != nil { + t.Fatal(err) + } + + // now have 10 goroutines try to visit field values for doc 1 + // in a random field + var wg sync.WaitGroup + for j := 0; j < 10; j++ { + wg.Add(1) + go func() { + r, err := idx.Reader() + if err != nil { + t.Errorf("error getting reader: %v", err) + wg.Done() + return + } + docNumber, err := r.InternalID("1") + if err != nil { + t.Errorf("error getting internal ID: %v", err) + wg.Done() + return + } + dvr, err := r.DocValueReader([]string{fmt.Sprintf("f%d", rand.Intn(100))}) + if err != nil { + t.Errorf("error getting doc value reader: %v", err) + wg.Done() + return + } + err = dvr.VisitDocValues(docNumber, func(field string, term []byte) {}) + if err != nil { + t.Errorf("error visiting doc values: %v", err) + wg.Done() + return + } + wg.Done() + }() + } + + wg.Wait() +} + +func TestConcurrentUpdate(t *testing.T) { + cfg := CreateConfig("TestConcurrentUpdate") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + // do some concurrent updates + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + go func(i int) { + doc := document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions(strconv.Itoa(i), []uint64{}, []byte(strconv.Itoa(i)), index.StoreField)) + err := idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + wg.Done() + }(i) + } + wg.Wait() + + // now load the name field and see what we get + r, err := idx.Reader() + if err != nil { + log.Fatal(err) + } + defer func() { + err := r.Close() + if err != nil { + t.Fatal(err) + } + }() + + docInt, err := r.Document("1") + if err != nil { + log.Fatal(err) + } + + doc := docInt.(*document.Document) + + if len(doc.Fields) > 2 { + t.Errorf("expected no more than 2 fields, found %d", len(doc.Fields)) + } +} + +func TestLargeField(t *testing.T) { + cfg := CreateConfig("TestLargeField") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + var largeFieldValue []byte + for len(largeFieldValue) < 4096 { + largeFieldValue = append(largeFieldValue, bleveWikiArticle1K...) + } + + d := document.NewDocument("large") + f := document.NewTextFieldWithIndexingOptions("desc", nil, largeFieldValue, index.IndexField|index.StoreField) + d.AddField(f) + + err = idx.Update(d) + if err != nil { + t.Fatal(err) + } +} + +var bleveWikiArticle1K = []byte(`Boiling liquid expanding vapor explosion +From Wikipedia, the free encyclopedia +See also: Boiler explosion and Steam explosion + +Flames subsequent to a flammable liquid BLEVE from a tanker. BLEVEs do not necessarily involve fire. + +This article's tone or style may not reflect the encyclopedic tone used on Wikipedia. See Wikipedia's guide to writing better articles for suggestions. (July 2013) +A boiling liquid expanding vapor explosion (BLEVE, /ˈblɛviː/ blev-ee) is an explosion caused by the rupture of a vessel containing a pressurized liquid above its boiling point.[1] +Contents [hide] +1 Mechanism +1.1 Water example +1.2 BLEVEs without chemical reactions +2 Fires +3 Incidents +4 Safety measures +5 See also +6 References +7 External links +Mechanism[edit] + +This section needs additional citations for verification. Please help improve this article by adding citations to reliable sources. Unsourced material may be challenged and removed. (July 2013) +There are three characteristics of liquids which are relevant to the discussion of a BLEVE:`) + +func TestIndexDocValueReaderWithMultipleDocs(t *testing.T) { + cfg := CreateConfig("TestIndexDocumentVisitFieldTermsWithMultipleDocs") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + doc := document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), index.IndexField|index.StoreField|index.IncludeTermVectors)) + doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), index.IndexField|index.StoreField|index.IncludeTermVectors)) + + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + + actualFieldTerms := make(fieldTerms) + docNumber, err := indexReader.InternalID("1") + if err != nil { + t.Fatal(err) + } + + dvr, err := indexReader.DocValueReader([]string{"name", "title"}) + if err != nil { + t.Fatal(err) + } + + err = dvr.VisitDocValues(docNumber, func(field string, term []byte) { + actualFieldTerms[field] = append(actualFieldTerms[field], string(term)) + }) + if err != nil { + t.Error(err) + } + expectedFieldTerms := fieldTerms{ + "name": []string{"test"}, + "title": []string{"mister"}, + } + if !reflect.DeepEqual(actualFieldTerms, expectedFieldTerms) { + t.Errorf("expected field terms: %#v, got: %#v", expectedFieldTerms, actualFieldTerms) + } + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + + doc2 := document.NewDocument("2") + doc2.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test2"), index.IndexField|index.StoreField|index.IncludeTermVectors)) + doc2.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister2"), index.IndexField|index.StoreField|index.IncludeTermVectors)) + err = idx.Update(doc2) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + indexReader, err = idx.Reader() + if err != nil { + t.Error(err) + } + + actualFieldTerms = make(fieldTerms) + docNumber, err = indexReader.InternalID("2") + if err != nil { + t.Fatal(err) + } + + dvr, err = indexReader.DocValueReader([]string{"name", "title"}) + if err != nil { + t.Fatal(err) + } + + err = dvr.VisitDocValues(docNumber, func(field string, term []byte) { + actualFieldTerms[field] = append(actualFieldTerms[field], string(term)) + }) + if err != nil { + t.Error(err) + } + expectedFieldTerms = fieldTerms{ + "name": []string{"test2"}, + "title": []string{"mister2"}, + } + if !reflect.DeepEqual(actualFieldTerms, expectedFieldTerms) { + t.Errorf("expected field terms: %#v, got: %#v", expectedFieldTerms, actualFieldTerms) + } + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + + doc3 := document.NewDocument("3") + doc3.AddField(document.NewTextFieldWithIndexingOptions("name3", []uint64{}, []byte("test3"), index.IndexField|index.StoreField|index.IncludeTermVectors)) + doc3.AddField(document.NewTextFieldWithIndexingOptions("title3", []uint64{}, []byte("mister3"), index.IndexField|index.StoreField|index.IncludeTermVectors)) + err = idx.Update(doc3) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + indexReader, err = idx.Reader() + if err != nil { + t.Error(err) + } + + actualFieldTerms = make(fieldTerms) + docNumber, err = indexReader.InternalID("3") + if err != nil { + t.Fatal(err) + } + + dvr, err = indexReader.DocValueReader([]string{"name3", "title3"}) + if err != nil { + t.Fatal(err) + } + + err = dvr.VisitDocValues(docNumber, func(field string, term []byte) { + actualFieldTerms[field] = append(actualFieldTerms[field], string(term)) + }) + if err != nil { + t.Error(err) + } + expectedFieldTerms = fieldTerms{ + "name3": []string{"test3"}, + "title3": []string{"mister3"}, + } + if !reflect.DeepEqual(actualFieldTerms, expectedFieldTerms) { + t.Errorf("expected field terms: %#v, got: %#v", expectedFieldTerms, actualFieldTerms) + } + + actualFieldTerms = make(fieldTerms) + docNumber, err = indexReader.InternalID("1") + if err != nil { + t.Fatal(err) + } + + dvr, err = indexReader.DocValueReader([]string{"name", "title"}) + if err != nil { + t.Fatal(err) + } + + err = dvr.VisitDocValues(docNumber, func(field string, term []byte) { + actualFieldTerms[field] = append(actualFieldTerms[field], string(term)) + }) + if err != nil { + t.Error(err) + } + expectedFieldTerms = fieldTerms{ + "name": []string{"test"}, + "title": []string{"mister"}, + } + if !reflect.DeepEqual(actualFieldTerms, expectedFieldTerms) { + t.Errorf("expected field terms: %#v, got: %#v", expectedFieldTerms, actualFieldTerms) + } + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestIndexDocValueReaderWithMultipleFieldOptions(t *testing.T) { + cfg := CreateConfig("TestIndexDocumentVisitFieldTermsWithMultipleFieldOptions") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + // mix of field options, this exercises the run time/ on the fly un inverting of + // doc values for custom options enabled field like designation, dept. + options := index.IndexField | index.StoreField | index.IncludeTermVectors + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) // default doc value persisted + doc.AddField(document.NewTextField("title", []uint64{}, []byte("mister"))) // default doc value persisted + doc.AddField(document.NewTextFieldWithIndexingOptions("designation", []uint64{}, []byte("engineer"), options)) + doc.AddField(document.NewTextFieldWithIndexingOptions("dept", []uint64{}, []byte("bleve"), options)) + + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + + actualFieldTerms := make(fieldTerms) + docNumber, err := indexReader.InternalID("1") + if err != nil { + t.Fatal(err) + } + + dvr, err := indexReader.DocValueReader([]string{"name", "designation", "dept"}) + if err != nil { + t.Fatal(err) + } + + err = dvr.VisitDocValues(docNumber, func(field string, term []byte) { + actualFieldTerms[field] = append(actualFieldTerms[field], string(term)) + }) + if err != nil { + t.Error(err) + } + expectedFieldTerms := fieldTerms{ + "name": []string{"test"}, + "designation": []string{"engineer"}, + "dept": []string{"bleve"}, + } + if !reflect.DeepEqual(actualFieldTerms, expectedFieldTerms) { + t.Errorf("expected field terms: %#v, got: %#v", expectedFieldTerms, actualFieldTerms) + } + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestAllFieldWithDifferentTermVectorsEnabled(t *testing.T) { + // Based on https://github.com/blevesearch/bleve/issues/895 from xeizmendi + cfg := CreateConfig("TestAllFieldWithDifferentTermVectorsEnabled") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + testConfig := cfg + mp := mapping.NewIndexMapping() + + keywordMapping := mapping.NewTextFieldMapping() + keywordMapping.Analyzer = keyword.Name + keywordMapping.IncludeTermVectors = false + keywordMapping.IncludeInAll = true + + textMapping := mapping.NewTextFieldMapping() + textMapping.Analyzer = standard.Name + textMapping.IncludeTermVectors = true + textMapping.IncludeInAll = true + + docMapping := mapping.NewDocumentStaticMapping() + docMapping.AddFieldMappingsAt("keyword", keywordMapping) + docMapping.AddFieldMappingsAt("text", textMapping) + + mp.DefaultMapping = docMapping + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch("storeName", testConfig, analysisQueue) + if err != nil { + log.Fatalln(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + data := map[string]string{ + "keyword": "something", + "text": "A sentence that includes something within.", + } + + doc := document.NewDocument("1") + err = mp.MapDocument(doc, data) + if err != nil { + t.Errorf("error mapping doc: %v", err) + } + + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } +} + +func TestForceVersion(t *testing.T) { + cfg := map[string]interface{}{} + cfg["forceSegmentType"] = "zap" + cfg["forceSegmentVersion"] = 11 + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("error opening a supported version: %v", err) + } + s := idx.(*Scorch) + if s.segPlugin.Version() != 11 { + t.Fatalf("wrong segment wrapper version loaded, expected %d got %d", 11, s.segPlugin.Version()) + } + cfg["forceSegmentVersion"] = 12 + idx, err = NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("error opening a supported version: %v", err) + } + s = idx.(*Scorch) + if s.segPlugin.Version() != 12 { + t.Fatalf("wrong segment wrapper version loaded, expected %d got %d", 12, s.segPlugin.Version()) + } + cfg["forceSegmentVersion"] = 10 + _, err = NewScorch(Name, cfg, analysisQueue) + if err == nil { + t.Fatalf("expected an error opening an unsupported version, got nil") + } +} + +func TestIndexForceMerge(t *testing.T) { + cfg := CreateConfig("TestIndexForceMerge") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + tmp := struct { + MaxSegmentsPerTier int `json:"maxSegmentsPerTier"` + SegmentsPerMergeTask int `json:"segmentsPerMergeTask"` + FloorSegmentSize int64 `json:"floorSegmentSize"` + }{ + int(1), + int(1), + int64(2), + } + cfg["scorchMergePlanOptions"] = &tmp + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + + var expectedCount uint64 + batch := index.NewBatch() + for i := 0; i < 10; i++ { + doc := document.NewDocument(fmt.Sprintf("doc1-%d", i)) + doc.AddField(document.NewTextField("name", []uint64{}, []byte(fmt.Sprintf("text1-%d", i)))) + batch.Update(doc) + doc = document.NewDocument(fmt.Sprintf("doc2-%d", i)) + doc.AddField(document.NewTextField("name", []uint64{}, []byte(fmt.Sprintf("text2-%d", i)))) + batch.Update(doc) + err = idx.Batch(batch) + if err != nil { + t.Error(err) + } + batch.Reset() + expectedCount += 2 + } + + // verify doc count + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + + docCount, err := indexReader.DocCount() + if err != nil { + t.Fatal(err) + } + + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + + var si *Scorch + var ok bool + if si, ok = idx.(*Scorch); !ok { + t.Errorf("expects a scorch index") + } + + nfs := atomic.LoadUint64(&si.stats.TotFileSegmentsAtRoot) + if nfs != 10 { + t.Errorf("expected 10 root file segments, got: %d", nfs) + } + + ctx := context.Background() + for atomic.LoadUint64(&si.stats.TotFileSegmentsAtRoot) != 1 { + err := si.ForceMerge(ctx, &mergeplan.MergePlanOptions{ + MaxSegmentsPerTier: 1, + MaxSegmentSize: 10000, + SegmentsPerMergeTask: 10, + FloorSegmentSize: 10000, + }) + if err != nil { + t.Errorf("ForceMerge failed, err: %v", err) + } + } + + // verify the final root segment count + if atomic.LoadUint64(&si.stats.TotFileSegmentsAtRoot) != 1 { + t.Errorf("expected a single root file segments, got: %d", + atomic.LoadUint64(&si.stats.TotFileSegmentsAtRoot)) + } + + // verify with an invalid merge plan + err = si.ForceMerge(ctx, &mergeplan.MergePlanOptions{ + MaxSegmentsPerTier: 1, + MaxSegmentSize: 1 << 33, + SegmentsPerMergeTask: 10, + FloorSegmentSize: 10000, + }) + if err != mergeplan.ErrMaxSegmentSizeTooLarge { + t.Errorf("ForceMerge expected to fail with ErrMaxSegmentSizeTooLarge") + } + + err = idx.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestCancelIndexForceMerge(t *testing.T) { + cfg := CreateConfig("TestCancelIndexForceMerge") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + tmp := struct { + MaxSegmentsPerTier int `json:"maxSegmentsPerTier"` + SegmentsPerMergeTask int `json:"segmentsPerMergeTask"` + FloorSegmentSize int64 `json:"floorSegmentSize"` + }{ + int(1), + int(1), + int64(2), + } + cfg["scorchMergePlanOptions"] = &tmp + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + + var expectedCount uint64 + batch := index.NewBatch() + for i := 0; i < 20; i++ { + doc := document.NewDocument(fmt.Sprintf("doc1-%d", i)) + doc.AddField(document.NewTextField("name", []uint64{}, []byte(fmt.Sprintf("text1-%d", i)))) + batch.Update(doc) + doc = document.NewDocument(fmt.Sprintf("doc2-%d", i)) + doc.AddField(document.NewTextField("name", []uint64{}, []byte(fmt.Sprintf("text2-%d", i)))) + batch.Update(doc) + err = idx.Batch(batch) + if err != nil { + t.Error(err) + } + batch.Reset() + expectedCount += 2 + } + + // verify doc count + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + docCount, err := indexReader.DocCount() + if err != nil { + t.Fatal(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + + var si *Scorch + var ok bool + if si, ok = idx.(*Scorch); !ok { + t.Fatal("expects a scorch index") + } + + // no merge operations are expected as per the original merge policy. + nfsr := atomic.LoadUint64(&si.stats.TotFileSegmentsAtRoot) + if nfsr != 20 { + t.Errorf("expected 20 root file segments, got: %d", nfsr) + } + + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + + // cancel the force merge operation once the root has some new merge + // introductions. ie if the root has lesser file segments than earlier. + go func() { + for { + nval := atomic.LoadUint64(&si.stats.TotFileSegmentsAtRoot) + if nval < nfsr { + cancel() + return + } + time.Sleep(time.Millisecond * 5) + } + }() + + err = si.ForceMerge(ctx, &mergeplan.MergePlanOptions{ + MaxSegmentsPerTier: 1, + MaxSegmentSize: 10000, + SegmentsPerMergeTask: 5, + FloorSegmentSize: 10000, + }) + if err != nil { + t.Errorf("ForceMerge failed, err: %v", err) + } + + // verify the final root file segment count or forceMerge completion + if atomic.LoadUint64(&si.stats.TotFileSegmentsAtRoot) == 1 { + t.Errorf("expected many files at root, but got: %d segments", + atomic.LoadUint64(&si.stats.TotFileSegmentsAtRoot)) + } + + err = idx.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestIndexSeekBackwardsStats(t *testing.T) { + cfg := CreateConfig("TestIndexOpenReopen") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + // insert a doc + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("cat"))) + err = idx.Update(doc) + if err != nil { + t.Fatalf("error updating index: %v", err) + } + + // insert another doc + doc = document.NewDocument("2") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("cat"))) + err = idx.Update(doc) + if err != nil { + t.Fatalf("error updating index: %v", err) + } + + reader, err := idx.Reader() + if err != nil { + t.Fatalf("error getting index reader: %v", err) + } + defer reader.Close() + + tfr, err := reader.TermFieldReader(context.TODO(), []byte("cat"), "name", false, false, false) + if err != nil { + t.Fatalf("error getting term field readyer for name/cat: %v", err) + } + + tfdFirst, err := tfr.Next(nil) + if err != nil { + t.Fatalf("error getting first tfd: %v", err) + } + + _, err = tfr.Next(nil) + if err != nil { + t.Fatalf("error getting second tfd: %v", err) + } + + // seek backwards to the first + _, err = tfr.Advance(tfdFirst.ID, nil) + if err != nil { + t.Fatalf("error adancing backwards: %v", err) + } + + err = tfr.Close() + if err != nil { + t.Fatalf("error closing term field reader: %v", err) + } + + if idx.(*Scorch).stats.TotTermSearchersStarted != idx.(*Scorch).stats.TotTermSearchersFinished { + t.Errorf("expected term searchers started %d to equal term searchers finished %d", + idx.(*Scorch).stats.TotTermSearchersStarted, + idx.(*Scorch).stats.TotTermSearchersFinished) + } +} + +// fieldTerms contains the terms used by a document, keyed by field +type fieldTerms map[string][]string + +// FieldsNotYetCached returns a list of fields not yet cached out of a larger list of fields +func (f fieldTerms) FieldsNotYetCached(fields []string) []string { + rv := make([]string, 0, len(fields)) + for _, field := range fields { + if _, ok := f[field]; !ok { + rv = append(rv, field) + } + } + return rv +} + +// Merge will combine two fieldTerms +// it assumes that the terms lists are complete (thus do not need to be merged) +// field terms from the other list always replace the ones in the receiver +func (f fieldTerms) Merge(other fieldTerms) { + for field, terms := range other { + f[field] = terms + } +} + +func TestOpenBoltTimeout(t *testing.T) { + cfg := CreateConfig("TestIndexOpenReopen") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch("storeName", cfg, analysisQueue) + if err != nil { + log.Fatalln(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + + // new config + cfg2 := CreateConfig("TestIndexOpenReopen") + // copy path from original config + cfg2["path"] = cfg["path"] + // set timeout in this cfg + cfg2["bolt_timeout"] = "100ms" + + idx2, err := NewScorch("storeName", cfg2, analysisQueue) + if err != nil { + log.Fatalln(err) + } + err = idx2.Open() + if err == nil { + t.Error("expected timeout error opening index again") + } +} + +func TestReadOnlyIndex(t *testing.T) { + // https://github.com/blevesearch/bleve/issues/1623 + cfg := CreateConfig("TestReadOnlyIndex") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + writeIdx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = writeIdx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + writeIdxClosed := false + defer func() { + if !writeIdxClosed { + err := writeIdx.Close() + if err != nil { + t.Fatal(err) + } + } + }() + + // Add a single document to the index. + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = writeIdx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + writeIdx.Close() + writeIdxClosed = true + + // After the index is written, change permissions on every file + // in the index to read-only. + var permissionsFunc func(folder string) + permissionsFunc = func(folder string) { + entries, _ := os.ReadDir(folder) + for _, entry := range entries { + fullName := filepath.Join(folder, entry.Name()) + if entry.IsDir() { + permissionsFunc(fullName) + } else { + if err := os.Chmod(fullName, 0o555); err != nil { + t.Fatal(err) + } + } + } + } + permissionsFunc(cfg["path"].(string)) + + // Now reopen the index in read-only mode and attempt to read from it. + cfg["read_only"] = true + readIdx, err := NewScorch(Name, cfg, analysisQueue) + defer func() { + err := readIdx.Close() + if err != nil { + t.Fatal(err) + } + }() + + if err != nil { + t.Fatal(err) + } + err = readIdx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + reader, err := readIdx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err := reader.DocCount() + if err != nil { + t.Fatal(err) + } + if docCount != 1 { + t.Errorf("Expected document count to be %d got %d", 1, docCount) + } +} + +func BenchmarkAggregateFieldStats(b *testing.B) { + fieldStatsArray := make([]*fieldStats, 1000) + + for i := range fieldStatsArray { + fieldStatsArray[i] = newFieldStats() + + fieldStatsArray[i].Store("num_vectors", "vector", uint64(rand.Intn(1000))) + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + aggFieldStats := newFieldStats() + + for _, fs := range fieldStatsArray { + aggFieldStats.Aggregate(fs) + } + } +} + +func TestPersistorMergerOptions(t *testing.T) { + type test struct { + config string + expectErr bool + } + tests := []test{ + { + // valid config and no error expected + config: `{ + "scorchPersisterOptions": { + "persisterNapTimeMSec": 1110, + "memoryPressurePauseThreshold" : 333 + } + }`, + expectErr: false, + }, + { + // valid json with invalid config values + // and error expected + config: `{ + "scorchPersisterOptions": { + "persisterNapTimeMSec": "1110", + "memoryPressurePauseThreshold" : [333] + } + }`, + expectErr: true, + }, + { + // valid json with invalid config values + // and error expected + config: `{ + "scorchPersisterOptions": { + "persisterNapTimeMSec": 1110.2, + "memoryPressurePauseThreshold" : 333 + } + }`, + expectErr: true, + }, + { + // invalid setting for scorchMergePlanOptions + config: `{ + "scorchPersisterOptions": { + "persisterNapTimeMSec": 1110, + "memoryPressurePauseThreshold" : 333 + }, + "scorchMergePlanOptions": [{ + "maxSegmentSize": 10000, + "maxSegmentsPerTier": 10, + "segmentsPerMergeTask": 10 + }] + }`, + expectErr: true, + }, + { + // valid setting + config: `{ + "scorchPersisterOptions": { + "persisterNapTimeMSec": 1110, + "memoryPressurePauseThreshold" : 333 + }, + "scorchMergePlanOptions": { + "maxSegmentSize": 10000, + "maxSegmentsPerTier": 10, + "segmentsPerMergeTask": 10 + } + }`, + expectErr: false, + }, + { + config: `{ + "scorchPersisterOptions": { + "persisterNapTimeMSec": 1110, + "memoryPressurePauseThreshold" : 333 + }, + "scorchMergePlanOptions": { + "maxSegmentSize": 5.6, + "maxSegmentsPerTier": 10, + "segmentsPerMergeTask": 10 + } + }`, + expectErr: true, + }, + { + config: `{ + "scorchPersisterOptions": { + "numPersisterWorkers": 4, + "maxSizeInMemoryMergePerWorker" : 20971520 + }, + "scorchMergePlanOptions": { + "floorSegmentFileSize": 20971520 + } + }`, + expectErr: false, + }, + { + config: `{ + "scorchPersisterOptions": { + "numPersisterWorkers": 1, + "maxSizeInMemoryMergePerWorker" : 0 + } + }`, + expectErr: false, + }, + { + config: `{ + "scorchPersisterOptions": { + "numPersisterWorkers": 1, + "maxSizeInMemoryMergePerWorker" : 20971520 + } + }`, + expectErr: false, + }, + { + config: `{ + "scorchPersisterOptions": { + "numPersisterWorkers": -1, + "maxSizeInMemoryMergePerWorker" : 20971520 + }, + "scorchMergePlanOptions": { + "floorSegmentFileSize": 20971520 + } + }`, + expectErr: true, + }, + { + config: `{ + "scorchPersisterOptions": { + "numPersisterWorkers": 4, + "maxSizeInMemoryMergePerWorker" : 0 + } + }`, + expectErr: true, + }, + } + for i, test := range tests { + cfg := map[string]interface{}{} + err := json.Unmarshal([]byte(test.config), &cfg) + if err != nil { + t.Fatalf("test %d: error unmarshalling config: %v", i, err) + } + analysisQueue := index.NewAnalysisQueue(1) + _, err = NewScorch(Name, cfg, analysisQueue) + if test.expectErr { + if err == nil { + t.Errorf("test %d: expected error, got nil", i) + } + } else { + if err != nil { + t.Errorf("test %d: unexpected error: %v", i, err) + } + } + } +} + +// genDoc generates a random document with a random ID and a single field "name" with value "test". +func genDoc(t *testing.T) index.Document { + t.Helper() + randNum := rand.Intn(1000000) + doc := document.NewDocument(fmt.Sprintf("test%d", randNum)) + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + doc.AddIDField() + doc.VisitFields(func(field index.Field) { + field.Analyze() + }) + return doc +} + +// getBatches generates a list of index.Batches. +// 1. for each VB, create a batch with a random doc +// 2. merge all the batches into a single batch +// 3. repeat until numBatches is reached +func getBatches(t *testing.T, numVB int, numBatches int) []*index.Batch { + t.Helper() + batches := make([]*index.Batch, 0, numBatches) + vbSeqNumbers := make([]uint64, numVB) + for i := 0; i < numVB; i++ { + vbSeqNumbers[i] = 0 + } + for i := 0; i < numBatches; i++ { + mergedBatch := index.NewBatch() + for j := 0; j < numVB; j++ { + vbBatch := index.NewBatch() + doc := genDoc(t) + vbBatch.Update(doc) + vbKey := fmt.Sprintf("%d", j) + vbValue := make([]byte, 8) + binary.BigEndian.PutUint64(vbValue, vbSeqNumbers[j]) + vbBatch.SetInternal([]byte(vbKey), vbValue) + vbSeqNumbers[j]++ + mergedBatch.Merge(vbBatch) + } + batches = append(batches, mergedBatch) + } + return batches +} + +func TestPersistenceMergedBatches(t *testing.T) { + cfg := CreateConfig("TestPersistenceMergedBatches") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { _ = DestroyTest(cfg) }() + idxPath := cfg["path"].(string) + os.Mkdir(idxPath, 0o755) + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("failed to create Scorch: %v", err) + } + s, ok := idx.(*Scorch) + if !ok { + t.Fatalf("expected *Scorch, got %T", s) + } + s.path = idxPath + err = s.openBolt() + if err != nil { + t.Fatalf("failed to open bolt: %v", err) + } + const ( + numSegments = 5 + numVBs = 5 + ) + batches := getBatches(t, numVBs, numSegments) + for i := 0; i < numSegments; i++ { + batch := batches[i] + var docs []index.Document + var ids []string + for _, doc := range batch.IndexOps { + docs = append(docs, doc) + ids = append(ids, doc.ID()) + } + seg, _, err := s.segPlugin.New(docs) + if err != nil { + t.Fatalf("failed to create segment %d: %v", i, err) + } + intro := &segmentIntroduction{ + id: atomic.AddUint64(&s.nextSegmentID, 1), + data: seg, + ids: ids, + internal: batch.InternalOps, + applied: make(chan error), + } + err = s.introduceSegment(intro) + if err != nil { + t.Fatalf("introduceSegment %d failed: %v", i, err) + } + } + snapshot := s.root + if len(snapshot.segment) != numSegments { + t.Fatalf("expected %d segments, got %d", numSegments, len(snapshot.segment)) + } + var errCh = make(chan error, 1) + var doneCh = make(chan struct{}) + go func() { + snapshot.AddRef() + defer snapshot.DecRef() + err := s.persistSnapshot(snapshot, s.persisterOptions) + if err != nil { + errCh <- err + return + } + doneCh <- struct{}{} + }() + select { + case merge := <-s.merges: + s.introduceMerge(merge) + case err := <-errCh: + t.Fatalf("unexpected error during persist: %v", err) + } + <-doneCh + // validate our snapshot's VB-seqMax mappings. + /* + Each Merged Batch has the following VB-seqMax mappings: + Each VB has its own batch, and when flushed to scorch, all 5 batches are merged into a single batch. + Batch 0: + 0 - 0 + 1 - 0 + 2 - 0 + 3 - 0 + 4 - 0 + Batch 1: + 0 - 1 + 1 - 1 + 2 - 1 + 3 - 1 + 4 - 1 + Batch 2: + 0 - 2 + 1 - 2 + 2 - 2 + 3 - 2 + 4 - 2 + Batch 3: + 0 - 3 + 1 - 3 + 2 - 3 + 3 - 3 + 4 - 3 + Batch 4: + 0 - 4 + 1 - 4 + 2 - 4 + 3 - 4 + 4 - 4 + We expect that after persistance, the VB-seqMax mappings will be as follows: + 1. In the Bolt Snapshot: + 0 - 4 + 1 - 4 + 2 - 4 + 3 - 4 + 4 - 4 + 2. In the in-memory snapshot: + 0 - 4 + 1 - 4 + 2 - 4 + 3 - 4 + 4 - 4 + This is because we MUST ensure that the + - In-memory snapshot and Persisted snapshot are in sync with each other. + - The VB-seqMax mappings always pick the highest sequence number for each VB. + */ + reader, err := idx.Reader() + if err != nil { + t.Fatalf("failed to get reader: %v", err) + } + expectedMappings := make([]uint64, numVBs) + for i := 0; i < numVBs; i++ { + expectedMappings[i] = 4 + } + actualMappings := make([]uint64, numVBs) + for i := 0; i < numVBs; i++ { + vbKey := []byte(strconv.Itoa(i)) + val, err := reader.GetInternal(vbKey) + if err != nil { + t.Fatalf("failed to get internal value for VB %d: %v", i, err) + } + if val == nil { + actualMappings[i] = 0 + } else { + actualMappings[i] = binary.BigEndian.Uint64(val) + } + } + if !slices.Equal(expectedMappings, actualMappings) { + t.Fatalf("expected in-memory snapshot VB-seqMax mappings %v, got %v", expectedMappings, actualMappings) + } + err = reader.Close() + if err != nil { + t.Fatalf("failed to close reader: %v", err) + } + err = idx.Close() + if err != nil { + t.Fatalf("failed to close index: %v", err) + } + idx, err = NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("failed to create Scorch: %v", err) + } + s, ok = idx.(*Scorch) + if !ok { + t.Fatalf("expected *Scorch, got %T", s) + } + s.path = idxPath + err = s.openBolt() + if err != nil { + t.Fatalf("failed to open bolt: %v", err) + } + if len(s.root.segment) != 1 { + t.Fatalf("expected 1 segment in root after reopen, got %d", len(s.root.segment)) + } + reader, err = idx.Reader() + if err != nil { + t.Fatalf("failed to get reader after reopen: %v", err) + } + expectedMappings = make([]uint64, numVBs) + for i := 0; i < numVBs; i++ { + expectedMappings[i] = 4 + } + actualMappings = make([]uint64, numVBs) + for i := 0; i < numVBs; i++ { + vbKey := []byte(strconv.Itoa(i)) + val, err := reader.GetInternal(vbKey) + if err != nil { + t.Fatalf("failed to get internal value for VB %d: %v", i, err) + } + if val == nil { + actualMappings[i] = 0 + } else { + actualMappings[i] = binary.BigEndian.Uint64(val) + } + } + if !slices.Equal(expectedMappings, actualMappings) { + t.Fatalf("expected Bolt snapshot VB-seqMax mappings %v, got %v", expectedMappings, actualMappings) + } + err = reader.Close() + if err != nil { + t.Fatalf("failed to close reader: %v", err) + } + err = idx.Close() + if err != nil { + t.Fatalf("failed to close index: %v", err) + } +} + +func TestIneligibleForRemovalOnSkippedMerge(t *testing.T) { + cfg := CreateConfig("TestIneligibleForRemovalOnSkippedMerge") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { _ = DestroyTest(cfg) }() + dirPath := cfg["path"].(string) + if err = os.Mkdir(dirPath, 0o755); err != nil { + t.Fatal(err) + } + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("failed to create Scorch: %v", err) + } + s, ok := idx.(*Scorch) + if !ok { + t.Fatalf("expected *Scorch, got %T", s) + } + s.path = dirPath + err = s.openBolt() + if err != nil { + t.Fatalf("failed to open bolt: %v", err) + } + const ( + numDocs = 3 + ) + docs := make([]index.Document, numDocs) + ids := make([]string, numDocs) + for i := 0; i < numDocs; i++ { + doc := genDoc(t) + docs[i] = doc + ids[i] = doc.ID() + } + for i := 0; i < numDocs; i++ { + doc := docs[i] + id := ids[i] + seg, _, err := s.segPlugin.New([]index.Document{doc}) + if err != nil { + t.Fatalf("failed to create segment: %v", err) + } + intro := &segmentIntroduction{ + id: atomic.AddUint64(&s.nextSegmentID, 1), + data: seg, + ids: []string{id}, + applied: make(chan error), + } + if err = s.introduceSegment(intro); err != nil { + t.Fatalf("introduceSegment: %v", err) + } + } + snapshot := s.root + if len(snapshot.segment) != numDocs { + t.Fatalf("expected %d segments, got %d", numDocs, len(snapshot.segment)) + } + for _, seg := range snapshot.segment { + if seg.Count() != 1 { + t.Fatalf("expected %d docs, got %d", 1, seg.Count()) + } + } + var errCh = make(chan error, 1) + var doneCh = make(chan struct{}) + go func() { + snapshot.AddRef() + defer snapshot.DecRef() + err := s.persistSnapshot(snapshot, s.persisterOptions) + if err != nil { + errCh <- err + return + } + doneCh <- struct{}{} + }() + select { + case merge := <-s.merges: + intro := &segmentIntroduction{ + id: atomic.AddUint64(&s.nextSegmentID, 1), + data: nil, + ids: ids, + applied: make(chan error), + } + if err = s.introduceSegment(intro); err != nil { + t.Fatalf("introduceSegment: %v", err) + } + snapshot = s.root + if len(snapshot.segment) != 0 { + t.Fatalf("expected %d segments, got %d", 0, len(snapshot.segment)) + } + docCount, err := snapshot.DocCount() + if err != nil { + t.Fatalf("failed to get doc count: %v", err) + } + if docCount != 0 { + t.Fatalf("expected %d docs, got %d", 0, docCount) + } + s.introduceMerge(merge) + case err := <-errCh: + t.Fatalf("unexpected error during persist: %v", err) + } + select { + case persist := <-s.persists: + s.introducePersist(persist) + case err := <-errCh: + t.Fatalf("unexpected error during persist: %v", err) + } + <-doneCh + snapshot = s.root + if len(snapshot.segment) != 0 { + t.Fatalf("expected %d segments, got %d", 0, len(snapshot.segment)) + } + docCount, err := snapshot.DocCount() + if err != nil { + t.Fatalf("failed to get doc count: %v", err) + } + if docCount != 0 { + t.Fatalf("expected %d docs, got %d", 0, docCount) + } + s.removeOldData() + files, err := os.ReadDir(dirPath) + if err != nil { + t.Fatalf("failed to read dir: %v", err) + } + var numZapFiles int + for _, file := range files { + if strings.HasSuffix(file.Name(), ".zap") { + numZapFiles++ + } + } + if numZapFiles != numDocs { + t.Fatalf("expected %d zap files, got %d", numDocs, numZapFiles) + } + err = idx.Close() + if err != nil { + t.Fatalf("failed to close index: %v", err) + } + idx, err = NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("failed to create Scorch: %v", err) + } + s, ok = idx.(*Scorch) + if !ok { + t.Fatalf("expected *Scorch, got %T", s) + } + s.path = dirPath + err = s.openBolt() + if err != nil { + t.Fatalf("failed to open bolt: %v", err) + } + snapshot = s.root + if len(snapshot.segment) != numDocs { + t.Fatalf("expected %d segments, got %d", numDocs, len(snapshot.segment)) + } + docCount, err = snapshot.DocCount() + if err != nil { + t.Fatalf("failed to get doc count: %v", err) + } + if docCount != numDocs { + t.Fatalf("expected %d docs, got %d", numDocs, docCount) + } + files, err = os.ReadDir(dirPath) + if err != nil { + t.Fatalf("failed to read dir: %v", err) + } + numZapFiles = 0 + for _, file := range files { + if strings.HasSuffix(file.Name(), ".zap") { + numZapFiles++ + } + } + if numZapFiles != numDocs { + t.Fatalf("expected %d zap files, got %d", numDocs, numZapFiles) + } + err = idx.Close() + if err != nil { + t.Fatalf("failed to close index: %v", err) + } +} + +func TestDeletionsBetweenMergeAndIntroduction(t *testing.T) { + cfg := CreateConfig("TestDeletionsBetweenMergeAndIntroduction") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { _ = DestroyTest(cfg) }() + dirPath := cfg["path"].(string) + if err = os.Mkdir(dirPath, 0o755); err != nil { + t.Fatal(err) + } + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("failed to create Scorch: %v", err) + } + s, ok := idx.(*Scorch) + if !ok { + t.Fatalf("expected *Scorch, got %T", s) + } + s.path = dirPath + err = s.openBolt() + if err != nil { + t.Fatalf("failed to open bolt: %v", err) + } + const ( + numDocs = 6 + numSegments = 2 + ) + docs := make([]index.Document, numDocs) + ids := make([]string, numDocs) + for i := 0; i < numDocs; i++ { + doc := genDoc(t) + docs[i] = doc + ids[i] = doc.ID() + } + for i := 0; i < numSegments; i++ { + doc1 := docs[i] + doc2 := docs[i+numSegments] + doc3 := docs[i+2*numSegments] + id1 := ids[i] + id2 := ids[i+numSegments] + id3 := ids[i+2*numSegments] + docs := []index.Document{doc1, doc2, doc3} + ids := []string{id1, id2, id3} + seg, _, err := s.segPlugin.New(docs) + if err != nil { + t.Fatalf("failed to create segment: %v", err) + } + intro := &segmentIntroduction{ + id: atomic.AddUint64(&s.nextSegmentID, 1), + data: seg, + ids: ids, + applied: make(chan error), + } + if err = s.introduceSegment(intro); err != nil { + t.Fatalf("introduceSegment: %v", err) + } + } + snapshot := s.root + if len(snapshot.segment) != numSegments { + t.Fatalf("expected %d segments, got %d", numSegments, len(snapshot.segment)) + } + docCount, err := snapshot.DocCount() + if err != nil { + t.Fatalf("failed to get doc count: %v", err) + } + if docCount != numDocs { + t.Fatalf("expected %d docs, got %d", numDocs, docCount) + } + var errCh = make(chan error, 1) + var doneCh = make(chan struct{}) + go func() { + snapshot.AddRef() + defer snapshot.DecRef() + err := s.persistSnapshotDirect(snapshot) + if err != nil { + errCh <- err + return + } + doneCh <- struct{}{} + }() + select { + case persist := <-s.persists: + s.introducePersist(persist) + case err := <-errCh: + t.Fatalf("unexpected error during persist: %v", err) + } + <-doneCh + snapshot = s.root + if len(snapshot.segment) != 2 { + t.Fatalf("expected %d segments, got %d", 2, len(snapshot.segment)) + } + docCount, err = snapshot.DocCount() + if err != nil { + t.Fatalf("failed to get doc count: %v", err) + } + if docCount != numDocs { + t.Fatalf("expected %d docs, got %d", numDocs, docCount) + } + intro := &segmentIntroduction{ + id: atomic.AddUint64(&s.nextSegmentID, 1), + data: nil, + ids: []string{ids[0], ids[1]}, + applied: make(chan error), + } + if err = s.introduceSegment(intro); err != nil { + t.Fatalf("introduceSegment: %v", err) + } + snapshot = s.root + if len(snapshot.segment) != 2 { + t.Fatalf("expected %d segments, got %d", 2, len(snapshot.segment)) + } + docCount, err = snapshot.DocCount() + if err != nil { + t.Fatalf("failed to get doc count: %v", err) + } + if docCount != 4 { + t.Fatalf("expected %d docs, got %d", 4, docCount) + } + go func() { + ctrlMsgDflt := &mergerCtrl{ctx: context.Background(), + options: s.mergePlannerOptions, + doneCh: nil, + } + snapshot.AddRef() + defer snapshot.DecRef() + err := s.planMergeAtSnapshot(ctrlMsgDflt, snapshot) + if err != nil { + errCh <- err + return + } + doneCh <- struct{}{} + }() + select { + case merge := <-s.merges: + intro := &segmentIntroduction{ + id: atomic.AddUint64(&s.nextSegmentID, 1), + data: nil, + ids: []string{ids[2], ids[3], ids[4]}, + applied: make(chan error), + } + if err = s.introduceSegment(intro); err != nil { + t.Fatalf("introduceSegment: %v", err) + } + snapshot = s.root + if len(snapshot.segment) != 1 { + t.Fatalf("expected %d segments, got %d", 1, len(snapshot.segment)) + } + docCount, err := snapshot.DocCount() + if err != nil { + t.Fatalf("failed to get doc count: %v", err) + } + if docCount != 1 { + t.Fatalf("expected %d docs, got %d", 1, docCount) + } + s.introduceMerge(merge) + case err := <-errCh: + t.Fatalf("unexpected error during persist: %v", err) + } + <-doneCh + snapshot = s.root + if len(snapshot.segment) != 1 { + t.Fatalf("expected %d segments, got %d", 1, len(snapshot.segment)) + } + docCount, err = snapshot.DocCount() + if err != nil { + t.Fatalf("failed to get doc count: %v", err) + } + if docCount != 1 { + t.Fatalf("expected %d docs, got %d", 1, docCount) + } + reader, err := snapshot.DocIDReaderAll() + if err != nil { + t.Fatalf("failed to get DocIDReaderAll: %v", err) + } + var liveIDs []string + for { + id, err := reader.Next() + if err != nil { + t.Fatalf("failed to read next doc ID: %v", err) + } + if id == nil { + break + } + eid, err := snapshot.ExternalID(id) + if err != nil { + t.Fatalf("failed to get external ID: %v", err) + } + liveIDs = append(liveIDs, eid) + } + if len(liveIDs) != 1 || liveIDs[0] != ids[5] { + t.Fatalf("expected live doc ID %s, got %v", ids[5], liveIDs) + } + err = idx.Close() + if err != nil { + t.Fatalf("failed to close index: %v", err) + } +} + +func TestPersistSnapshotMaybeMergeLegacy(t *testing.T) { + cfg := CreateConfig("TestPersistSnapshotMaybeMerge") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + _ = DestroyTest(cfg) + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("failed to create Scorch: %v", err) + } + defer idx.Close() + + s, ok := idx.(*Scorch) + if !ok { + t.Fatalf("expected *Scorch, got %T", idx) + } + + err = s.openBolt() + if err != nil { + t.Fatalf("failed to open bolt: %v", err) + } + + for i, docID := range []string{"doc1", "doc2"} { + doc := document.NewDocument(docID) + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + doc.AddIDField() + doc.VisitFields(func(field index.Field) { + field.Analyze() + }) + + seg, _, err := s.segPlugin.New([]index.Document{doc}) + if err != nil { + t.Fatalf("failed to create segment %d: %v", i, err) + } + + intro := &segmentIntroduction{ + id: atomic.AddUint64(&s.nextSegmentID, 1), + data: seg, + ids: []string{docID}, + applied: make(chan error), + } + + err = s.introduceSegment(intro) + if err != nil { + t.Fatalf("introduceSegment failed for %s: %v", docID, err) + } + } + + snapshot := s.root + if len(snapshot.segment) != 2 { + t.Fatalf("expected 2 segments at root, got %d", len(snapshot.segment)) + } + + mergeDone := make(chan struct{}) + go func() { + defer close(mergeDone) + select { + case nextMerge := <-s.merges: + s.introduceMerge(nextMerge) + case <-s.closeCh: + } + }() + + po := &persisterOptions{ + NumPersisterWorkers: 1, + MaxSizeInMemoryMergePerWorker: 0, + } + + persisted, err := s.persistSnapshotMaybeMerge(snapshot, po) + if err != nil { + t.Fatalf("persistSnapshotMaybeMerge failed: %v", err) + } + if !persisted { + t.Fatalf("expected persistSnapshotMaybeMerge to persist merged segment") + } + + select { + case <-mergeDone: + case <-time.After(2 * time.Second): + t.Fatalf("timed out waiting for merge introduction") + } + + if len(s.root.segment) != 1 { + t.Fatalf("expected merged root to have 1 segment, got %d", len(s.root.segment)) + } + + if _, ok := s.root.segment[0].segment.(segment.PersistedSegment); !ok { + t.Fatalf("expected merged segment to be persisted") + } + + epochs, err := s.RootBoltSnapshotEpochs() + if err != nil { + t.Fatalf("failed to read bolt snapshot epochs: %v", err) + } + if len(epochs) == 0 { + t.Fatalf("expected at least one persisted bolt snapshot") + } +} + +func TestPersistSnapshotMaybeMergeMultipleWorkers(t *testing.T) { + cfg := CreateConfig("TestPersistSnapshotMaybeMergeMultipleWorkers") + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + defer func() { + _ = DestroyTest(cfg) + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("failed to create Scorch: %v", err) + } + defer idx.Close() + + s, ok := idx.(*Scorch) + if !ok { + t.Fatalf("expected *Scorch, got %T", idx) + } + + err = s.openBolt() + if err != nil { + t.Fatalf("failed to open bolt: %v", err) + } + + totalMemoryUsed := 0 + for i := 0; i < 100; i++ { + docID := fmt.Sprintf("doc%d", i) + doc := document.NewDocument(docID) + testCorpus := []byte(fmt.Sprintf("test %v", docID)) + doc.AddField(document.NewTextField("name", []uint64{}, testCorpus)) + doc.AddIDField() + doc.VisitFields(func(field index.Field) { + field.Analyze() + }) + + seg, _, err := s.segPlugin.New([]index.Document{doc}) + if err != nil { + t.Fatalf("failed to create segment %d: %v", i, err) + } + + totalMemoryUsed += seg.Size() + intro := &segmentIntroduction{ + id: atomic.AddUint64(&s.nextSegmentID, 1), + data: seg, + ids: []string{docID}, + applied: make(chan error), + } + + err = s.introduceSegment(intro) + if err != nil { + t.Fatalf("introduceSegment failed for %s: %v", docID, err) + } + } + + snapshot := s.root + if len(snapshot.segment) != 100 { + t.Fatalf("expected 100 segments at root, got %d", len(snapshot.segment)) + } + + mergeDone := make(chan struct{}) + go func() { + defer close(mergeDone) + select { + case nextMerge := <-s.merges: + s.introduceMerge(nextMerge) + case <-s.closeCh: + } + }() + + po := &persisterOptions{ + NumPersisterWorkers: 2, + MaxSizeInMemoryMergePerWorker: totalMemoryUsed / 2, + } + + persisted, err := s.persistSnapshotMaybeMerge(snapshot, po) + if err != nil { + t.Fatalf("persistSnapshotMaybeMerge failed: %v", err) + } + if !persisted { + t.Fatalf("expected persistSnapshotMaybeMerge to persist merged segment") + } + + select { + case <-mergeDone: + case <-time.After(2 * time.Second): + t.Fatalf("timed out waiting for merge introduction") + } + + if len(s.root.segment) != 2 { + t.Fatalf("expected merged root to have 2 segments, got %d", len(s.root.segment)) + } + + for _, seg := range s.root.segment { + if _, ok := seg.segment.(segment.PersistedSegment); !ok { + t.Fatalf("expected merged segment to be persisted") + } + } + + epochs, err := s.RootBoltSnapshotEpochs() + if err != nil { + t.Fatalf("failed to read bolt snapshot epochs: %v", err) + } + + if len(epochs) == 0 { + t.Fatalf("expected at least one persisted bolt snapshot") + } +} + +// ------------------------------------------------------------------------ + +// mockSegmentBase satisfies segment.Segment but does NOT implement +// VectorFieldStatsReporter. Both mock types embed this so the stubs are +// not duplicated, while keeping the interface sets distinct. +type mockSegmentBase struct { + fields []string +} + +func (m *mockSegmentBase) Dictionary(_ string) (segment.TermDictionary, error) { return nil, nil } +func (m *mockSegmentBase) VisitStoredFields(_ uint64, _ segment.StoredFieldValueVisitor) error { + return nil +} +func (m *mockSegmentBase) DocID(_ uint64) ([]byte, error) { return nil, nil } +func (m *mockSegmentBase) Count() uint64 { return 0 } +func (m *mockSegmentBase) DocNumbers(_ []string) (*roaring.Bitmap, error) { + return roaring.New(), nil +} +func (m *mockSegmentBase) Fields() []string { return m.fields } +func (m *mockSegmentBase) Close() error { return nil } +func (m *mockSegmentBase) Size() int { return 0 } +func (m *mockSegmentBase) AddRef() {} +func (m *mockSegmentBase) DecRef() error { return nil } +func (m *mockSegmentBase) BytesRead() uint64 { return 0 } +func (m *mockSegmentBase) BytesWritten() uint64 { return 0 } +func (m *mockSegmentBase) ResetBytesRead(_ uint64) {} +func (m *mockSegmentBase) Ancestors(_ uint64, prealloc []index.AncestorID) []index.AncestorID { + return prealloc +} + +// mockVectorSegment adds VectorFieldStatsReporter on top of the base. +// inGPU controls whether the index is reported as residing in GPU or CPU memory. +type mockVectorSegment struct { + mockSegmentBase + inGPU bool +} + +func (m *mockVectorSegment) UpdateVectorFieldStats(stats segment.FieldStats) { + for _, f := range m.fields { + if m.inGPU { + stats.Store("num_vector_indexes_in_gpu", f, 1) + } else { + stats.Store("num_vector_indexes_in_cpu", f, 1) + } + } +} + +// mockPlainSegment is a segment that does NOT implement VectorFieldStatsReporter. +// It is used to verify that non-vector segments are silently skipped. +type mockPlainSegment struct { + mockSegmentBase +} + +// makeSegmentSnapshot wraps a segment in a SegmentSnapshot without any static +// field stats (stats == nil), matching the state of a live in-memory segment +// before it has been persisted. +func makeSegmentSnapshot(id uint64, seg segment.Segment) *SegmentSnapshot { + return &SegmentSnapshot{ + id: id, + segment: seg, + cachedDocs: &cachedDocs{cache: nil}, + cachedMeta: newCachedMeta(), + } +} + +// TestVectorFieldStatsAggregation verifies that StatsMap correctly aggregates +// num_vector_indexes_in_gpu and num_vector_indexes_in_cpu across multiple segments. +// +// Setup: +// - seg1: field "vec" -> index in GPU memory +// - seg2: field "vec" -> index in GPU memory +// - seg3: field "vec" -> index in CPU memory +// - seg4: plain segment (no VectorFieldStatsReporter) -> must be ignored +// +// Expected: +// +// field:vec:num_vector_indexes_in_gpu = 2 +// field:vec:num_vector_indexes_in_cpu = 1 +func TestVectorFieldStatsAggregation(t *testing.T) { + cfg := CreateConfig("TestVectorFieldStatsAggregation") + if err := InitTest(cfg); err != nil { + t.Fatal(err) + } + defer func() { + if err := DestroyTest(cfg); err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + s := idx.(*Scorch) + if err = s.Open(); err != nil { + t.Fatal(err) + } + defer func() { + if err := s.Close(); err != nil { + t.Log(err) + } + }() + + seg1 := &mockVectorSegment{mockSegmentBase: mockSegmentBase{fields: []string{"vec"}}, inGPU: true} + seg2 := &mockVectorSegment{mockSegmentBase: mockSegmentBase{fields: []string{"vec"}}, inGPU: true} + seg3 := &mockVectorSegment{mockSegmentBase: mockSegmentBase{fields: []string{"vec"}}, inGPU: false} + seg4 := &mockPlainSegment{mockSegmentBase: mockSegmentBase{fields: []string{"vec"}}} + + s.rootLock.Lock() + s.root.segment = append(s.root.segment, + makeSegmentSnapshot(100, seg1), + makeSegmentSnapshot(101, seg2), + makeSegmentSnapshot(102, seg3), + makeSegmentSnapshot(103, seg4), + ) + s.rootLock.Unlock() + + m := s.StatsMap() + if m == nil { + t.Fatal("StatsMap returned nil") + } + + checkUint64Stat(t, m, "field:vec:num_vector_indexes_in_gpu", 2) + checkUint64Stat(t, m, "field:vec:num_vector_indexes_in_cpu", 1) +} + +// TestVectorFieldStatsMultipleFields verifies that stats are tracked independently +// per field when a segment exposes more than one vector field. +func TestVectorFieldStatsMultipleFields(t *testing.T) { + cfg := CreateConfig("TestVectorFieldStatsMultipleFields") + if err := InitTest(cfg); err != nil { + t.Fatal(err) + } + defer func() { + if err := DestroyTest(cfg); err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + s := idx.(*Scorch) + if err = s.Open(); err != nil { + t.Fatal(err) + } + defer func() { + if err := s.Close(); err != nil { + t.Log(err) + } + }() + + // seg1: fieldA in GPU, fieldB in GPU + // seg2: fieldA in CPU + // seg3: fieldB in GPU + seg1 := &mockVectorSegment{mockSegmentBase: mockSegmentBase{fields: []string{"fieldA", "fieldB"}}, inGPU: true} + seg2 := &mockVectorSegment{mockSegmentBase: mockSegmentBase{fields: []string{"fieldA"}}, inGPU: false} + seg3 := &mockVectorSegment{mockSegmentBase: mockSegmentBase{fields: []string{"fieldB"}}, inGPU: true} + + s.rootLock.Lock() + s.root.segment = append(s.root.segment, + makeSegmentSnapshot(200, seg1), + makeSegmentSnapshot(201, seg2), + makeSegmentSnapshot(202, seg3), + ) + s.rootLock.Unlock() + + m := s.StatsMap() + if m == nil { + t.Fatal("StatsMap returned nil") + } + + // fieldA: 1 in GPU (seg1), 1 in CPU (seg2) + checkUint64Stat(t, m, "field:fieldA:num_vector_indexes_in_gpu", 1) + checkUint64Stat(t, m, "field:fieldA:num_vector_indexes_in_cpu", 1) + + // fieldB: 2 in GPU (seg1 + seg3), 0 in CPU + checkUint64Stat(t, m, "field:fieldB:num_vector_indexes_in_gpu", 2) + if _, ok := m["field:fieldB:num_vector_indexes_in_cpu"]; ok { + t.Errorf("expected no num_vector_indexes_in_cpu stat for fieldB, but got one") + } +} + +// TestVectorFieldStatsNoVectorSegments verifies that when no segment implements +// VectorFieldStatsReporter, the vector stat keys are absent from StatsMap. +func TestVectorFieldStatsNoVectorSegments(t *testing.T) { + cfg := CreateConfig("TestVectorFieldStatsNoVectorSegments") + if err := InitTest(cfg); err != nil { + t.Fatal(err) + } + defer func() { + if err := DestroyTest(cfg); err != nil { + t.Log(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + s := idx.(*Scorch) + if err = s.Open(); err != nil { + t.Fatal(err) + } + defer func() { + if err := s.Close(); err != nil { + t.Log(err) + } + }() + + s.rootLock.Lock() + s.root.segment = append(s.root.segment, + makeSegmentSnapshot(300, &mockPlainSegment{mockSegmentBase: mockSegmentBase{fields: []string{"vec"}}}), + ) + s.rootLock.Unlock() + + m := s.StatsMap() + if m == nil { + t.Fatal("StatsMap returned nil") + } + + for _, key := range []string{ + "field:vec:num_vector_indexes_in_gpu", + "field:vec:num_vector_indexes_in_cpu", + } { + if _, ok := m[key]; ok { + t.Errorf("expected key %q to be absent for non-vector segments, but it was present", key) + } + } +} + +func checkUint64Stat(t *testing.T, m map[string]interface{}, key string, want uint64) { + t.Helper() + v, ok := m[key] + if !ok { + t.Errorf("expected stat %q to be present in StatsMap, but it was missing", key) + return + } + got, ok := v.(uint64) + if !ok { + t.Errorf("stat %q: expected uint64, got %T (%v)", key, v, v) + return + } + if got != want { + t.Errorf("stat %q: got %d, want %d", key, got, want) + } +} diff --git a/index/scorch/segment_plugin.go b/index/scorch/segment_plugin.go new file mode 100644 index 0000000..16be8e4 --- /dev/null +++ b/index/scorch/segment_plugin.go @@ -0,0 +1,154 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + + "github.com/RoaringBitmap/roaring/v2" + "github.com/blevesearch/bleve/v2/geo" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + + zapv11 "github.com/blevesearch/zapx/v11" + zapv12 "github.com/blevesearch/zapx/v12" + zapv13 "github.com/blevesearch/zapx/v13" + zapv14 "github.com/blevesearch/zapx/v14" + zapv15 "github.com/blevesearch/zapx/v15" + zapv16 "github.com/blevesearch/zapx/v16" + zapv17 "github.com/blevesearch/zapx/v17" +) + +// SegmentPlugin represents the essential functions required by a package to plug in +// it's segment implementation +type SegmentPlugin interface { + + // Type is the name for this segment plugin + Type() string + + // Version is a numeric value identifying a specific version of this type. + // When incompatible changes are made to a particular type of plugin, the + // version must be incremented. + Version() uint32 + + // New takes a set of Documents and turns them into a new Segment + New(results []index.Document) (segment.Segment, uint64, error) + + NewUsing(results []index.Document, config map[string]interface{}) (segment.Segment, uint64, error) + + // Open attempts to open the file at the specified path and + // return the corresponding Segment + Open(path string) (segment.Segment, error) + + OpenUsing(path string, config map[string]interface{}) (segment.Segment, error) + + // Merge takes a set of Segments, and creates a new segment on disk at + // the specified path. + // Drops is a set of bitmaps (one for each segment) indicating which + // documents can be dropped from the segments during the merge. + // If the closeCh channel is closed, Merge will cease doing work at + // the next opportunity, and return an error (closed). + // StatsReporter can optionally be provided, in which case progress + // made during the merge is reported while operation continues. + // Returns: + // A slice of new document numbers (one for each input segment), + // this allows the caller to know a particular document's new + // document number in the newly merged segment. + // The number of bytes written to the new segment file. + // An error, if any occurred. + Merge(segments []segment.Segment, drops []*roaring.Bitmap, path string, + closeCh chan struct{}, s segment.StatsReporter) ( + [][]uint64, uint64, error) + + MergeUsing(segments []segment.Segment, drops []*roaring.Bitmap, path string, + closeCh chan struct{}, s segment.StatsReporter, config map[string]interface{}) ( + [][]uint64, uint64, error) +} + +var supportedSegmentPlugins map[string]map[uint32]SegmentPlugin +var defaultSegmentPlugin SegmentPlugin + +func init() { + ResetSegmentPlugins() + RegisterSegmentPlugin(&zapv17.ZapPlugin{}, true) + RegisterSegmentPlugin(&zapv16.ZapPlugin{}, false) + RegisterSegmentPlugin(&zapv15.ZapPlugin{}, false) + RegisterSegmentPlugin(&zapv14.ZapPlugin{}, false) + RegisterSegmentPlugin(&zapv13.ZapPlugin{}, false) + RegisterSegmentPlugin(&zapv12.ZapPlugin{}, false) + RegisterSegmentPlugin(&zapv11.ZapPlugin{}, false) +} + +func ResetSegmentPlugins() { + supportedSegmentPlugins = map[string]map[uint32]SegmentPlugin{} +} + +func RegisterSegmentPlugin(plugin SegmentPlugin, makeDefault bool) { + if _, ok := supportedSegmentPlugins[plugin.Type()]; !ok { + supportedSegmentPlugins[plugin.Type()] = map[uint32]SegmentPlugin{} + } + supportedSegmentPlugins[plugin.Type()][plugin.Version()] = plugin + if makeDefault { + defaultSegmentPlugin = plugin + } +} + +func SupportedSegmentTypes() (rv []string) { + for k := range supportedSegmentPlugins { + rv = append(rv, k) + } + return +} + +func SupportedSegmentTypeVersions(typ string) (rv []uint32) { + for k := range supportedSegmentPlugins[typ] { + rv = append(rv, k) + } + return rv +} + +func chooseSegmentPlugin(forcedSegmentType string, + forcedSegmentVersion uint32) (SegmentPlugin, error) { + if versions, ok := supportedSegmentPlugins[forcedSegmentType]; ok { + if segPlugin, ok := versions[uint32(forcedSegmentVersion)]; ok { + return segPlugin, nil + } + return nil, fmt.Errorf( + "unsupported version %d for segment type: %s, supported: %v", + forcedSegmentVersion, forcedSegmentType, + SupportedSegmentTypeVersions(forcedSegmentType)) + } + return nil, fmt.Errorf("unsupported segment type: %s, supported: %v", + forcedSegmentType, SupportedSegmentTypes()) +} + +func (s *Scorch) loadSegmentPlugin(forcedSegmentType string, + forcedSegmentVersion uint32) error { + segPlugin, err := chooseSegmentPlugin(forcedSegmentType, + forcedSegmentVersion) + if err != nil { + return err + } + s.segPlugin = segPlugin + return nil +} + +func (s *Scorch) loadSpatialAnalyzerPlugin(typ string) error { + s.spatialPlugin = geo.GetSpatialAnalyzerPlugin(typ) + if s.spatialPlugin == nil { + return fmt.Errorf("unsupported spatial plugin type: %s", typ) + } + return nil +} diff --git a/index/scorch/snapshot_index.go b/index/scorch/snapshot_index.go new file mode 100644 index 0000000..a363ecb --- /dev/null +++ b/index/scorch/snapshot_index.go @@ -0,0 +1,1504 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "container/heap" + "context" + "fmt" + "os" + "path/filepath" + "reflect" + "sort" + "sync" + "sync/atomic" + + "github.com/RoaringBitmap/roaring/v2" + "github.com/blevesearch/bleve/v2/document" + geov2 "github.com/blevesearch/bleve/v2/geov2" + "github.com/blevesearch/bleve/v2/size" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" + "github.com/blevesearch/vellum" + lev "github.com/blevesearch/vellum/levenshtein" +) + +// re usable, threadsafe levenshtein builders +var lb1, lb2 *lev.LevenshteinAutomatonBuilder + +type asynchSegmentResult struct { + dict segment.TermDictionary + dictItr segment.DictionaryIterator + + index int + docs *roaring.Bitmap + + thesItr segment.ThesaurusIterator + + err error +} + +var reflectStaticSizeIndexSnapshot int +var reflectStaticSizeIndexSnapshotGeoShapeV2Reader int +var reflectStaticSizeRoaringIntIterator int + +func init() { + var is interface{} = IndexSnapshot{} + reflectStaticSizeIndexSnapshot = int(reflect.TypeOf(is).Size()) + var err error + lb1, err = lev.NewLevenshteinAutomatonBuilder(1, true) + if err != nil { + panic(fmt.Errorf("levenshtein automaton ed1 builder err: %v", err)) + } + lb2, err = lev.NewLevenshteinAutomatonBuilder(2, true) + if err != nil { + panic(fmt.Errorf("levenshtein automaton ed2 builder err: %v", err)) + } + var gcr IndexSnapshotGeoShapeV2Reader + reflectStaticSizeIndexSnapshotGeoShapeV2Reader = int(reflect.TypeOf(gcr).Size()) + var rip roaring.IntIterator + reflectStaticSizeRoaringIntIterator = int(reflect.TypeOf(rip).Size()) +} + +type IndexSnapshot struct { + parent *Scorch + segment []*SegmentSnapshot + offsets []uint64 + internal map[string][]byte + epoch uint64 + size uint64 + creator string + + m sync.Mutex // Protects the fields that follow. + refs int64 + + m2 sync.Mutex // Protects the fields that follow. + fieldTFRs map[string][]*IndexSnapshotTermFieldReader // keyed by field, recycled TFR's + + m3 sync.RWMutex // bm25 metrics specific - not to interfere with TFR creation + fieldCardinality map[string]int + + // Stores information about zapx fields that have been + // fully deleted (indicated by UpdateFieldInfo.Deleted) or + // partially deleted index, store or docvalues (indicated by + // UpdateFieldInfo.Index or .Store or .DocValues). + // Used to short circuit queries trying to read stale data + updatedFields map[string]*index.UpdateFieldInfo + + fileWriterID string // the file callback writer id associated with this snapshot +} + +func (i *IndexSnapshot) Segments() []*SegmentSnapshot { + return i.segment +} + +func (i *IndexSnapshot) Internal() map[string][]byte { + return i.internal +} + +func (i *IndexSnapshot) AddRef() { + i.m.Lock() + i.refs++ + i.m.Unlock() +} + +func (i *IndexSnapshot) DecRef() (err error) { + i.m.Lock() + i.refs-- + if i.refs == 0 { + for _, s := range i.segment { + if s != nil { + err2 := s.segment.DecRef() + if err == nil { + err = err2 + } + } + } + if i.parent != nil { + go i.parent.AddEligibleForRemoval(i.epoch) + } + } + i.m.Unlock() + return err +} + +func (i *IndexSnapshot) Close() error { + return i.DecRef() +} + +func (i *IndexSnapshot) Size() int { + return int(i.size) +} + +func (i *IndexSnapshot) updateSize() { + i.size += uint64(reflectStaticSizeIndexSnapshot) + for _, s := range i.segment { + i.size += uint64(s.Size()) + } +} + +func (is *IndexSnapshot) newIndexSnapshotFieldDict(field string, + makeItr func(i segment.TermDictionary) segment.DictionaryIterator, + randomLookup bool, +) (*IndexSnapshotFieldDict, error) { + results := make(chan *asynchSegmentResult, len(is.segment)) + var totalBytesRead uint64 + var fieldCardinality int64 + for _, s := range is.segment { + go func(s *SegmentSnapshot) { + dict, err := s.segment.Dictionary(field) + if err != nil { + results <- &asynchSegmentResult{err: err} + } else { + if dictStats, ok := dict.(segment.DiskStatsReporter); ok { + atomic.AddUint64(&totalBytesRead, dictStats.BytesRead()) + } + atomic.AddInt64(&fieldCardinality, int64(dict.Cardinality())) + if randomLookup { + results <- &asynchSegmentResult{dict: dict} + } else { + results <- &asynchSegmentResult{dictItr: makeItr(dict)} + } + } + }(s) + } + + var err error + rv := &IndexSnapshotFieldDict{ + snapshot: is, + cursors: make([]*segmentDictCursor, 0, len(is.segment)), + } + + for count := 0; count < len(is.segment); count++ { + asr := <-results + if asr.err != nil && err == nil { + err = asr.err + } else { + if !randomLookup { + next, err2 := asr.dictItr.Next() + if err2 != nil && err == nil { + err = err2 + } + if next != nil { + rv.cursors = append(rv.cursors, &segmentDictCursor{ + itr: asr.dictItr, + curr: *next, + }) + } + } else { + rv.cursors = append(rv.cursors, &segmentDictCursor{ + dict: asr.dict, + }) + } + } + } + rv.cardinality = int(fieldCardinality) + rv.bytesRead = totalBytesRead + // after ensuring we've read all items on channel + if err != nil { + return nil, err + } + + if !randomLookup { + // prepare heap + heap.Init(rv) + } + + return rv, nil +} + +func (is *IndexSnapshot) FieldCardinality(field string) (rv int, err error) { + is.m3.RLock() + rv, ok := is.fieldCardinality[field] + is.m3.RUnlock() + if ok { + return rv, nil + } + + is.m3.Lock() + defer is.m3.Unlock() + if is.fieldCardinality == nil { + is.fieldCardinality = make(map[string]int) + } + // check again to avoid redundant fieldDict creation + if rv, ok := is.fieldCardinality[field]; ok { + return rv, nil + } + + fd, err := is.FieldDict(field) + if err != nil { + return rv, err + } + rv = fd.Cardinality() + is.fieldCardinality[field] = rv + return rv, nil +} + +func (is *IndexSnapshot) FieldDict(field string) (index.FieldDict, error) { + return is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator { + return is.AutomatonIterator(nil, nil, nil) + }, false) +} + +// calculateExclusiveEndFromInclusiveEnd produces the next key +// when sorting using memcmp style comparisons, suitable to +// use as the end key in a traditional (inclusive, exclusive] +// start/end range +func calculateExclusiveEndFromInclusiveEnd(inclusiveEnd []byte) []byte { + rv := inclusiveEnd + if len(inclusiveEnd) > 0 { + rv = make([]byte, len(inclusiveEnd)) + copy(rv, inclusiveEnd) + if rv[len(rv)-1] < 0xff { + // last byte can be incremented by one + rv[len(rv)-1]++ + } else { + // last byte is already 0xff, so append 0 + // next key is simply one byte longer + rv = append(rv, 0x0) + } + } + return rv +} + +func (is *IndexSnapshot) FieldDictRange(field string, startTerm []byte, + endTerm []byte, +) (index.FieldDict, error) { + return is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator { + endTermExclusive := calculateExclusiveEndFromInclusiveEnd(endTerm) + return is.AutomatonIterator(nil, startTerm, endTermExclusive) + }, false) +} + +// calculateExclusiveEndFromPrefix produces the first key that +// does not have the same prefix as the input bytes, suitable +// to use as the end key in a traditional (inclusive, exclusive] +// start/end range +func calculateExclusiveEndFromPrefix(in []byte) []byte { + if len(in) == 0 { + return nil + } + rv := make([]byte, len(in)) + copy(rv, in) + for i := len(rv) - 1; i >= 0; i-- { + rv[i]++ + if rv[i] != 0 { + return rv // didn't overflow, so stop + } + } + // all bytes were 0xff, so return nil + // as there is no end key for this prefix + return nil +} + +func (is *IndexSnapshot) FieldDictPrefix(field string, + termPrefix []byte, +) (index.FieldDict, error) { + termPrefixEnd := calculateExclusiveEndFromPrefix(termPrefix) + return is.newIndexSnapshotFieldDict(field, func(is segment.TermDictionary) segment.DictionaryIterator { + return is.AutomatonIterator(nil, termPrefix, termPrefixEnd) + }, false) +} + +func (is *IndexSnapshot) FieldDictRegexp(field string, + termRegex string, +) (index.FieldDict, error) { + fd, _, err := is.FieldDictRegexpAutomaton(field, termRegex) + return fd, err +} + +func (is *IndexSnapshot) FieldDictRegexpAutomaton(field string, + termRegex string, +) (index.FieldDict, index.RegexAutomaton, error) { + return is.fieldDictRegexp(field, termRegex) +} + +func (is *IndexSnapshot) fieldDictRegexp(field string, + termRegex string, +) (index.FieldDict, index.RegexAutomaton, error) { + // TODO: potential optimization where the literal prefix represents the, + // entire regexp, allowing us to use PrefixIterator(prefixTerm)? + + a, prefixBeg, prefixEnd, err := parseRegexp(termRegex) + if err != nil { + return nil, nil, err + } + + fd, err := is.newIndexSnapshotFieldDict(field, func(dict segment.TermDictionary) segment.DictionaryIterator { + // regexp/wildcard candidate collection discards DictEntry.Count. + return automatonIteratorOmitCount(dict, a, prefixBeg, prefixEnd) + }, false) + if err != nil { + return nil, nil, err + } + return fd, a, nil +} + +func (is *IndexSnapshot) getLevAutomaton(term string, + fuzziness uint8, +) (vellum.Automaton, error) { + switch fuzziness { + case 1: + return lb1.BuildDfa(term, fuzziness) + case 2: + return lb2.BuildDfa(term, fuzziness) + } + return nil, fmt.Errorf("fuzziness exceeds the max limit") +} + +func (is *IndexSnapshot) FieldDictFuzzy(field string, + term string, fuzziness int, prefix string, +) (index.FieldDict, error) { + fd, _, err := is.FieldDictFuzzyAutomaton(field, term, fuzziness, prefix) + return fd, err +} + +func (is *IndexSnapshot) FieldDictFuzzyAutomaton(field string, + term string, fuzziness int, prefix string, +) (index.FieldDict, index.FuzzyAutomaton, error) { + return is.fieldDictFuzzy(field, term, fuzziness, prefix) +} + +func (is *IndexSnapshot) fieldDictFuzzy(field string, + term string, fuzziness int, prefix string, +) (index.FieldDict, index.FuzzyAutomaton, error) { + a, err := is.getLevAutomaton(term, uint8(fuzziness)) + if err != nil { + return nil, nil, err + } + var fa index.FuzzyAutomaton + if vfa, ok := a.(vellum.FuzzyAutomaton); ok { + fa = vfa + } + var prefixBeg, prefixEnd []byte + if prefix != "" { + prefixBeg = []byte(prefix) + prefixEnd = calculateExclusiveEndFromPrefix(prefixBeg) + } + fd, err := is.newIndexSnapshotFieldDict(field, func(dict segment.TermDictionary) segment.DictionaryIterator { + // fuzzy candidate collection discards DictEntry.Count. + return automatonIteratorOmitCount(dict, a, prefixBeg, prefixEnd) + }, false) + if err != nil { + return nil, nil, err + } + return fd, fa, nil +} + +func (is *IndexSnapshot) FieldDictContains(field string) (index.FieldDictContains, error) { + return is.newIndexSnapshotFieldDict(field, nil, true) +} + +func (is *IndexSnapshot) DocIDReaderAll() (index.DocIDReader, error) { + results := make(chan *asynchSegmentResult, len(is.segment)) + for index, segment := range is.segment { + go func(index int, segment *SegmentSnapshot) { + results <- &asynchSegmentResult{ + index: index, + docs: segment.DocNumbersLive(), + } + }(index, segment) + } + + return is.newDocIDReader(results) +} + +func (is *IndexSnapshot) DocIDReaderOnly(ids []string) (index.DocIDReader, error) { + results := make(chan *asynchSegmentResult, len(is.segment)) + for index, segment := range is.segment { + go func(index int, segment *SegmentSnapshot) { + docs, err := segment.DocNumbers(ids) + if err != nil { + results <- &asynchSegmentResult{err: err} + } else { + results <- &asynchSegmentResult{ + index: index, + docs: docs, + } + } + }(index, segment) + } + + return is.newDocIDReader(results) +} + +func (is *IndexSnapshot) newDocIDReader(results chan *asynchSegmentResult) (index.DocIDReader, error) { + rv := &IndexSnapshotDocIDReader{ + snapshot: is, + iterators: make([]roaring.IntIterable, len(is.segment)), + } + var err error + for count := 0; count < len(is.segment); count++ { + asr := <-results + if asr.err != nil { + if err == nil { + // returns the first error encountered + err = asr.err + } + } else if err == nil { + rv.iterators[asr.index] = asr.docs.Iterator() + } + } + + if err != nil { + return nil, err + } + + return rv, nil +} + +func (is *IndexSnapshot) Fields() ([]string, error) { + // FIXME not making this concurrent for now as it's not used in hot path + // of any searches at the moment (just a debug aid) + fieldsMap := make(map[string]struct{}) + for _, segment := range is.segment { + fields := segment.Fields() + for _, field := range fields { + fieldsMap[field] = struct{}{} + } + } + rv := make([]string, 0, len(fieldsMap)) + for k := range fieldsMap { + rv = append(rv, k) + } + return rv, nil +} + +func (is *IndexSnapshot) GetInternal(key []byte) ([]byte, error) { + _, ok := is.internal[string(key)] + if !ok { + return is.parent.getInternal(key) + } + return is.internal[string(key)], nil +} + +func (is *IndexSnapshot) DocCount() (uint64, error) { + var rv uint64 + for _, segment := range is.segment { + rv += segment.CountRoot() + } + return rv, nil +} + +func (is *IndexSnapshot) Document(id string) (rv index.Document, err error) { + // FIXME could be done more efficiently directly, but reusing for simplicity + tfr, err := is.TermFieldReader(context.TODO(), []byte(id), "_id", false, false, false) + if err != nil { + return nil, err + } + defer func() { + if cerr := tfr.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + next, err := tfr.Next(nil) + if err != nil { + return nil, err + } + + if next == nil { + // no such doc exists + return nil, nil + } + + docNum := next.ID.Value() + segmentIndex, localDocNum := is.segmentIndexAndLocalDocNumFromGlobal(docNum) + + rvd := document.NewDocument(id) + + err = is.segment[segmentIndex].VisitDocument(localDocNum, func(name string, typ byte, val []byte, pos []uint64) bool { + if name == "_id" { + return true + } + + // track uncompressed stored fields bytes as part of IO stats. + // However, ideally we'd need to track the compressed on-disk value + // Keeping that TODO for now until we have a cleaner way. + rvd.StoredFieldsSize += uint64(len(val)) + + // Skip fields that have been completely deleted or had their + // store data deleted + if info, ok := is.updatedFields[name]; ok && + (info.Deleted || info.Store) { + return true + } + + // copy value, array positions to preserve them beyond the scope of this callback + value := append([]byte(nil), val...) + arrayPos := append([]uint64(nil), pos...) + + switch typ { + case 't': + rvd.AddField(document.NewTextField(name, arrayPos, value)) + case 'n': + rvd.AddField(document.NewNumericFieldFromBytes(name, arrayPos, value)) + case 'i': + rvd.AddField(document.NewIPFieldFromBytes(name, arrayPos, value)) + case 'd': + rvd.AddField(document.NewDateTimeFieldFromBytes(name, arrayPos, value)) + case 'b': + rvd.AddField(document.NewBooleanFieldFromBytes(name, arrayPos, value)) + case 'g': + rvd.AddField(document.NewGeoPointFieldFromBytes(name, arrayPos, value)) + case 's': + rvd.AddField(document.NewGeoShapeFieldFromBytes(name, arrayPos, value)) + } + + return true + }) + if err != nil { + return nil, err + } + + return rvd, nil +} + +// In a multi-segment index, each document has: +// 1. a local docnum - local to the segment +// 2. a global docnum - unique identifier across the index +// This function returns the segment index(the segment in which the docnum is present) +// and local docnum of a document. +func (is *IndexSnapshot) segmentIndexAndLocalDocNumFromGlobal(docNum uint64) (int, uint64) { + segmentIndex := sort.Search(len(is.offsets), + func(x int) bool { + return is.offsets[x] > docNum + }) - 1 + + return int(segmentIndex), docNum - is.offsets[segmentIndex] +} + +func (is *IndexSnapshot) ExternalID(id index.IndexInternalID) (string, error) { + docNum := id.Value() + segmentIndex, localDocNum := is.segmentIndexAndLocalDocNumFromGlobal(docNum) + + v, err := is.segment[segmentIndex].DocID(localDocNum) + if err != nil { + return "", err + } + if v == nil { + return "", fmt.Errorf("document number %d not found", docNum) + } + + return string(v), nil +} + +func (is *IndexSnapshot) segmentIndexAndLocalDocNum(id index.IndexInternalID) (int, uint64, error) { + docNum := id.Value() + segIdx, localDocNum := is.segmentIndexAndLocalDocNumFromGlobal(docNum) + return segIdx, localDocNum, nil +} + +func (is *IndexSnapshot) InternalID(id string) (rv index.IndexInternalID, err error) { + // FIXME could be done more efficiently directly, but reusing for simplicity + tfr, err := is.TermFieldReader(context.TODO(), []byte(id), "_id", false, false, false) + if err != nil { + return nil, err + } + defer func() { + if cerr := tfr.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + next, err := tfr.Next(nil) + if err != nil || next == nil { + return nil, err + } + + return next.ID, nil +} + +func (is *IndexSnapshot) TermFieldReader(ctx context.Context, term []byte, field string, includeFreq, + includeNorm, includeTermVectors bool, +) (index.TermFieldReader, error) { + rv := is.allocTermFieldReaderDicts(field) + + rv.ctx = ctx + rv.term = term + rv.field = field + rv.snapshot = is + if rv.postings == nil { + rv.postings = make([]segment.PostingsList, len(is.segment)) + } + if rv.iterators == nil { + rv.iterators = make([]segment.PostingsIterator, len(is.segment)) + } + rv.segmentOffset = 0 + rv.includeFreq = includeFreq + rv.includeNorm = includeNorm + rv.includeTermVectors = includeTermVectors + rv.currPosting = nil + rv.currID = rv.currID[:0] + + if rv.dicts == nil { + rv.dicts = make([]segment.TermDictionary, len(is.segment)) + for i, s := range is.segment { + // the intention behind this compare and swap operation is + // to make sure that the accounting of the metadata is happening + // only once(which corresponds to this persisted segment's most + // recent segPlugin.Open() call), and any subsequent queries won't + // incur this cost which would essentially be a double counting. + if atomic.CompareAndSwapUint32(&s.mmaped, 1, 0) { + segBytesRead := s.segment.BytesRead() + rv.incrementBytesRead(segBytesRead) + } + + var dict segment.TermDictionary + var err error + + // Skip fields that have been completely deleted or had their + // index data deleted + if info, ok := is.updatedFields[field]; ok && + (info.Index || info.Deleted) { + dict, err = s.segment.Dictionary("") + } else { + dict, err = s.segment.Dictionary(field) + } + if err != nil { + return nil, err + } + + if dictStats, ok := dict.(segment.DiskStatsReporter); ok { + bytesRead := dictStats.BytesRead() + rv.incrementBytesRead(bytesRead) + } + rv.dicts[i] = dict + } + } + + for i, s := range is.segment { + var prevBytesReadPL uint64 + if rv.postings[i] != nil { + prevBytesReadPL = rv.postings[i].BytesRead() + } + pl, err := rv.dicts[i].PostingsList(term, s.deleted, rv.postings[i]) + if err != nil { + return nil, err + } + rv.postings[i] = pl + + var prevBytesReadItr uint64 + if rv.iterators[i] != nil { + prevBytesReadItr = rv.iterators[i].BytesRead() + } + rv.iterators[i] = pl.Iterator(includeFreq, includeNorm, includeTermVectors, rv.iterators[i]) + + if bytesRead := rv.postings[i].BytesRead(); prevBytesReadPL < bytesRead { + rv.incrementBytesRead(bytesRead - prevBytesReadPL) + } + + if bytesRead := rv.iterators[i].BytesRead(); prevBytesReadItr < bytesRead { + rv.incrementBytesRead(bytesRead - prevBytesReadItr) + } + } + // ONLY update the bytes read value beyond this point for this TFR if scoring is enabled + rv.updateBytesRead = rv.includeFreq || rv.includeNorm || rv.includeTermVectors + atomic.AddUint64(&is.parent.stats.TotTermSearchersStarted, uint64(1)) + return rv, nil +} + +func (is *IndexSnapshot) allocTermFieldReaderDicts(field string) (tfr *IndexSnapshotTermFieldReader) { + is.m2.Lock() + if is.fieldTFRs != nil { + tfrs := is.fieldTFRs[field] + last := len(tfrs) - 1 + if last >= 0 { + tfr = tfrs[last] + tfrs[last] = nil + is.fieldTFRs[field] = tfrs[:last] + is.m2.Unlock() + return + } + } + is.m2.Unlock() + return &IndexSnapshotTermFieldReader{ + recycle: true, + } +} + +// DefaultFieldTFRCacheThreshold limits the number of TermFieldReaders(TFR) for +// a field in an index snapshot. Without this limit, when recycling TFRs, it is +// possible that a very large number of TFRs may be added to the recycle +// cache, which could eventually lead to significant memory consumption. +// This threshold can be overwritten by users at the library level by changing the +// exported variable, or at the index level by setting the "fieldTFRCacheThreshold" +// in the kvConfig. +var DefaultFieldTFRCacheThreshold int = 0 // disabled because it causes MB-64604 + +func (is *IndexSnapshot) getFieldTFRCacheThreshold() int { + if is.parent.config != nil { + if val, exists := is.parent.config["fieldTFRCacheThreshold"]; exists { + if x, ok := val.(float64); ok { + // JSON unmarshal-ed into a map[string]interface{} will default + // to float64 for numbers, so we need to check for float64 first. + return int(x) + } else if x, ok := val.(int); ok { + // If library users provided an int in the config, we'll honor it. + return x + } + } + } + return DefaultFieldTFRCacheThreshold +} + +func (is *IndexSnapshot) recycleTermFieldReader(tfr *IndexSnapshotTermFieldReader) { + if !tfr.recycle { + // Do not recycle an optimized unadorned term field reader (used for + // ConjunctionUnadorned or DisjunctionUnadorned), during when a fresh + // roaring.Bitmap is built by AND-ing or OR-ing individual bitmaps, + // and we'll need to release them for GC. (See MB-40916) + return + } + + is.parent.rootLock.RLock() + obsolete := is.parent.root != is + is.parent.rootLock.RUnlock() + if obsolete { + // if we're not the current root (mutations happened), don't bother recycling + return + } + + is.m2.Lock() + if is.fieldTFRs == nil { + is.fieldTFRs = make(map[string][]*IndexSnapshotTermFieldReader) + } + if len(is.fieldTFRs[tfr.field]) < is.getFieldTFRCacheThreshold() { + tfr.bytesRead = 0 + is.fieldTFRs[tfr.field] = append(is.fieldTFRs[tfr.field], tfr) + } + is.m2.Unlock() +} + +func (is *IndexSnapshot) documentVisitFieldTermsOnSegment( + segmentIndex int, localDocNum uint64, fields []string, cFields []string, + visitor index.DocValueVisitor, dvs segment.DocVisitState) ( + cFieldsOut []string, dvsOut segment.DocVisitState, err error, +) { + ss := is.segment[segmentIndex] + + var vFields []string // fields that are visitable via the segment + + ssv, ssvOk := ss.segment.(segment.DocValueVisitable) + if ssvOk && ssv != nil { + vFields, err = ssv.VisitableDocValueFields() + if err != nil { + return nil, nil, err + } + } + + // Filter out fields that have been completely deleted or had their + // docvalues data deleted from the visitable fields + filterUpdatedDVFields := func(fields []string) []string { + filteredFields := make([]string, 0, len(fields)) + for _, field := range fields { + if info, ok := is.updatedFields[field]; ok && + (info.DocValues || info.Deleted) { + continue + } + filteredFields = append(filteredFields, field) + } + return filteredFields + } + + // Filter out fields that have been completely deleted or had their + // index data deleted from the cached fields + filterUpdatedIndexFields := func(fields []string) []string { + filteredFields := make([]string, 0, len(fields)) + for _, field := range fields { + if info, ok := is.updatedFields[field]; ok && + (info.Index || info.Deleted) { + continue + } + filteredFields = append(filteredFields, field) + } + return filteredFields + } + + if len(is.updatedFields) > 0 { + vFields = filterUpdatedDVFields(vFields) + } + + var errCh chan error + + // cFields represents the fields that we'll need from the + // cachedDocs, and might be optionally be provided by the caller, + // if the caller happens to know we're on the same segmentIndex + // from a previous invocation + if cFields == nil { + cFields = subtractStrings(fields, vFields) + } + if len(is.updatedFields) > 0 { + cFields = filterUpdatedIndexFields(cFields) + } + + if len(cFields) > 0 && !ss.cachedDocs.hasFields(cFields) { + errCh = make(chan error, 1) + + go func() { + err := ss.cachedDocs.prepareFields(cFields, ss) + if err != nil { + errCh <- err + } + close(errCh) + }() + } + + if ssvOk && ssv != nil && len(vFields) > 0 { + dvs, err = ssv.VisitDocValues(localDocNum, fields, visitor, dvs) + if err != nil { + return nil, nil, err + } + } + + if errCh != nil { + err = <-errCh + if err != nil { + return nil, nil, err + } + } + + if len(cFields) > 0 { + ss.cachedDocs.visitDoc(localDocNum, cFields, visitor) + } + + return cFields, dvs, nil +} + +func (is *IndexSnapshot) DocValueReader(fields []string) ( + index.DocValueReader, error, +) { + return &DocValueReader{i: is, fields: fields, currSegmentIndex: -1}, nil +} + +type DocValueReader struct { + i *IndexSnapshot + fields []string + dvs segment.DocVisitState + + currSegmentIndex int + currCachedFields []string + + totalBytesRead uint64 + bytesRead uint64 +} + +func (dvr *DocValueReader) BytesRead() uint64 { + return dvr.totalBytesRead + dvr.bytesRead +} + +func (dvr *DocValueReader) VisitDocValues(id index.IndexInternalID, + visitor index.DocValueVisitor, +) (err error) { + docNum := id.Value() + + segmentIndex, localDocNum := dvr.i.segmentIndexAndLocalDocNumFromGlobal(docNum) + if segmentIndex >= len(dvr.i.segment) { + return nil + } + + if dvr.currSegmentIndex != segmentIndex { + dvr.currSegmentIndex = segmentIndex + dvr.currCachedFields = nil + dvr.totalBytesRead += dvr.bytesRead + dvr.bytesRead = 0 + } + + dvr.currCachedFields, dvr.dvs, err = dvr.i.documentVisitFieldTermsOnSegment( + dvr.currSegmentIndex, localDocNum, dvr.fields, dvr.currCachedFields, visitor, dvr.dvs) + + if dvr.dvs != nil { + dvr.bytesRead = dvr.dvs.BytesRead() + } + return err +} + +func (is *IndexSnapshot) DumpAll() chan interface{} { + rv := make(chan interface{}) + go func() { + close(rv) + }() + return rv +} + +func (is *IndexSnapshot) DumpDoc(id string) chan interface{} { + rv := make(chan interface{}) + go func() { + close(rv) + }() + return rv +} + +func (is *IndexSnapshot) DumpFields() chan interface{} { + rv := make(chan interface{}) + go func() { + close(rv) + }() + return rv +} + +func (is *IndexSnapshot) diskSegmentsPaths() map[string]struct{} { + rv := make(map[string]struct{}, len(is.segment)) + for _, s := range is.segment { + if seg, ok := s.segment.(segment.PersistedSegment); ok { + rv[seg.Path()] = struct{}{} + } + } + return rv +} + +// reClaimableDocsRatio gives a ratio about the obsoleted or +// reclaimable documents present in a given index snapshot. +func (is *IndexSnapshot) reClaimableDocsRatio() float64 { + var totalCount, liveCount uint64 + for _, s := range is.segment { + if _, ok := s.segment.(segment.PersistedSegment); ok { + totalCount += uint64(s.FullSize()) + liveCount += uint64(s.Count()) + } + } + + if totalCount > 0 { + return float64(totalCount-liveCount) / float64(totalCount) + } + return 0 +} + +// subtractStrings returns set a minus elements of set b. +func subtractStrings(a, b []string) []string { + if len(b) == 0 { + return a + } + + rv := make([]string, 0, len(a)) +OUTER: + for _, as := range a { + for _, bs := range b { + if as == bs { + continue OUTER + } + } + rv = append(rv, as) + } + return rv +} + +func (is *IndexSnapshot) CopyTo(d index.Directory) error { + // get the root bolt file. + w, err := d.GetWriter(filepath.Join("store", "root.bolt")) + if err != nil || w == nil { + return fmt.Errorf("failed to create the root.bolt file, err: %v", err) + } + rootFile, ok := w.(*os.File) + if !ok { + return fmt.Errorf("invalid root.bolt file found") + } + + copyBolt, err := util.OpenBolt(rootFile.Name(), 0o600, nil) + if err != nil { + return err + } + defer func() { + w.Close() + if cerr := copyBolt.Close(); cerr != nil && err == nil { + err = cerr + } + }() + + // start a write transaction + tx, err := copyBolt.Begin(true) + if err != nil { + return err + } + + _, _, err = prepareBoltSnapshot(is, tx, "", is.parent.segPlugin, d) + if err != nil { + _ = tx.Rollback() + return fmt.Errorf("error backing up index snapshot: %v", err) + } + + // commit bolt data + err = tx.Commit() + if err != nil { + return fmt.Errorf("error commit tx to backup root bolt: %v", err) + } + + return copyBolt.Sync() +} + +func (is *IndexSnapshot) UpdateIOStats(val uint64) { + atomic.AddUint64(&is.parent.stats.TotBytesReadAtQueryTime, val) +} + +func (is *IndexSnapshot) GetSpatialAnalyzerPlugin(typ string) ( + index.SpatialAnalyzerPlugin, error, +) { + var rv index.SpatialAnalyzerPlugin + is.m.Lock() + rv = is.parent.spatialPlugin + is.m.Unlock() + + if rv == nil { + return nil, fmt.Errorf("no spatial plugin type: %s found", typ) + } + return rv, nil +} + +func (is *IndexSnapshot) CloseCopyReader() error { + // first unmark the segments that were marked for backup by this index snapshot + is.parent.rootLock.Lock() + for _, seg := range is.segment { + var fileName string + if perSeg, ok := seg.segment.(segment.PersistedSegment); ok { + // segment is persisted + fileName = filepath.Base(perSeg.Path()) + } else { + // segment is not persisted + // the name of the segment file that is generated if the + // the segment is persisted in the future. + fileName = zapFileName(seg.id) + } + if is.parent.copyScheduled[fileName]--; is.parent.copyScheduled[fileName] <= 0 { + delete(is.parent.copyScheduled, fileName) + } + } + is.parent.rootLock.Unlock() + // close the index snapshot normally + return is.Close() +} + +func (is *IndexSnapshot) ThesaurusTermReader(ctx context.Context, thesaurusName string, term []byte) (index.ThesaurusTermReader, error) { + rv := &IndexSnapshotThesaurusTermReader{ + name: thesaurusName, + snapshot: is, + postings: make([]segment.SynonymsList, len(is.segment)), + iterators: make([]segment.SynonymsIterator, len(is.segment)), + thesauri: make([]segment.Thesaurus, len(is.segment)), + segmentOffset: 0, + } + + for i, s := range is.segment { + if synSeg, ok := s.segment.(segment.ThesaurusSegment); ok { + thes, err := synSeg.Thesaurus(thesaurusName) + if err != nil { + return nil, err + } + rv.thesauri[i] = thes + pl, err := rv.thesauri[i].SynonymsList(term, s.deleted, rv.postings[i]) + if err != nil { + return nil, err + } + rv.postings[i] = pl + + rv.iterators[i] = pl.Iterator(rv.iterators[i]) + } + } + return rv, nil +} + +func (is *IndexSnapshot) newIndexSnapshotThesaurusKeys(name string, + makeItr func(i segment.Thesaurus) segment.ThesaurusIterator, +) (*IndexSnapshotThesaurusKeys, error) { + results := make(chan *asynchSegmentResult, len(is.segment)) + var wg sync.WaitGroup + wg.Add(len(is.segment)) + for _, s := range is.segment { + go func(s *SegmentSnapshot) { + defer wg.Done() + if synSeg, ok := s.segment.(segment.ThesaurusSegment); ok { + thes, err := synSeg.Thesaurus(name) + if err != nil { + results <- &asynchSegmentResult{err: err} + } else { + results <- &asynchSegmentResult{thesItr: makeItr(thes)} + } + } + }(s) + } + // Close the channel after all goroutines complete + go func() { + wg.Wait() + close(results) + }() + + var err error + rv := &IndexSnapshotThesaurusKeys{ + snapshot: is, + cursors: make([]*segmentThesCursor, 0, len(is.segment)), + } + for asr := range results { + if asr.err != nil && err == nil { + err = asr.err + } else { + next, err2 := asr.thesItr.Next() + if err2 != nil && err == nil { + err = err2 + } + if next != nil { + rv.cursors = append(rv.cursors, &segmentThesCursor{ + itr: asr.thesItr, + curr: *next, + }) + } + } + } + // after ensuring we've read all items on channel + if err != nil { + return nil, err + } + + return rv, nil +} + +func (is *IndexSnapshot) ThesaurusKeys(name string) (index.ThesaurusKeys, error) { + return is.newIndexSnapshotThesaurusKeys(name, func(is segment.Thesaurus) segment.ThesaurusIterator { + return is.AutomatonIterator(nil, nil, nil) + }) +} + +func (is *IndexSnapshot) ThesaurusKeysFuzzy(name string, + term string, fuzziness int, prefix string, +) (index.ThesaurusKeys, error) { + a, err := is.getLevAutomaton(term, uint8(fuzziness)) + if err != nil { + return nil, err + } + var prefixBeg, prefixEnd []byte + if prefix != "" { + prefixBeg = []byte(prefix) + prefixEnd = calculateExclusiveEndFromPrefix(prefixBeg) + } + return is.newIndexSnapshotThesaurusKeys(name, func(is segment.Thesaurus) segment.ThesaurusIterator { + return is.AutomatonIterator(a, prefixBeg, prefixEnd) + }) +} + +func (is *IndexSnapshot) ThesaurusKeysPrefix(name string, + termPrefix []byte, +) (index.ThesaurusKeys, error) { + termPrefixEnd := calculateExclusiveEndFromPrefix(termPrefix) + return is.newIndexSnapshotThesaurusKeys(name, func(is segment.Thesaurus) segment.ThesaurusIterator { + return is.AutomatonIterator(nil, termPrefix, termPrefixEnd) + }) +} + +func (is *IndexSnapshot) ThesaurusKeysRegexp(name string, + termRegex string, +) (index.ThesaurusKeys, error) { + a, prefixBeg, prefixEnd, err := parseRegexp(termRegex) + if err != nil { + return nil, err + } + return is.newIndexSnapshotThesaurusKeys(name, func(is segment.Thesaurus) segment.ThesaurusIterator { + return is.AutomatonIterator(a, prefixBeg, prefixEnd) + }) +} + +func (is *IndexSnapshot) UpdateSynonymSearchCount(delta uint64) { + atomic.AddUint64(&is.parent.stats.TotSynonymSearches, delta) +} + +// Update current snapshot updated field data as well as pass it on to all segments and segment bases +func (is *IndexSnapshot) UpdateFieldsInfo(updatedFields map[string]*index.UpdateFieldInfo) { + is.m.Lock() + defer is.m.Unlock() + + is.MergeUpdateFieldsInfo(updatedFields) + + for _, segmentSnapshot := range is.segment { + segmentSnapshot.UpdateFieldsInfo(is.updatedFields) + } +} + +// Merge given updated field information with existing updated field information +func (is *IndexSnapshot) MergeUpdateFieldsInfo(updatedFields map[string]*index.UpdateFieldInfo) { + if is.updatedFields == nil { + is.updatedFields = updatedFields + } else { + for fieldName, info := range updatedFields { + if val, ok := is.updatedFields[fieldName]; ok { + val.Deleted = val.Deleted || info.Deleted + val.Index = val.Index || info.Index + val.DocValues = val.DocValues || info.DocValues + val.Store = val.Store || info.Store + } else { + is.updatedFields[fieldName] = info + } + } + } +} + +// TermFrequencies returns the top N terms ordered by the frequencies +// for a given field across all segments in the index snapshot. +func (is *IndexSnapshot) TermFrequencies(field string, limit int, descending bool) ( + termFreqs []index.TermFreq, err error) { + if len(is.segment) == 0 { + return nil, nil + } + + if limit <= 0 { + return nil, fmt.Errorf("limit must be positive") + } + + // Use FieldDict which aggregates term frequencies across all segments + fieldDict, err := is.FieldDict(field) + if err != nil { + return nil, fmt.Errorf("failed to get field dictionary for field %s: %v", field, err) + } + defer fieldDict.Close() + + // Preallocate slice with capacity equal to the number of unique terms + // in the field dictionary + termFreqs = make([]index.TermFreq, 0, fieldDict.Cardinality()) + + // Iterate through all terms using FieldDict + for { + dictEntry, err := fieldDict.Next() + if err != nil { + return nil, fmt.Errorf("error iterating field dictionary: %v", err) + } + if dictEntry == nil { + break // End of terms + } + + termFreqs = append(termFreqs, index.TermFreq{ + Term: dictEntry.Term, + Frequency: dictEntry.Count, + }) + } + + // Sort by frequency (descending or ascending) + sort.Slice(termFreqs, func(i, j int) bool { + if termFreqs[i].Frequency == termFreqs[j].Frequency { + // If frequencies are equal, sort by term lexicographically + return termFreqs[i].Term < termFreqs[j].Term + } + if descending { + return termFreqs[i].Frequency > termFreqs[j].Frequency + } + return termFreqs[i].Frequency < termFreqs[j].Frequency + }) + + if limit >= len(termFreqs) { + return termFreqs, nil + } + + return termFreqs[:limit], nil +} + +// Ancestors returns the ancestor IDs for the given document ID. The prealloc +// slice can be provided to avoid allocations downstream, and MUST be empty. +func (i *IndexSnapshot) Ancestors(ID index.IndexInternalID, prealloc []index.AncestorID) ([]index.AncestorID, error) { + // get segment and local doc num for the ID + seg, ldoc, err := i.segmentIndexAndLocalDocNum(ID) + if err != nil { + return nil, err + } + // get ancestors from the segment + prealloc = i.segment[seg].Ancestors(ldoc, prealloc) + // get global offset for the segment (correcting factor for multi-segment indexes) + globalOffset := i.offsets[seg] + // adjust ancestors to global doc numbers, not local to segment + for idx := range prealloc { + prealloc[idx] = prealloc[idx].Add(globalOffset) + } + // return adjusted ancestors + return prealloc, nil +} + +func (i *IndexSnapshot) GeoShapeV2FieldReader(ctx context.Context, field string) ( + index.GeoShapeV2FieldReader, error) { + + rv := &IndexSnapshotGeoShapeV2Reader{ + field: field, + postings: make([]*roaring.Bitmap, len(i.segment)), + iterators: make([]roaring.IntPeekable, len(i.segment)), + snapshot: i, + } + + return rv, nil +} + +type IndexSnapshotGeoShapeV2Reader struct { + field string + + postings []*roaring.Bitmap + iterators []roaring.IntPeekable + segmentOffset int + + snapshot *IndexSnapshot +} + +// Search performs a spatial search for the given GeoJSON shape and relation +// across all segments in the index snapshot. +func (g *IndexSnapshotGeoShapeV2Reader) Search(shape index.GeoJSON, + relation string) error { + + numSegments := len(g.snapshot.segment) + // create a single query object that is thread safe + // to be used across all segments + query := geov2.NewQuery(shape, relation) + + var wg sync.WaitGroup + wg.Add(numSegments) + + var errm sync.Mutex + var err error + // search each segment concurrently + for i := 0; i < numSegments; i++ { + go func(segID int) { + defer wg.Done() + err2 := g.searchSeg(segID, query) + if err2 != nil { + errm.Lock() + if err == nil { + err = err2 + } + errm.Unlock() + } + }(i) + } + wg.Wait() + + return err +} + +// searchSeg performs a spatial search for the given GeoJSON shape and relation +// on a single segment in the index snapshot. +func (g *IndexSnapshotGeoShapeV2Reader) searchSeg(segID int, + query geov2.Query) error { + + snapshot := g.snapshot.segment[segID] + geoSeg, ok := snapshot.segment.(segment.GeoShapeV2Segment) + if !ok { + return nil + } + + // obtain the geo shape data from the segment + geoData, err := geoSeg.GeoShapeV2Data(g.field, snapshot.deleted) + if err != nil { + return err + } + // return if segment does not have any geo shape data for the field + if geoData == nil { + return nil + } + // release the reference on the segment's cached geo data once the + // evaluation is done, so that the cache is free to evict it + defer geoData.Close() + + // evaluate the query against the geo shape data to + // get the matching document IDs + hits := query.Evaluate(geoData) + postings := roaring.New() + + docNums := geoData.DocNums() + + addFunc := func(docNumInternal int) { + postings.Add(docNums[docNumInternal]) + } + + hits.Iterate(addFunc) + + g.postings[segID] = postings + g.iterators[segID] = postings.Iterator() + + return nil +} + +// Next returns the next GeoShapeV2FieldDoc from the postings list across all segments in the index snapshot. +func (g *IndexSnapshotGeoShapeV2Reader) Next(preAlloced *index.GeoShapeV2FieldDoc) ( + *index.GeoShapeV2FieldDoc, error) { + rv := preAlloced + if rv == nil { + rv = &index.GeoShapeV2FieldDoc{} + } + + for g.segmentOffset < len(g.iterators) { + if !g.iterators[g.segmentOffset].HasNext() { + g.segmentOffset++ + continue + } + + next := g.iterators[g.segmentOffset].Next() + globalOffset := g.snapshot.offsets[g.segmentOffset] + rv.ID = index.NewIndexInternalID(rv.ID, uint64(next)+globalOffset) + return rv, nil + } + + return nil, nil +} + +// Advance moves the reader to the specified document ID, returning the corresponding GeoShapeV2FieldDoc if it exists. +func (g *IndexSnapshotGeoShapeV2Reader) Advance(ID index.IndexInternalID, + preAlloced *index.GeoShapeV2FieldDoc) (*index.GeoShapeV2FieldDoc, error) { + rv := preAlloced + if rv == nil { + rv = &index.GeoShapeV2FieldDoc{} + } + + num := ID.Value() + + segIdx, localDocNum := g.snapshot.segmentIndexAndLocalDocNumFromGlobal(num) + if segIdx >= len(g.iterators) { + return nil, fmt.Errorf("error advancing to doc number %d, segment "+ + "index %d out of bounds", num, segIdx) + } + + if g.segmentOffset > segIdx { + return nil, fmt.Errorf("error advancing to doc number %d, segment "+ + "index %d is less than current segment offset %d", num, segIdx, g.segmentOffset) + } + + g.segmentOffset = segIdx + g.iterators[g.segmentOffset].AdvanceIfNeeded(uint32(localDocNum)) + + return g.Next(rv) +} + +// Close is a no-op: the reader holds no resources of its own, and the +// segment-level geo data it reads from is owned and evicted by the +// segment's cache +func (g *IndexSnapshotGeoShapeV2Reader) Close() error { + return nil +} + +// Count returns the total number of documents across all segments +// in the index snapshot that match the GeoShapeV2FieldReader's criteria. +func (g *IndexSnapshotGeoShapeV2Reader) Count() uint64 { + var rv uint64 + for _, posting := range g.postings { + if posting != nil { + rv += uint64(posting.GetCardinality()) + } + } + return rv +} + +// Size returns the estimated size in bytes of the +// IndexSnapshotGeoShapeV2Reader, including its postings and iterators. +func (g *IndexSnapshotGeoShapeV2Reader) Size() int { + rv := reflectStaticSizeIndexSnapshotGeoShapeV2Reader + size.SizeOfPtr + + len(g.field) + size.SizeOfInt + + for _, posting := range g.postings { + rv += int(posting.GetSizeInBytes()) + } + + rv += (reflectStaticSizeRoaringIntIterator + size.SizeOfPtr) * len(g.iterators) + + return rv +} diff --git a/index/scorch/snapshot_index_dict.go b/index/scorch/snapshot_index_dict.go new file mode 100644 index 0000000..b3204e8 --- /dev/null +++ b/index/scorch/snapshot_index_dict.go @@ -0,0 +1,139 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "container/heap" + + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +// termDictionaryOmitCount is an optional interface a segment's TermDictionary +// may implement to return an automaton iterator that skips the per-term +// postings read used to populate DictEntry.Count. It is used by candidate-term +// collection (fuzzy/regexp) where the count is discarded, and falls back to the +// regular AutomatonIterator when the segment does not implement it. +type termDictionaryOmitCount interface { + AutomatonIteratorOmitCount(a segment.Automaton, + startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator +} + +// automatonIteratorOmitCount returns a count-omitting automaton iterator when +// the dictionary supports it, otherwise the standard one. +func automatonIteratorOmitCount(dict segment.TermDictionary, a segment.Automaton, + startKeyInclusive, endKeyExclusive []byte) segment.DictionaryIterator { + if oc, ok := dict.(termDictionaryOmitCount); ok { + return oc.AutomatonIteratorOmitCount(a, startKeyInclusive, endKeyExclusive) + } + return dict.AutomatonIterator(a, startKeyInclusive, endKeyExclusive) +} + +type segmentDictCursor struct { + dict segment.TermDictionary + itr segment.DictionaryIterator + curr index.DictEntry +} + +type IndexSnapshotFieldDict struct { + cardinality int + bytesRead uint64 + + snapshot *IndexSnapshot + cursors []*segmentDictCursor + entry index.DictEntry +} + +func (i *IndexSnapshotFieldDict) BytesRead() uint64 { + return i.bytesRead +} + +func (i *IndexSnapshotFieldDict) Len() int { return len(i.cursors) } +func (i *IndexSnapshotFieldDict) Less(a, b int) bool { + return i.cursors[a].curr.Term < i.cursors[b].curr.Term +} +func (i *IndexSnapshotFieldDict) Swap(a, b int) { + i.cursors[a], i.cursors[b] = i.cursors[b], i.cursors[a] +} + +func (i *IndexSnapshotFieldDict) Push(x interface{}) { + i.cursors = append(i.cursors, x.(*segmentDictCursor)) +} + +func (i *IndexSnapshotFieldDict) Pop() interface{} { + n := len(i.cursors) + x := i.cursors[n-1] + i.cursors = i.cursors[0 : n-1] + return x +} + +func (i *IndexSnapshotFieldDict) Next() (*index.DictEntry, error) { + if len(i.cursors) == 0 { + return nil, nil + } + i.entry = i.cursors[0].curr + next, err := i.cursors[0].itr.Next() + if err != nil { + return nil, err + } + if next == nil { + // at end of this cursor, remove it + heap.Pop(i) + } else { + // modified heap, fix it + i.cursors[0].curr = *next + heap.Fix(i, 0) + } + // look for any other entries with the exact same term + for len(i.cursors) > 0 && i.cursors[0].curr.Term == i.entry.Term { + i.entry.Count += i.cursors[0].curr.Count + next, err := i.cursors[0].itr.Next() + if err != nil { + return nil, err + } + if next == nil { + // at end of this cursor, remove it + heap.Pop(i) + } else { + // modified heap, fix it + i.cursors[0].curr = *next + heap.Fix(i, 0) + } + } + + return &i.entry, nil +} + +func (i *IndexSnapshotFieldDict) Cardinality() int { + return i.cardinality +} + +func (i *IndexSnapshotFieldDict) Close() error { + return nil +} + +func (i *IndexSnapshotFieldDict) Contains(key []byte) (bool, error) { + if len(i.cursors) == 0 { + return false, nil + } + + for _, cursor := range i.cursors { + if found, _ := cursor.dict.Contains(key); found { + return true, nil + } + } + + return false, nil +} diff --git a/index/scorch/snapshot_index_doc.go b/index/scorch/snapshot_index_doc.go new file mode 100644 index 0000000..4048a19 --- /dev/null +++ b/index/scorch/snapshot_index_doc.go @@ -0,0 +1,79 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "reflect" + + "github.com/RoaringBitmap/roaring/v2" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeIndexSnapshotDocIDReader int + +func init() { + var isdr IndexSnapshotDocIDReader + reflectStaticSizeIndexSnapshotDocIDReader = int(reflect.TypeOf(isdr).Size()) +} + +type IndexSnapshotDocIDReader struct { + snapshot *IndexSnapshot + iterators []roaring.IntIterable + segmentOffset int +} + +func (i *IndexSnapshotDocIDReader) Size() int { + return reflectStaticSizeIndexSnapshotDocIDReader + size.SizeOfPtr +} + +func (i *IndexSnapshotDocIDReader) Next() (index.IndexInternalID, error) { + for i.segmentOffset < len(i.iterators) { + if !i.iterators[i.segmentOffset].HasNext() { + i.segmentOffset++ + continue + } + next := i.iterators[i.segmentOffset].Next() + // make segment number into global number by adding offset + globalOffset := i.snapshot.offsets[i.segmentOffset] + return index.NewIndexInternalID(nil, uint64(next)+globalOffset), nil + } + return nil, nil +} + +func (i *IndexSnapshotDocIDReader) Advance(ID index.IndexInternalID) (index.IndexInternalID, error) { + // FIXME do something better + next, err := i.Next() + if err != nil { + return nil, err + } + if next == nil { + return nil, nil + } + for next.Compare(ID) < 0 { + next, err = i.Next() + if err != nil { + return nil, err + } + if next == nil { + break + } + } + return next, nil +} + +func (i *IndexSnapshotDocIDReader) Close() error { + return nil +} diff --git a/index/scorch/snapshot_index_str.go b/index/scorch/snapshot_index_str.go new file mode 100644 index 0000000..c73b56f --- /dev/null +++ b/index/scorch/snapshot_index_str.go @@ -0,0 +1,79 @@ +// Copyright (c) 2024 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "reflect" + + "github.com/blevesearch/bleve/v2/size" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +var reflectStaticSizeIndexSnapshotThesaurusTermReader int + +func init() { + var istr IndexSnapshotThesaurusTermReader + reflectStaticSizeIndexSnapshotThesaurusTermReader = int(reflect.TypeOf(istr).Size()) +} + +type IndexSnapshotThesaurusTermReader struct { + name string + snapshot *IndexSnapshot + thesauri []segment.Thesaurus + postings []segment.SynonymsList + iterators []segment.SynonymsIterator + segmentOffset int +} + +func (i *IndexSnapshotThesaurusTermReader) Size() int { + sizeInBytes := reflectStaticSizeIndexSnapshotThesaurusTermReader + size.SizeOfPtr + + len(i.name) + size.SizeOfString + + for _, postings := range i.postings { + if postings != nil { + sizeInBytes += postings.Size() + } + } + + for _, iterator := range i.iterators { + if iterator != nil { + sizeInBytes += iterator.Size() + } + } + + return sizeInBytes +} + +func (i *IndexSnapshotThesaurusTermReader) Next() (string, error) { + // find the next hit + for i.segmentOffset < len(i.iterators) { + if i.iterators[i.segmentOffset] != nil { + next, err := i.iterators[i.segmentOffset].Next() + if err != nil { + return "", err + } + if next != nil { + synTerm := next.Term() + return synTerm, nil + } + } + i.segmentOffset++ + } + return "", nil +} + +func (i *IndexSnapshotThesaurusTermReader) Close() error { + return nil +} diff --git a/index/scorch/snapshot_index_test.go b/index/scorch/snapshot_index_test.go new file mode 100644 index 0000000..bf38fba --- /dev/null +++ b/index/scorch/snapshot_index_test.go @@ -0,0 +1,90 @@ +package scorch + +import ( + "testing" + + "github.com/blevesearch/vellum" +) + +func TestIndexSnapshot_getLevAutomaton(t *testing.T) { + // Create a dummy IndexSnapshot (parent doesn't matter for this method) + is := &IndexSnapshot{} + + tests := []struct { + name string + term string + fuzziness uint8 + expectError bool + errorMsg string // Optional: check specific error message + }{ + { + name: "fuzziness 1", + term: "test", + fuzziness: 1, + expectError: false, + }, + { + name: "fuzziness 2", + term: "another", + fuzziness: 2, + expectError: false, + }, + { + name: "fuzziness 0", + term: "zero", + fuzziness: 0, + expectError: true, + errorMsg: "fuzziness exceeds the max limit", + }, + { + name: "fuzziness 3", + term: "three", + fuzziness: 3, + expectError: true, + errorMsg: "fuzziness exceeds the max limit", + }, + { + name: "empty term fuzziness 1", + term: "", + fuzziness: 1, + expectError: false, + }, + { + name: "empty term fuzziness 2", + term: "", + fuzziness: 2, + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotAutomaton, err := is.getLevAutomaton(tt.term, tt.fuzziness) + + if tt.expectError { + if err == nil { + t.Errorf("getLevAutomaton() expected an error but got nil") + } else if tt.errorMsg != "" && err.Error() != tt.errorMsg { + t.Errorf("getLevAutomaton() expected error msg %q but got %q", tt.errorMsg, err.Error()) + } + if gotAutomaton != nil { + t.Errorf("getLevAutomaton() expected nil automaton on error but got %v", gotAutomaton) + } + } else { + if err != nil { + t.Errorf("getLevAutomaton() got unexpected error: %v", err) + } + if gotAutomaton == nil { + t.Errorf("getLevAutomaton() expected a valid automaton but got nil") + } + // Optional: Check type if needed, though non-nil is usually sufficient + _, ok := gotAutomaton.(vellum.Automaton) + if !ok { + t.Errorf("getLevAutomaton() returned type is not vellum.Automaton") + } + } + }) + } +} + +// Add other tests for snapshot_index.go below if needed... diff --git a/index/scorch/snapshot_index_tfr.go b/index/scorch/snapshot_index_tfr.go new file mode 100644 index 0000000..8d2ea3a --- /dev/null +++ b/index/scorch/snapshot_index_tfr.go @@ -0,0 +1,239 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "context" + "fmt" + "reflect" + "sync/atomic" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +var reflectStaticSizeIndexSnapshotTermFieldReader int + +func init() { + var istfr IndexSnapshotTermFieldReader + reflectStaticSizeIndexSnapshotTermFieldReader = int(reflect.TypeOf(istfr).Size()) +} + +type IndexSnapshotTermFieldReader struct { + term []byte + field string + snapshot *IndexSnapshot + dicts []segment.TermDictionary + postings []segment.PostingsList + iterators []segment.PostingsIterator + segmentOffset int + includeFreq bool + includeNorm bool + includeTermVectors bool + currPosting segment.Posting + currID index.IndexInternalID + recycle bool + bytesRead uint64 + ctx context.Context + unadorned bool + // flag to indicate whether to increment our bytesRead + // value after creation of the TFR while iterating our postings + // lists + updateBytesRead bool +} + +func (i *IndexSnapshotTermFieldReader) incrementBytesRead(val uint64) { + i.bytesRead += val +} + +func (i *IndexSnapshotTermFieldReader) Size() int { + sizeInBytes := reflectStaticSizeIndexSnapshotTermFieldReader + size.SizeOfPtr + + len(i.term) + + len(i.field) + + len(i.currID) + + for _, entry := range i.postings { + sizeInBytes += entry.Size() + } + + for _, entry := range i.iterators { + sizeInBytes += entry.Size() + } + + if i.currPosting != nil { + sizeInBytes += i.currPosting.Size() + } + + return sizeInBytes +} + +func (i *IndexSnapshotTermFieldReader) Next(preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) { + rv := preAlloced + if rv == nil { + rv = &index.TermFieldDoc{} + } + var prevBytesRead uint64 + // find the next hit + for i.segmentOffset < len(i.iterators) { + // get our current postings iterator + curItr := i.iterators[i.segmentOffset] + if i.updateBytesRead { + prevBytesRead = curItr.BytesRead() + } + next, err := curItr.Next() + if err != nil { + return nil, err + } + if next != nil { + // make segment number into global number by adding offset + globalOffset := i.snapshot.offsets[i.segmentOffset] + nnum := next.Number() + rv.ID = index.NewIndexInternalID(rv.ID, nnum+globalOffset) + i.postingToTermFieldDoc(next, rv) + + i.currID = rv.ID + i.currPosting = next + if i.updateBytesRead { + // postingsIterators maintains the bytesRead stat in a cumulative fashion. + // this is because there are chances of having a series of loadChunk calls, + // and they have to be added together before sending the bytesRead at this point + // upstream. + bytesRead := curItr.BytesRead() + if bytesRead > prevBytesRead { + i.incrementBytesRead(bytesRead - prevBytesRead) + } + } + return rv, nil + } + i.segmentOffset++ + } + return nil, nil +} + +func (i *IndexSnapshotTermFieldReader) postingToTermFieldDoc(next segment.Posting, rv *index.TermFieldDoc) { + if i.includeFreq { + rv.Freq = next.Frequency() + } + if i.includeNorm { + rv.Norm = next.Norm() + } + if i.includeTermVectors { + locs := next.Locations() + if cap(rv.Vectors) < len(locs) { + rv.Vectors = make([]*index.TermFieldVector, len(locs)) + backing := make([]index.TermFieldVector, len(locs)) + for i := range backing { + rv.Vectors[i] = &backing[i] + } + } + rv.Vectors = rv.Vectors[:len(locs)] + for i, loc := range locs { + *rv.Vectors[i] = index.TermFieldVector{ + Start: loc.Start(), + End: loc.End(), + Pos: loc.Pos(), + ArrayPositions: loc.ArrayPositions(), + Field: loc.Field(), + } + } + } +} + +func (i *IndexSnapshotTermFieldReader) Advance(ID index.IndexInternalID, preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) { + // FIXME do something better + // for now, if we need to seek backwards, then restart from the beginning + if i.currPosting != nil && i.currID.Compare(ID) >= 0 { + // Check if the TFR is a special unadorned composite optimization. + // Such a TFR will NOT have a valid `term` or `field` set, making it + // impossible for the TFR to replace itself with a new one. + if !i.unadorned { + i2, err := i.snapshot.TermFieldReader(context.TODO(), i.term, i.field, + i.includeFreq, i.includeNorm, i.includeTermVectors) + if err != nil { + return nil, err + } + // close the current term field reader before replacing it with a new one + _ = i.Close() + *i = *(i2.(*IndexSnapshotTermFieldReader)) + } else { + // unadorned composite optimization + // we need to reset all the iterators + // back to the beginning, which effectively + // achieves the same thing as the above + for _, iter := range i.iterators { + if optimizedIterator, ok := iter.(ResetablePostingsIterator); ok { + optimizedIterator.ResetIterator() + } + } + } + } + num := ID.Value() + segIndex, ldocNum := i.snapshot.segmentIndexAndLocalDocNumFromGlobal(num) + if segIndex >= len(i.snapshot.segment) { + return nil, fmt.Errorf("computed segment index %d out of bounds %d", + segIndex, len(i.snapshot.segment)) + } + // skip directly to the target segment + i.segmentOffset = segIndex + next, err := i.iterators[i.segmentOffset].Advance(ldocNum) + if err != nil { + return nil, err + } + if next == nil { + // we jumped directly to the segment that should have contained it + // but it wasn't there, so reuse Next() which should correctly + // get the next hit after it (we moved i.segmentOffset) + return i.Next(preAlloced) + } + + if preAlloced == nil { + preAlloced = &index.TermFieldDoc{} + } + preAlloced.ID = index.NewIndexInternalID(preAlloced.ID, next.Number()+ + i.snapshot.offsets[segIndex]) + i.postingToTermFieldDoc(next, preAlloced) + i.currID = preAlloced.ID + i.currPosting = next + return preAlloced, nil +} + +func (i *IndexSnapshotTermFieldReader) Count() uint64 { + var rv uint64 + for _, posting := range i.postings { + rv += posting.Count() + } + return rv +} + +func (i *IndexSnapshotTermFieldReader) Close() error { + if i.ctx != nil { + statsCallbackFn := i.ctx.Value(search.SearchIOStatsCallbackKey) + if statsCallbackFn != nil { + // essentially before you close the TFR, you must report this + // reader's bytesRead value + statsCallbackFn.(search.SearchIOStatsCallbackFunc)(i.bytesRead) + } + + search.RecordSearchCost(i.ctx, search.AddM, i.bytesRead) + } + + if i.snapshot != nil { + atomic.AddUint64(&i.snapshot.parent.stats.TotTermSearchersFinished, uint64(1)) + i.snapshot.recycleTermFieldReader(i) + } + return nil +} diff --git a/index/scorch/snapshot_index_thes.go b/index/scorch/snapshot_index_thes.go new file mode 100644 index 0000000..6f3aae8 --- /dev/null +++ b/index/scorch/snapshot_index_thes.go @@ -0,0 +1,107 @@ +// Copyright (c) 2024 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "container/heap" + + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +type segmentThesCursor struct { + thes segment.Thesaurus + itr segment.ThesaurusIterator + curr index.ThesaurusEntry +} + +type IndexSnapshotThesaurusKeys struct { + snapshot *IndexSnapshot + cursors []*segmentThesCursor + entry index.ThesaurusEntry +} + +func (i *IndexSnapshotThesaurusKeys) Len() int { return len(i.cursors) } +func (i *IndexSnapshotThesaurusKeys) Less(a, b int) bool { + return i.cursors[a].curr.Term < i.cursors[b].curr.Term +} +func (i *IndexSnapshotThesaurusKeys) Swap(a, b int) { + i.cursors[a], i.cursors[b] = i.cursors[b], i.cursors[a] +} + +func (i *IndexSnapshotThesaurusKeys) Push(x interface{}) { + i.cursors = append(i.cursors, x.(*segmentThesCursor)) +} + +func (i *IndexSnapshotThesaurusKeys) Pop() interface{} { + n := len(i.cursors) + x := i.cursors[n-1] + i.cursors = i.cursors[0 : n-1] + return x +} + +func (i *IndexSnapshotThesaurusKeys) Next() (*index.ThesaurusEntry, error) { + if len(i.cursors) == 0 { + return nil, nil + } + i.entry = i.cursors[0].curr + next, err := i.cursors[0].itr.Next() + if err != nil { + return nil, err + } + if next == nil { + // at end of this cursor, remove it + heap.Pop(i) + } else { + // modified heap, fix it + i.cursors[0].curr = *next + heap.Fix(i, 0) + } + // look for any other entries with the exact same term + for len(i.cursors) > 0 && i.cursors[0].curr.Term == i.entry.Term { + next, err := i.cursors[0].itr.Next() + if err != nil { + return nil, err + } + if next == nil { + // at end of this cursor, remove it + heap.Pop(i) + } else { + // modified heap, fix it + i.cursors[0].curr = *next + heap.Fix(i, 0) + } + } + + return &i.entry, nil +} + +func (i *IndexSnapshotThesaurusKeys) Close() error { + return nil +} + +func (i *IndexSnapshotThesaurusKeys) Contains(key []byte) (bool, error) { + if len(i.cursors) == 0 { + return false, nil + } + + for _, cursor := range i.cursors { + if found, _ := cursor.thes.Contains(key); found { + return true, nil + } + } + + return false, nil +} diff --git a/index/scorch/snapshot_index_vr.go b/index/scorch/snapshot_index_vr.go new file mode 100644 index 0000000..3ed0d5e --- /dev/null +++ b/index/scorch/snapshot_index_vr.go @@ -0,0 +1,217 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package scorch + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "sort" + "sync/atomic" + + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" + segment_api "github.com/blevesearch/scorch_segment_api/v2" +) + +const VectorSearchSupportedSegmentVersion = 16 + +var reflectStaticSizeIndexSnapshotVectorReader int + +func init() { + var istfr IndexSnapshotVectorReader + reflectStaticSizeIndexSnapshotVectorReader = int(reflect.TypeOf(istfr).Size()) +} + +type IndexSnapshotVectorReader struct { + vector []float32 + field string + k int64 + snapshot *IndexSnapshot + postings []segment_api.VecPostingsList + iterators []segment_api.VecPostingsIterator + segmentOffset int + currPosting segment_api.VecPosting + currID index.IndexInternalID + ctx context.Context + + searchParams json.RawMessage + eligibleSelector index.EligibleDocumentSelector +} + +func (i *IndexSnapshotVectorReader) Size() int { + sizeInBytes := reflectStaticSizeIndexSnapshotVectorReader + size.SizeOfPtr + + len(i.vector)*size.SizeOfFloat32 + + len(i.field) + + len(i.currID) + + for _, entry := range i.postings { + sizeInBytes += entry.Size() + } + + for _, entry := range i.iterators { + sizeInBytes += entry.Size() + } + + if i.currPosting != nil { + sizeInBytes += i.currPosting.Size() + } + + return sizeInBytes +} + +func (i *IndexSnapshotVectorReader) Next(preAlloced *index.VectorDoc) ( + *index.VectorDoc, error) { + rv := preAlloced + if rv == nil { + rv = &index.VectorDoc{} + } + + for i.segmentOffset < len(i.iterators) { + if i.iterators[i.segmentOffset] == nil { + i.segmentOffset++ + continue + } + next, err := i.iterators[i.segmentOffset].Next() + if err != nil { + return nil, err + } + if next != nil { + // make segment number into global number by adding offset + globalOffset := i.snapshot.offsets[i.segmentOffset] + nnum := next.Number() + rv.ID = index.NewIndexInternalID(rv.ID, nnum+globalOffset) + rv.Score = float64(next.Score()) + + i.currID = rv.ID + i.currPosting = next + + return rv, nil + } + i.segmentOffset++ + } + + return nil, nil +} + +func (i *IndexSnapshotVectorReader) Advance(ID index.IndexInternalID, + preAlloced *index.VectorDoc) (*index.VectorDoc, error) { + + if i.currPosting != nil && i.currID.Compare(ID) >= 0 { + i2, err := i.snapshot.VectorReader(i.ctx, i.vector, i.field, i.k, + i.searchParams, i.eligibleSelector) + if err != nil { + return nil, err + } + // close the current term field reader before replacing it with a new one + _ = i.Close() + *i = *(i2.(*IndexSnapshotVectorReader)) + } + + num := ID.Value() + segIndex, ldocNum := i.snapshot.segmentIndexAndLocalDocNumFromGlobal(num) + if segIndex >= len(i.snapshot.segment) { + return nil, fmt.Errorf("computed segment index %d out of bounds %d", + segIndex, len(i.snapshot.segment)) + } + // skip directly to the target segment + i.segmentOffset = segIndex + next, err := i.iterators[i.segmentOffset].Advance(ldocNum) + if err != nil { + return nil, err + } + if next == nil { + // we jumped directly to the segment that should have contained it + // but it wasn't there, so reuse Next() which should correctly + // get the next hit after it (we moved i.segmentOffset) + return i.Next(preAlloced) + } + + if preAlloced == nil { + preAlloced = &index.VectorDoc{} + } + preAlloced.ID = index.NewIndexInternalID(preAlloced.ID, next.Number()+ + i.snapshot.offsets[segIndex]) + i.currID = preAlloced.ID + i.currPosting = next + return preAlloced, nil +} + +func (i *IndexSnapshotVectorReader) Count() uint64 { + var rv uint64 + for _, posting := range i.postings { + rv += posting.Count() + } + return rv +} + +func (i *IndexSnapshotVectorReader) Close() error { + if i.snapshot != nil { + atomic.AddUint64(&i.snapshot.parent.stats.TotKNNSearches, 1) + } + return nil +} + +func (i *IndexSnapshot) CentroidCardinalities(field string, limit int, descending bool) ( + []index.CentroidCardinality, error) { + if len(i.segment) == 0 { + return nil, nil + } + + if limit <= 0 { + return nil, fmt.Errorf("limit must be positive") + } + + centroids := make([]index.CentroidCardinality, 0, limit*len(i.segment)) + + for _, segment := range i.segment { + if sv, ok := segment.segment.(segment_api.VectorSegment); ok { + vecIndex, err := sv.InterpretVectorIndex(field, segment.deleted) + if err != nil { + return nil, fmt.Errorf("failed to interpret vector index for field %s in segment: %v", field, err) + } + + centroidCardinalities, err := vecIndex.ObtainKCentroidCardinalitiesFromIVFIndex(limit, descending) + if err != nil { + return nil, fmt.Errorf("failed to obtain top k centroid cardinalities for field %s in segment: %v", field, err) + } + + if len(centroidCardinalities) > 0 { + centroids = append(centroids, centroidCardinalities...) + } + } + } + + if len(centroids) == 0 { + return nil, nil + } + + sort.Slice(centroids, func(i, j int) bool { + if descending { + return centroids[i].Cardinality > centroids[j].Cardinality + } + return centroids[i].Cardinality < centroids[j].Cardinality + }) + + if limit >= len(centroids) { + return centroids, nil + } + + return centroids[:limit], nil +} diff --git a/index/scorch/snapshot_segment.go b/index/scorch/snapshot_segment.go new file mode 100644 index 0000000..806ca2f --- /dev/null +++ b/index/scorch/snapshot_segment.go @@ -0,0 +1,408 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "bytes" + "os" + "sync" + "sync/atomic" + + "github.com/RoaringBitmap/roaring/v2" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +type SegmentSnapshot struct { + // this flag is needed to identify whether this + // segment was mmaped recently, in which case + // we consider the loading cost of the metadata + // as part of IO stats. + mmaped uint32 + id uint64 + segment segment.Segment + deleted *roaring.Bitmap + creator string + stats *fieldStats + + updatedFields map[string]*index.UpdateFieldInfo + + cachedMeta *cachedMeta + + cachedDocs *cachedDocs + + rootCountOnce sync.Once + rootCount uint64 +} + +func (s *SegmentSnapshot) Segment() segment.Segment { + return s.segment +} + +func (s *SegmentSnapshot) Deleted() *roaring.Bitmap { + return s.deleted +} + +func (s *SegmentSnapshot) Id() uint64 { + return s.id +} + +func (s *SegmentSnapshot) FullSize() int64 { + return int64(s.segment.Count()) +} + +func (s *SegmentSnapshot) LiveSize() int64 { + return int64(s.Count()) +} + +func (s *SegmentSnapshot) HasVector() bool { + // number of vectors, for each vector field in the segment + numVecs := s.stats.Fetch()["num_vectors"] + return len(numVecs) > 0 +} + +func (s *SegmentSnapshot) FileSize() int64 { + ps, ok := s.segment.(segment.PersistedSegment) + if !ok { + return 0 + } + + path := ps.Path() + if path == "" { + return 0 + } + + fi, err := os.Stat(path) + if err != nil { + return 0 + } + + return fi.Size() +} + +func (s *SegmentSnapshot) LiveFileSize() int64 { + fullSize := float64(s.FullSize()) + if fullSize <= 0 { + return 0 + } + + liveSize := float64(s.LiveSize()) + if liveSize <= 0 { + return 0 + } + + fileSize := float64(s.FileSize()) + if fileSize <= 0 { + return 0 + } + + liveRatio := liveSize / fullSize + + return int64(fileSize * liveRatio) +} + +func (s *SegmentSnapshot) Close() error { + return s.segment.Close() +} + +func (s *SegmentSnapshot) VisitDocument(num uint64, visitor segment.StoredFieldValueVisitor) error { + return s.segment.VisitStoredFields(num, visitor) +} + +func (s *SegmentSnapshot) DocID(num uint64) ([]byte, error) { + return s.segment.DocID(num) +} + +func (s *SegmentSnapshot) Count() uint64 { + rv := s.segment.Count() + if s.deleted != nil { + rv -= s.deleted.GetCardinality() + } + return rv +} + +// this counts the root documents in the segment this differs from Count() in that +// Count() counts all live documents including nested children, whereas this method +// counts only root live documents +func (s *SegmentSnapshot) CountRoot() uint64 { + s.rootCountOnce.Do(func() { + if nsb, ok := s.segment.(segment.NestedSegment); ok { + s.rootCount = nsb.CountRoot(s.deleted) + } else { + s.rootCount = s.Count() + } + }) + return s.rootCount +} + +func (s *SegmentSnapshot) DocNumbers(docIDs []string) (*roaring.Bitmap, error) { + rv, err := s.segment.DocNumbers(docIDs) + if err != nil { + return nil, err + } + if s.deleted != nil { + rv.AndNot(s.deleted) + } + return rv, nil +} + +// DocNumbersLive returns a bitmap containing doc numbers for all live docs +func (s *SegmentSnapshot) DocNumbersLive() *roaring.Bitmap { + rv := roaring.NewBitmap() + rv.AddRange(0, s.segment.Count()) + if s.deleted != nil { + rv.AndNot(s.deleted) + } + return rv +} + +func (s *SegmentSnapshot) Fields() []string { + return s.segment.Fields() +} + +func (s *SegmentSnapshot) Size() (rv int) { + rv = s.segment.Size() + if s.deleted != nil { + rv += int(s.deleted.GetSizeInBytes()) + } + rv += s.cachedDocs.Size() + return +} + +// Merge given updated field information with existing and pass it on to the segment base +func (s *SegmentSnapshot) UpdateFieldsInfo(updatedFields map[string]*index.UpdateFieldInfo) { + if s.updatedFields == nil { + s.updatedFields = updatedFields + } else { + for fieldName, info := range updatedFields { + if val, ok := s.updatedFields[fieldName]; ok { + val.Deleted = val.Deleted || info.Deleted + val.Index = val.Index || info.Index + val.DocValues = val.DocValues || info.DocValues + val.Store = val.Store || info.Store + } else { + s.updatedFields[fieldName] = info + } + } + } + + if segment, ok := s.segment.(segment.UpdatableSegment); ok { + segment.SetUpdatedFields(s.updatedFields) + } +} + +type cachedFieldDocs struct { + m sync.Mutex + readyCh chan struct{} // closed when the cachedFieldDocs.docs is ready to be used. + err error // Non-nil if there was an error when preparing this cachedFieldDocs. + docs map[uint64][]byte // Keyed by localDocNum, value is a list of terms delimited by 0xFF. + size uint64 +} + +func (cfd *cachedFieldDocs) Size() int { + var rv int + cfd.m.Lock() + for _, entry := range cfd.docs { + rv += 8 /* size of uint64 */ + len(entry) + } + cfd.m.Unlock() + return rv +} + +func (cfd *cachedFieldDocs) prepareField(field string, ss *SegmentSnapshot) { + cfd.m.Lock() + defer func() { + close(cfd.readyCh) + cfd.m.Unlock() + }() + + cfd.size += uint64(size.SizeOfUint64) /* size field */ + dict, err := ss.segment.Dictionary(field) + if err != nil { + cfd.err = err + return + } + + var postings segment.PostingsList + var postingsItr segment.PostingsIterator + + dictItr := dict.AutomatonIterator(nil, nil, nil) + next, err := dictItr.Next() + for err == nil && next != nil { + var err1 error + postings, err1 = dict.PostingsList([]byte(next.Term), nil, postings) + if err1 != nil { + cfd.err = err1 + return + } + + cfd.size += uint64(size.SizeOfUint64) /* map key */ + postingsItr = postings.Iterator(false, false, false, postingsItr) + nextPosting, err2 := postingsItr.Next() + for err2 == nil && nextPosting != nil { + docNum := nextPosting.Number() + cfd.docs[docNum] = append(cfd.docs[docNum], []byte(next.Term)...) + cfd.docs[docNum] = append(cfd.docs[docNum], index.DocValueTermSeparator) + cfd.size += uint64(len(next.Term) + 1) // map value + nextPosting, err2 = postingsItr.Next() + } + + if err2 != nil { + cfd.err = err2 + return + } + + next, err = dictItr.Next() + } + + if err != nil { + cfd.err = err + return + } +} + +type cachedDocs struct { + size uint64 + m sync.RWMutex // As the cache is asynchronously prepared, need a lock + cache map[string]*cachedFieldDocs // Keyed by field +} + +func (c *cachedDocs) prepareFields(wantedFields []string, ss *SegmentSnapshot) error { + c.m.Lock() + + if c.cache == nil { + c.cache = make(map[string]*cachedFieldDocs, len(ss.Fields())) + } + + for _, field := range wantedFields { + _, exists := c.cache[field] + if !exists { + c.cache[field] = &cachedFieldDocs{ + readyCh: make(chan struct{}), + docs: make(map[uint64][]byte), + } + + go c.cache[field].prepareField(field, ss) + } + } + + for _, field := range wantedFields { + cachedFieldDocs := c.cache[field] + c.m.Unlock() + <-cachedFieldDocs.readyCh + + if cachedFieldDocs.err != nil { + return cachedFieldDocs.err + } + c.m.Lock() + } + + c.updateSizeLOCKED() + + c.m.Unlock() + return nil +} + +// hasFields returns true if the cache has all the given fields +func (c *cachedDocs) hasFields(fields []string) bool { + c.m.RLock() + for _, field := range fields { + if _, exists := c.cache[field]; !exists { + c.m.RUnlock() + return false // found a field not in cache + } + } + c.m.RUnlock() + return true +} + +func (c *cachedDocs) Size() int { + return int(atomic.LoadUint64(&c.size)) +} + +func (c *cachedDocs) updateSizeLOCKED() { + sizeInBytes := 0 + for k, v := range c.cache { // cachedFieldDocs + sizeInBytes += len(k) + if v != nil { + sizeInBytes += v.Size() + } + } + atomic.StoreUint64(&c.size, uint64(sizeInBytes)) +} + +func (c *cachedDocs) visitDoc(localDocNum uint64, + fields []string, visitor index.DocValueVisitor) { + c.m.RLock() + + for _, field := range fields { + if cachedFieldDocs, exists := c.cache[field]; exists { + c.m.RUnlock() + <-cachedFieldDocs.readyCh + c.m.RLock() + + if tlist, exists := cachedFieldDocs.docs[localDocNum]; exists { + for { + i := bytes.IndexByte(tlist, index.DocValueTermSeparator) + if i < 0 { + break + } + visitor(field, tlist[0:i]) + tlist = tlist[i+1:] + } + } + } + } + + c.m.RUnlock() +} + +// cachedMeta is a simple wrapper around sync.Map to provide typed +// access to cached metadata values for segments. +type cachedMeta struct { + meta sync.Map +} + +func newCachedMeta() *cachedMeta { + return &cachedMeta{ + meta: sync.Map{}, + } +} + +// store the value for a field in the cache, overwriting any existing value. +func (c *cachedMeta) store(field string, val interface{}) { + c.meta.Store(field, val) +} + +// load the value for a field from the cache, returning the value +// and a boolean indicating whether the value was present. +func (c *cachedMeta) load(field string) (rv interface{}, ok bool) { + return c.meta.Load(field) +} + +// contains reports whether the cache has an entry for the given field. +func (c *cachedMeta) contains(field string) bool { + _, ok := c.meta.Load(field) + return ok +} + +func (s *SegmentSnapshot) Ancestors(docNum uint64, prealloc []index.AncestorID) []index.AncestorID { + nsb, ok := s.segment.(segment.NestedSegment) + if !ok { + return append(prealloc, index.NewAncestorID(docNum)) + } + return nsb.Ancestors(docNum, prealloc) +} diff --git a/index/scorch/snapshot_vector_index.go b/index/scorch/snapshot_vector_index.go new file mode 100644 index 0000000..1d0c03b --- /dev/null +++ b/index/scorch/snapshot_vector_index.go @@ -0,0 +1,152 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package scorch + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/bits-and-blooms/bitset" + index "github.com/blevesearch/bleve_index_api" + segment_api "github.com/blevesearch/scorch_segment_api/v2" +) + +func (is *IndexSnapshot) VectorReader(ctx context.Context, vector []float32, + field string, k int64, searchParams json.RawMessage, + eligibleSelector index.EligibleDocumentSelector) ( + index.VectorReader, error) { + rv := &IndexSnapshotVectorReader{ + vector: vector, + field: field, + k: k, + snapshot: is, + searchParams: searchParams, + eligibleSelector: eligibleSelector, + postings: make([]segment_api.VecPostingsList, len(is.segment)), + iterators: make([]segment_api.VecPostingsIterator, len(is.segment)), + } + + // initialize postings and iterators within the OptimizeVR's Finish() + return rv, nil +} + +// eligibleDocumentList represents the list of eligible documents within a segment. +type eligibleDocumentList struct { + bs *bitset.BitSet +} + +// Iterator returns an iterator for the eligible document IDs. +func (edl *eligibleDocumentList) Iterator() index.EligibleDocumentIterator { + if edl.bs == nil { + // no eligible documents + return emptyEligibleIterator + } + // return the iterator + return &eligibleDocumentIterator{ + bs: edl.bs, + } +} + +// Count returns the number of eligible document IDs. +func (edl *eligibleDocumentList) Count() uint64 { + if edl.bs == nil { + return 0 + } + return uint64(edl.bs.Count()) +} + +// emptyEligibleDocumentList is a reusable empty eligible document list. +var emptyEligibleDocumentList = &eligibleDocumentList{} + +// eligibleDocumentIterator iterates over eligible document IDs within a segment. +type eligibleDocumentIterator struct { + bs *bitset.BitSet + current uint +} + +// Next returns the next eligible document ID and whether it exists. +func (it *eligibleDocumentIterator) Next() (id uint64, ok bool) { + next, found := it.bs.NextSet(it.current) + if !found { + return 0, false + } + it.current = next + 1 + return uint64(next), true +} + +// emptyEligibleIterator is a reusable empty eligible document iterator. +var emptyEligibleIterator = &emptyEligibleDocumentIterator{} + +// emptyEligibleDocumentIterator is an iterator that always returns no documents. +type emptyEligibleDocumentIterator struct{} + +// Next always returns false for empty iterator. +func (it *emptyEligibleDocumentIterator) Next() (id uint64, ok bool) { + return 0, false +} + +// eligibleDocumentSelector is used to filter out documents that are eligible for +// the KNN search from a pre-filter query. +type eligibleDocumentSelector struct { + // segment ID -> segment local doc nums in a bitset + eligibleDocNums []*bitset.BitSet + is *IndexSnapshot +} + +// SegmentEligibleDocuments returns an EligibleDocumentList for the specified segment ID. +func (eds *eligibleDocumentSelector) SegmentEligibleDocuments(segmentID int) index.EligibleDocumentList { + if eds.eligibleDocNums == nil || segmentID < 0 || segmentID >= len(eds.eligibleDocNums) { + return emptyEligibleDocumentList + } + bs := eds.eligibleDocNums[segmentID] + if bs == nil { + // no eligible documents for this segment + return emptyEligibleDocumentList + } + return &eligibleDocumentList{ + bs: bs, + } +} + +// AddEligibleDocumentMatch adds a document match to the list of eligible documents. +func (eds *eligibleDocumentSelector) AddEligibleDocumentMatch(id index.IndexInternalID) error { + if eds.is == nil { + return fmt.Errorf("eligibleDocumentSelector is not initialized with IndexSnapshot") + } + // Get the segment number and the local doc number for this document. + segIdx, docNum, err := eds.is.segmentIndexAndLocalDocNum(id) + if err != nil { + return err + } + // allocate a bitset for this segment if needed + if eds.eligibleDocNums[segIdx] == nil { + // the size of the bitset is the full size of the segment (which is the max local doc num + 1) + eds.eligibleDocNums[segIdx] = bitset.New(uint(eds.is.segment[segIdx].FullSize())) + } + // Add the local doc number to the list of eligible doc numbers for this segment. + eds.eligibleDocNums[segIdx].Set(uint(docNum)) + return nil +} + +func (is *IndexSnapshot) NewEligibleDocumentSelector() index.EligibleDocumentSelector { + return &eligibleDocumentSelector{ + eligibleDocNums: make([]*bitset.BitSet, len(is.segment)), + is: is, + } +} diff --git a/index/scorch/stats.go b/index/scorch/stats.go new file mode 100644 index 0000000..20bda3f --- /dev/null +++ b/index/scorch/stats.go @@ -0,0 +1,172 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "reflect" + "sync/atomic" + + "github.com/blevesearch/bleve/v2/util" +) + +// Stats tracks statistics about the index, fields that are +// prefixed like CurXxxx are gauges (can go up and down), +// and fields that are prefixed like TotXxxx are monotonically +// increasing counters. +type Stats struct { + TotUpdates uint64 + TotDeletes uint64 + + TotBatches uint64 + TotBatchesEmpty uint64 + TotBatchIntroTime uint64 + MaxBatchIntroTime uint64 + + CurRootEpoch uint64 + LastPersistedEpoch uint64 + LastMergedEpoch uint64 + + TotOnErrors uint64 + + TotAnalysisTime uint64 + TotIndexTime uint64 + + TotIndexedPlainTextBytes uint64 + + TotBytesReadAtQueryTime uint64 + TotBytesWrittenAtIndexTime uint64 + + TotTermSearchersStarted uint64 + TotTermSearchersFinished uint64 + + TotKNNSearches uint64 + TotSynonymSearches uint64 + + TotEventTriggerStarted uint64 + TotEventTriggerCompleted uint64 + + TotIntroduceLoop uint64 + TotIntroduceSegmentBeg uint64 + TotIntroduceSegmentEnd uint64 + TotIntroducePersistBeg uint64 + TotIntroducePersistEnd uint64 + TotIntroduceMergeBeg uint64 + TotIntroduceMergeEnd uint64 + + TotIntroducedItems uint64 + TotIntroducedSegmentsBatch uint64 + TotIntroducedSegmentsMerge uint64 + + TotPersistLoopBeg uint64 + TotPersistLoopErr uint64 + TotPersistLoopProgress uint64 + TotPersistLoopWait uint64 + TotPersistLoopWaitNotified uint64 + TotPersistLoopEnd uint64 + + TotPersistedItems uint64 + TotItemsToPersist uint64 + TotPersistedSegments uint64 + TotMutationsFiltered uint64 + + TotPersisterSlowMergerPause uint64 + TotPersisterSlowMergerResume uint64 + + TotPersisterNapPauseCompleted uint64 + TotPersisterMergerNapBreak uint64 + + TotFileMergeLoopBeg uint64 + TotFileMergeLoopErr uint64 + TotFileMergeLoopEnd uint64 + + TotFileMergeForceOpsStarted uint64 + TotFileMergeForceOpsCompleted uint64 + + TotFileMergePlan uint64 + TotFileMergePlanErr uint64 + TotFileMergePlanNone uint64 + TotFileMergePlanOk uint64 + + TotFileMergePlanTasks uint64 + TotFileMergePlanTasksDone uint64 + TotFileMergePlanTasksErr uint64 + TotFileMergePlanTasksSegments uint64 + TotFileMergePlanTasksSegmentsEmpty uint64 + + TotFileMergeSegmentsEmpty uint64 + TotFileMergeSegments uint64 + TotFileSegmentsAtRoot uint64 + TotFileMergeWrittenBytes uint64 + + TotFileMergeZapBeg uint64 + TotFileMergeZapEnd uint64 + TotFileMergeZapTime uint64 + MaxFileMergeZapTime uint64 + TotFileMergeZapIntroductionTime uint64 + MaxFileMergeZapIntroductionTime uint64 + + TotFileMergeIntroductions uint64 + TotFileMergeIntroductionsDone uint64 + TotFileMergeIntroductionsSkipped uint64 + TotFileMergeIntroductionsObsoleted uint64 + + CurFilesIneligibleForRemoval uint64 + TotSnapshotsRemovedFromMetaStore uint64 + + TotMemMergeBeg uint64 + TotMemMergeErr uint64 + TotMemMergeDone uint64 + TotMemMergeSegments uint64 + + TotMemMergeZapBeg uint64 + TotMemMergeZapEnd uint64 + TotMemMergeZapTime uint64 + MaxMemMergeZapTime uint64 + TotMemMergeZapIntroductionTime uint64 + MaxMemMergeZapIntroductionTime uint64 + + TotMemMergeIntroductions uint64 + TotMemMergeIntroductionsDone uint64 + TotMemMergeIntroductionsSkipped uint64 + TotMemMergeIntroductionsObsoleted uint64 + + TotMemorySegmentsAtRoot uint64 + + TotTrainedSamples uint64 + TotTrainTime uint64 + TotTrainFireIndexEventTime uint64 + TotTrainFireIndexEvents uint64 +} + +// atomically populates the returned map +func (s *Stats) ToMap() map[string]interface{} { + m := map[string]interface{}{} + sve := reflect.ValueOf(s).Elem() + svet := sve.Type() + for i := 0; i < svet.NumField(); i++ { + svef := sve.Field(i) + if svef.CanAddr() { + svefp := svef.Addr().Interface() + m[svet.Field(i).Name] = atomic.LoadUint64(svefp.(*uint64)) + } + } + return m +} + +// MarshalJSON implements json.Marshaler, and in contrast to standard +// json marshaling provides atomic safety +func (s *Stats) MarshalJSON() ([]byte, error) { + return util.MarshalJSON(s.ToMap()) +} diff --git a/index/scorch/train_noop.go b/index/scorch/train_noop.go new file mode 100644 index 0000000..0a887a1 --- /dev/null +++ b/index/scorch/train_noop.go @@ -0,0 +1,63 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !vectors +// +build !vectors + +package scorch + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +func initTrainer(s *Scorch, config map[string]interface{}) *noopTrainer { + return nil +} + +type noopTrainer struct { +} + +func (t *noopTrainer) trainLoop() {} + +func (t *noopTrainer) train(batch *index.Batch) error { + return fmt.Errorf("training is not supported with this build") +} + +func (t *noopTrainer) loadTrainedData(bucket *util.BoltBucketImpl) error { + // noop + return nil +} + +func (t *noopTrainer) getInternal(key []byte) ([]byte, error) { + return nil, nil +} + +func (t *noopTrainer) copyFileLOCKED(file string, d index.IndexDirectory) error { + return nil +} + +func (t *noopTrainer) updateBolt(snapshotsBucket *util.BoltBucketImpl, key []byte, value []byte) error { + return nil +} + +func (t *noopTrainer) dropFileWriterIDs(ids map[string]struct{}) error { + return nil +} + +func (t *noopTrainer) fileWriterIDsInUse() (map[string]struct{}, error) { + return nil, nil +} diff --git a/index/scorch/train_vector.go b/index/scorch/train_vector.go new file mode 100644 index 0000000..c717eab --- /dev/null +++ b/index/scorch/train_vector.go @@ -0,0 +1,626 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package scorch + +import ( + "encoding/binary" + "fmt" + "maps" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/RoaringBitmap/roaring/v2" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +type trainRequest struct { + finalSample bool + sampleSize int + ackCh chan error + sample segment.Segment + trainingParams *index.TrainingParams + removedFileWriterIDs map[string]struct{} +} + +type vectorTrainer struct { + trainingComplete atomic.Bool + trainedSamples uint64 + parent *Scorch + config map[string]interface{} + + m sync.RWMutex + // not a searchable segment in the sense that it won't return + // the data vectors, returns trained centroid layout + trainedIndex *SegmentSnapshot + trainCh chan *trainRequest + // closed by trainLoop when it exits (via close, training completion, or + // error). Producers select on it so they never block on a request that the + // trainer will never consume/ack. + doneCh chan struct{} +} + +const IndexTrainedWithFastMerge = "vector_index_fast_merge" + +func initTrainer(s *Scorch, config map[string]interface{}) *vectorTrainer { + if f, ok := config[IndexTrainedWithFastMerge]; ok { + feature, ok := f.(bool) + if ok && feature { + trainer := vectorTrainer{ + parent: s, + config: maps.Clone(s.segmentConfig), + trainCh: make(chan *trainRequest, 1), + doneCh: make(chan struct{}), + } + // update the parent scorch config with the trainer's callback to fetch the trained index + s.segmentConfig[index.TrainedIndexCallback] = index.TrainedIndexCallbackFn(trainer.getTrainedIndex) + return &trainer + } + } + return nil +} + +func moveFile(sourcePath, destPath string) error { + // rename is supposed to be atomic on the same filesystem + err := os.Rename(sourcePath, destPath) + if err != nil { + return fmt.Errorf("error renaming file: %v", err) + } + return nil +} + +func (t *vectorTrainer) persistToBolt(trainReq *trainRequest) error { + tx, err := t.parent.rootBolt.Begin(true) + if err != nil { + return fmt.Errorf("error starting bolt transaction: %v", err) + } + defer tx.Rollback() + + snapshotsBucket, err := tx.CreateBucketIfNotExists(util.BoltSnapshotsBucket) + if err != nil { + return fmt.Errorf("error creating snapshots bucket: %v", err) + } + + trainerBucket, err := snapshotsBucket.CreateBucketIfNotExists(util.BoltTrainerKey) + if err != nil { + return fmt.Errorf("error creating trained index bucket: %v", err) + } + + segmentPath := t.parent.path + string(os.PathSeparator) + index.TrainedIndexFileName + if _, err := os.Stat(segmentPath); !os.IsNotExist(err) { + err = trainerBucket.Put(util.BoltPathKey, []byte(index.TrainedIndexFileName), nil) + if err != nil { + return fmt.Errorf("error updating trained index bucket: %v", err) + } + } + + t.trainingComplete.Store(trainReq.finalSample) + err = trainerBucket.Put(util.BoltTrainCompleteKey, []byte(strconv.FormatBool(trainReq.finalSample)), nil) + if err != nil { + return fmt.Errorf("error updating train complete key: %v", err) + } + + totSamples := atomic.AddUint64(&t.trainedSamples, uint64(trainReq.sampleSize)) + err = trainerBucket.Put(util.BoltTrainedSamplesKey, binary.LittleEndian.AppendUint64(nil, totSamples), nil) + if err != nil { + return fmt.Errorf("error updating trained samples key: %v", err) + } + + err = tx.Commit() + if err != nil { + return fmt.Errorf("error committing bolt transaction: %v", err) + } + + return t.parent.rootBolt.Sync() +} + +// trainLoop is not a routine that runs throughout the lifetime of the index. +// Its purpose is only to train the vector index before data ingestion starts. +// It is the sole consumer of trainCh and the sole owner of trainedIndex +// mutations originating from training. +func (t *vectorTrainer) trainLoop() { + defer t.parent.asyncTasks.Done() + // signal any blocked producers that no further requests will be serviced. + defer close(t.doneCh) + + start := time.Now() + path := filepath.Join(t.parent.path, index.TrainedIndexFileName) + // config is owned exclusively by this goroutine. t.config is read + // concurrently by train(), so it must not be mutated here. + config := maps.Clone(t.config) + + for { + // exit once the final sample set has been ingested and training is complete. + if t.trainingComplete.Load() { + t.recordTrainingStats(start) + return + } + select { + case <-t.parent.closeCh: + // the deferred close(doneCh) releases any producer blocked on + // trainCh or its ackCh, so there is no need to drain here. + return + case req := <-t.trainCh: + shutdown, err := t.handleRequest(req, path, config) + ackRequest(req, err) + if shutdown { + return + } + } + } +} + +// recordTrainingStats publishes the final training counters once the loop exits. +func (t *vectorTrainer) recordTrainingStats(start time.Time) { + atomic.StoreUint64(&t.parent.stats.TotTrainedSamples, atomic.LoadUint64(&t.trainedSamples)) + atomic.StoreUint64(&t.parent.stats.TotTrainTime, uint64(time.Since(start).Milliseconds())) +} + +// ackRequest delivers a request's result to the producer waiting in submit and +// then releases it. A non-nil err is sent before the close so submit observes +// it; the send blocks until submit receives, which the shutdown handshake in +// submit relies on. +func ackRequest(req *trainRequest, err error) { + if err != nil { + req.ackCh <- err + } + close(req.ackCh) +} + +// handleRequest processes a single training request. The returned err, if +// non-nil, is already formatted for the producer. shutdown reports whether the +// trainLoop must exit: every failure is fatal except a failed initial persist, +// which leaves the trainer able to retry on the next sample. +func (t *vectorTrainer) handleRequest(req *trainRequest, path string, config map[string]interface{}) (shutdown bool, err error) { + if req.trainingParams != nil { + // mutate the goroutine-local config only; t.config is read concurrently + // by train() and must stay immutable after init. + config[index.TrainingKey] = req.trainingParams + } + + // remove any file writer ids that are no longer in use + if req.removedFileWriterIDs != nil { + if err := t.removeFileWriterIDs(req.removedFileWriterIDs); err != nil { + return true, fmt.Errorf("error removing file writer ids: %v", err) + } + return false, nil + } + + // no sample segment: just persist state if this is the final sample. + if req.sample == nil { + if req.finalSample { + if err := t.persistToBolt(req); err != nil { + return true, fmt.Errorf("error persisting to bolt: %v", err) + } + } + return false, nil + } + + // snapshot the current trained segment under the read lock; MergeUsing below + // only reads it, so concurrent getTrainedIndex readers stay safe. + t.m.RLock() + prev := t.trainedIndex + t.m.RUnlock() + + if prev == nil { + if seg, ok := req.sample.(segment.UnpersistedSegment); ok { + if err := persistToDirectory(seg, nil, path); err != nil { + // recoverable: retry on the next sample rather than tear down. + return false, fmt.Errorf("error persisting segment: %v", err) + } + } + } else { + // merge the new sample with the existing index into a .tmp file; it is + // renamed into place by publishTrainedIndex (Os.Open on the live path is + // unsafe during the merge). + _, _, mErr := t.parent.segPlugin.MergeUsing([]segment.Segment{prev.segment, req.sample}, + []*roaring.Bitmap{nil, nil}, path+".tmp", t.parent.closeCh, nil, config) + if mErr != nil { + return true, fmt.Errorf("error merging trained index: %v", mErr) + } + } + + if err := t.publishTrainedIndex(req, prev, path, config); err != nil { + return true, err + } + return false, nil +} + +// publishTrainedIndex makes the freshly written index visible to readers. The +// old segment handle must be closed before moveFile can replace the backing +// file, so the close -> rename -> open -> pointer-swap sequence runs under the +// write lock; otherwise a reader holding RLock could observe a closed segment +// or a missing file. On any error past the close, the pointer is reset to nil +// so readers never see the freed prev segment (callers fall back to naive +// merge). The bolt write acts as a failover-recovery checkpoint: callers +// downstream can rely on the trained index being available once this completes. +// todo: rethink the frequency of bolt writes +func (t *vectorTrainer) publishTrainedIndex(req *trainRequest, prev *SegmentSnapshot, path string, config map[string]interface{}) error { + t.m.Lock() + defer t.m.Unlock() + + if prev != nil { + if err := prev.segment.Close(); err != nil { + t.trainedIndex = nil + return fmt.Errorf("error closing previous trained index: %v", err) + } + if err := moveFile(path+".tmp", path); err != nil { + t.trainedIndex = nil + return fmt.Errorf("error renaming trained index: %v", err) + } + } + + if err := t.persistToBolt(req); err != nil { + if prev != nil { + t.trainedIndex = nil + } + return fmt.Errorf("error persisting to bolt: %v", err) + } + + trainedIndex, err := t.parent.segPlugin.OpenUsing(path, config) + if err != nil { + t.trainedIndex = nil + return fmt.Errorf("error opening trained index: %v", err) + } + t.trainedIndex = &SegmentSnapshot{segment: trainedIndex} + return nil +} + +// loadTrainedData loads the trained-index metadata from boltdb during init. The +// trainedIndex write is guarded because callback-driven readers may already be +// registered by the time this runs. +func (t *vectorTrainer) loadTrainedData(bucket *util.BoltBucketImpl) error { + if bucket == nil { + return nil + } + + reader, err := util.NewFileReader("", nil) + if err != nil { + return fmt.Errorf("error creating file reader: %v", err) + } + + // segmentSnapshot is constructed only if the trained index file path exists. If + // application layer fails to create one and the underlying path doesn't exist, we'll + // just skip loading the trained index and fallback to naive merge + var segmentSnapshot *SegmentSnapshot + pathBytes, err := bucket.Get(util.BoltPathKey, nil) + if pathBytes != nil { + segmentSnapshot, err = t.parent.loadSegment(bucket, reader) + if err != nil { + return err + } + } + + // get the training status out of bolt + trainComplete, err := bucket.Get(util.BoltTrainCompleteKey, nil) + if err != nil { + return fmt.Errorf("error getting train complete: %v", err) + } + trainedSamples, err := bucket.Get(util.BoltTrainedSamplesKey, nil) + if err != nil { + return fmt.Errorf("error getting trained samples: %v", err) + } + if len(trainedSamples) == 8 { + atomic.StoreUint64(&t.trainedSamples, binary.LittleEndian.Uint64(trainedSamples)) + } + if len(trainComplete) > 0 { + comp, err := strconv.ParseBool(string(trainComplete)) + if err != nil { + return fmt.Errorf("error parsing train complete: %v", err) + } + t.trainingComplete.Store(comp) + } + + t.m.Lock() + defer t.m.Unlock() + t.trainedIndex = segmentSnapshot + return nil +} + +func (t *vectorTrainer) train(batch *index.Batch) error { + // regulate the Train function + start := time.Now() + t.parent.FireIndexEvent() + atomic.AddUint64(&t.parent.stats.TotTrainFireIndexEventTime, uint64(time.Since(start))) + atomic.AddUint64(&t.parent.stats.TotTrainFireIndexEvents, 1) + if t.trainingComplete.Load() { + return fmt.Errorf("training is already complete, cannot accept more training data") + } + + var trainData []index.Document + for _, doc := range batch.IndexOps { + if doc != nil { + // insert _id field + // no need to track updates/deletes over here since + // the API is singleton + doc.AddIDField() + } + trainData = append(trainData, doc) + } + + trainComplete := batch.InternalOps[string(util.BoltTrainCompleteKey)] + if trainComplete == nil { + trainComplete = []byte("false") + } + fin, err := strconv.ParseBool(string(trainComplete)) + if err != nil { + return fmt.Errorf("error parsing train complete: %v", err) + } + + trainReq := &trainRequest{ + finalSample: fin, + sampleSize: len(trainData), + ackCh: make(chan error), + } + // setting the training params using the internal value before the actual + // training has started + config := t.config + if atomic.LoadUint64(&t.trainedSamples) == 0 { + trainingParamsBytes := batch.InternalOps[index.TrainingKey] + var trainingParams index.TrainingParams + if trainingParamsBytes != nil { + err = util.UnmarshalJSON(trainingParamsBytes, &trainingParams) + if err != nil { + return fmt.Errorf("error parsing training params: %v", err) + } + trainReq.trainingParams = &trainingParams + config = maps.Clone(t.config) + config[index.TrainingKey] = &trainingParams + } + } + + // just builds a new vector index out of the train data provided + // this is not necessarily the final train data since this is submitted + // as a request to the trainer component to be merged. once the training + // is complete, the template will be used for other operations down the line + // like merge and search. + // + // todo: updates/deletes -> data drift detection + if len(trainData) > 0 { + trainReq.sample, _, err = t.parent.segPlugin.NewUsing(trainData, config) + if err != nil { + return err + } + } + + if err := t.submit(trainReq); err != nil { + return fmt.Errorf("train_vector: train() err'd out with: %w", err) + } + + return nil +} + +// submit hands a request to the trainLoop and waits for its ack. It never +// blocks past trainer shutdown: if the trainLoop has already exited (via close, +// training completion, or error) the doneCh releases the caller instead of +// letting it hang on a request that will never be consumed or acked. +func (t *vectorTrainer) submit(req *trainRequest) error { + select { + case t.trainCh <- req: + case <-t.doneCh: + return fmt.Errorf("trainer is closed") + } + + select { + case err := <-req.ackCh: + return err + case <-t.doneCh: + // trainLoop acks every request it consumes before returning, so if our + // request was processed the ack is already available — prefer it over + // reporting a shutdown that didn't actually drop this request. + select { + case err := <-req.ackCh: + return err + default: + return fmt.Errorf("trainer is closed") + } + } +} + +func (t *vectorTrainer) getInternal(key []byte) ([]byte, error) { + switch string(key) { + case string(util.BoltTrainCompleteKey): + return []byte(strconv.FormatBool(t.trainingComplete.Load())), nil + } + return nil, nil +} + +func (t *vectorTrainer) getTrainedIndex(field string) (interface{}, error) { + // return the coarse quantizer of the trained faiss index belonging to the field + // if its not available then zap performs naive merge + t.m.RLock() + defer t.m.RUnlock() + if t.trainedIndex != nil { + trainedSegment, ok := t.trainedIndex.segment.(segment.TrainedSegment) + if !ok { + return nil, fmt.Errorf("segment is not a trained index segment") + } + + coarseQuantizer, err := trainedSegment.GetCoarseQuantizer(field) + if err != nil { + return nil, err + } + return coarseQuantizer, nil + } + return nil, nil +} + +func (t *vectorTrainer) copyFileLOCKED(file string, d index.IndexDirectory) error { + if strings.HasSuffix(file, index.TrainedIndexFileName) { + // trained index file - this is outside the snapshots domain so the bolt update is different + err := d.SetPathInBolt(util.BoltTrainerKey, []byte(file)) + if err != nil { + return fmt.Errorf("error updating dest index bolt: %w", err) + } + + // update the stats in destination + err = d.SetPathInBolt(util.BoltTrainedSamplesKey, binary.LittleEndian.AppendUint64(nil, atomic.LoadUint64(&t.trainedSamples))) + if err != nil { + return fmt.Errorf("error updating trained samples key: %w", err) + } + } + + return nil +} + +func (t *vectorTrainer) updateBolt(snapshotsBucket *util.BoltBucketImpl, key []byte, value []byte) error { + switch string(key) { + case string(util.BoltTrainerKey): + trainerBucket, err := snapshotsBucket.CreateBucketIfNotExists(util.BoltTrainerKey) + if err != nil { + return err + } + if trainerBucket == nil { + return fmt.Errorf("trainer bucket not found") + } + + // guard against duplicate updates + existingValue, err := trainerBucket.Get(util.BoltPathKey, nil) + if err != nil { + return fmt.Errorf("error checking existing value: %v", err) + } + if existingValue != nil { + return fmt.Errorf("key already exists %v %v", t.parent.path, string(existingValue)) + } + + err = trainerBucket.Put(util.BoltPathKey, value, nil) + if err != nil { + return err + } + + reader, err := util.NewFileReader("", nil) + if err != nil { + return fmt.Errorf("error creating file reader: %v", err) + } + + // update the centroid index pointer under the write lock; getTrainedIndex + // and trainLoop may read t.trainedIndex concurrently. + seg, err := t.parent.loadSegment(trainerBucket, reader) + if err != nil { + return err + } + t.m.Lock() + t.trainedIndex = seg + t.m.Unlock() + + case string(util.BoltTrainedSamplesKey): + trainedSamples := binary.LittleEndian.Uint64(value) + atomic.StoreUint64(&t.trainedSamples, trainedSamples) + trainerBucket, err := snapshotsBucket.CreateBucketIfNotExists(util.BoltTrainerKey) + if err != nil { + return err + } + err = trainerBucket.Put(util.BoltTrainedSamplesKey, value, nil) + if err != nil { + return err + } + } + + return nil +} + +// dropFileWriterIDs removes the given file writer ids from the trained index. +// This is a lifecycle operation (key rotation) that must work for the entire +// life of the trained index, not just while training is running. +func (t *vectorTrainer) dropFileWriterIDs(ids map[string]struct{}) error { + if t.trainingComplete.Load() { + if err := t.removeFileWriterIDs(ids); err != nil { + return fmt.Errorf("train_vector: dropFileWriterIDs() err'd out with: %w", err) + } + return nil + } + + trainReq := &trainRequest{ + ackCh: make(chan error), + removedFileWriterIDs: ids, + } + if err := t.submit(trainReq); err != nil { + return fmt.Errorf("train_vector: dropFileWriterIDs() err'd out with: %w", err) + } + + return nil +} + +func (t *vectorTrainer) removeFileWriterIDs(ids map[string]struct{}) error { + // trainer bucket in the bolt file doesn't have any sensitive data to be encrypted, + // so we don't have to update it with new writer ids. + // invoke the merge API and let the zap side of things handle the update of writer IDs and also + // the data in the trained index. + t.m.Lock() + defer t.m.Unlock() + trainedIndexPath := filepath.Join(t.parent.path, index.TrainedIndexFileName) + if t.trainedIndex != nil { + if encryptedIndex, ok := t.trainedIndex.segment.(segment.SegmentWithCallbacks); ok { + if _, ok := ids[encryptedIndex.CallbackId()]; ok { + _, _, err := t.parent.segPlugin.MergeUsing([]segment.Segment{t.trainedIndex.segment}, + []*roaring.Bitmap{nil}, trainedIndexPath+"temp", t.parent.closeCh, nil, t.config) + if err != nil { + if serr := os.Remove(trainedIndexPath + "temp"); serr != nil { + return fmt.Errorf("error removing temp trained index: %v", serr) + } + return err + } + + err = t.trainedIndex.segment.Close() + if err != nil { + if serr := os.Remove(trainedIndexPath + "temp"); serr != nil { + return fmt.Errorf("error removing temp trained index: %v", serr) + } + t.trainedIndex = nil + return err + } + + if err := moveFile(trainedIndexPath+"temp", trainedIndexPath); err != nil { + if serr := os.Remove(trainedIndexPath + "temp"); serr != nil { + return fmt.Errorf("error removing temp trained index: %v", serr) + } + t.trainedIndex = nil + return err + } + + trainedIndex, err := t.parent.segPlugin.OpenUsing(trainedIndexPath, t.config) + if err != nil { + t.trainedIndex = nil + return err + } + + t.trainedIndex = &SegmentSnapshot{segment: trainedIndex} + } + } + } + return nil +} + +func (t *vectorTrainer) fileWriterIDsInUse() (map[string]struct{}, error) { + t.m.RLock() + defer t.m.RUnlock() + writerIDs := make(map[string]struct{}) + if t.trainedIndex != nil { + if seg, ok := t.trainedIndex.segment.(segment.SegmentWithCallbacks); ok { + writerIDs[seg.CallbackId()] = struct{}{} + } + } + + return writerIDs, nil +} diff --git a/index/scorch/train_vector_test.go b/index/scorch/train_vector_test.go new file mode 100644 index 0000000..f111ff8 --- /dev/null +++ b/index/scorch/train_vector_test.go @@ -0,0 +1,849 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package scorch + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +// newMinimalScorchForTrainer returns a *Scorch with just the fields required by +// vectorTrainer tests that do not touch BoltDB or the segment plugin. +func newMinimalScorchForTrainer(t *testing.T) *Scorch { + t.Helper() + return &Scorch{ + closeCh: make(chan struct{}), + segmentConfig: make(map[string]interface{}), + config: make(map[string]interface{}), + path: t.TempDir(), + } +} + +// waitGroupDone blocks until wg.Wait() returns or d elapses, returning true +// iff the wait completed in time. +func waitGroupDone(wg *sync.WaitGroup, d time.Duration) bool { + done := make(chan struct{}) + go func() { wg.Wait(); close(done) }() + select { + case <-done: + return true + case <-time.After(d): + return false + } +} + +// ----------------------------------------------------------------------- +// initTrainer +// ----------------------------------------------------------------------- + +func TestInitTrainerFlagAbsent(t *testing.T) { + s := newMinimalScorchForTrainer(t) + if tr := initTrainer(s, s.config); tr != nil { + t.Fatal("expected nil trainer when flag is absent, got non-nil") + } +} + +func TestInitTrainerFlagFalse(t *testing.T) { + s := newMinimalScorchForTrainer(t) + cfg := map[string]interface{}{IndexTrainedWithFastMerge: false} + if tr := initTrainer(s, cfg); tr != nil { + t.Fatal("expected nil trainer when flag is false, got non-nil") + } +} + +func TestInitTrainerReturnsTrainer(t *testing.T) { + s := newMinimalScorchForTrainer(t) + cfg := map[string]interface{}{IndexTrainedWithFastMerge: true} + if tr := initTrainer(s, cfg); tr == nil { + t.Fatal("expected non-nil trainer when flag is true") + } +} + +func TestInitTrainerClonesConfig(t *testing.T) { + s := newMinimalScorchForTrainer(t) + s.config["pre-existing"] = "value" + cfg := map[string]interface{}{IndexTrainedWithFastMerge: true} + tr := initTrainer(s, cfg) + if tr == nil { + t.Fatal("expected non-nil trainer") + } + // Mutations to the trainer's config must not affect the parent's config. + tr.config["canary"] = true + if _, leaked := s.config["canary"]; leaked { + t.Error("trainer config is the same map as parent config — expected a clone") + } +} + +// ----------------------------------------------------------------------- +// getInternal +// ----------------------------------------------------------------------- + +func TestGetInternalDefaultFalse(t *testing.T) { + s := newMinimalScorchForTrainer(t) + tr := &vectorTrainer{parent: s, config: map[string]interface{}{}} + + got, err := tr.getInternal(util.BoltTrainCompleteKey) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(got) != "false" { + t.Errorf("expected 'false', got %q", got) + } +} + +func TestGetInternalTrueAfterSet(t *testing.T) { + s := newMinimalScorchForTrainer(t) + tr := &vectorTrainer{parent: s, config: map[string]interface{}{}} + tr.trainingComplete.Store(true) + + got, err := tr.getInternal(util.BoltTrainCompleteKey) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(got) != "true" { + t.Errorf("expected 'true', got %q", got) + } +} + +func TestGetInternalUnknownKey(t *testing.T) { + s := newMinimalScorchForTrainer(t) + tr := &vectorTrainer{parent: s, config: map[string]interface{}{}} + + got, err := tr.getInternal([]byte("no-such-key")) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != nil { + t.Errorf("expected nil for unknown key, got %v", got) + } +} + +// ----------------------------------------------------------------------- +// getTrainedIndex +// ----------------------------------------------------------------------- + +func TestGetTrainedIndexNil(t *testing.T) { + s := newMinimalScorchForTrainer(t) + tr := &vectorTrainer{parent: s, config: map[string]interface{}{}} + + got, err := tr.getTrainedIndex("vec") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != nil { + t.Errorf("expected nil, got %v", got) + } +} + +func TestGetTrainedIndexNotTrainedSegment(t *testing.T) { + s := newMinimalScorchForTrainer(t) + tr := &vectorTrainer{ + parent: s, + config: map[string]interface{}{}, + // mockSegmentBase does NOT implement segment.TrainedSegment. + trainedIndex: &SegmentSnapshot{segment: &mockSegmentBase{}}, + } + + _, err := tr.getTrainedIndex("vec") + if err == nil { + t.Fatal("expected error for segment that does not implement TrainedSegment") + } +} + +// mockTrainedSeg embeds mockSegmentBase and additionally satisfies +// segment.TrainedSegment by implementing GetCoarseQuantizer. +type mockTrainedSeg struct { + mockSegmentBase + coarseQuantizerFn func(string) (interface{}, error) +} + +func (m *mockTrainedSeg) GetCoarseQuantizer(field string) (interface{}, error) { + if m.coarseQuantizerFn != nil { + return m.coarseQuantizerFn(field) + } + return nil, nil +} + +func TestGetTrainedIndexReturnsCQ(t *testing.T) { + sentinel := struct{ label string }{"centroid"} + s := newMinimalScorchForTrainer(t) + tr := &vectorTrainer{ + parent: s, + config: map[string]interface{}{}, + trainedIndex: &SegmentSnapshot{ + segment: &mockTrainedSeg{ + coarseQuantizerFn: func(string) (interface{}, error) { + return sentinel, nil + }, + }, + }, + } + + got, err := tr.getTrainedIndex("vec") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != sentinel { + t.Errorf("expected sentinel coarse quantizer, got %v", got) + } +} + +func TestGetTrainedIndexPropagatesError(t *testing.T) { + s := newMinimalScorchForTrainer(t) + tr := &vectorTrainer{ + parent: s, + config: map[string]interface{}{}, + trainedIndex: &SegmentSnapshot{ + segment: &mockTrainedSeg{ + coarseQuantizerFn: func(string) (interface{}, error) { + return nil, fmt.Errorf("coarse quantizer unavailable") + }, + }, + }, + } + + _, err := tr.getTrainedIndex("vec") + if err == nil { + t.Fatal("expected error from GetCoarseQuantizer, got nil") + } +} + +// ----------------------------------------------------------------------- +// trainLoop — exit paths that do not require BoltDB +// ----------------------------------------------------------------------- + +// TestTrainLoopExitsWhenComplete verifies that the loop returns +// immediately (without touching any channel) when trainingComplete is already +// set, and that it flushes the sample count into parent stats. +func TestTrainLoopExitsWhenComplete(t *testing.T) { + s := newMinimalScorchForTrainer(t) + tr := &vectorTrainer{ + parent: s, + config: map[string]interface{}{}, + trainCh: make(chan *trainRequest, 1), + } + tr.trainedSamples = 42 + tr.trainingComplete.Store(true) + + s.asyncTasks.Add(1) + go tr.trainLoop() + + if !waitGroupDone(&s.asyncTasks, 2*time.Second) { + t.Fatal("trainLoop did not exit within deadline") + } + if got := atomic.LoadUint64(&s.stats.TotTrainedSamples); got != 42 { + t.Errorf("expected TotTrainedSamples=42, got %d", got) + } +} + +// TestTrainLoopCloseChNoRequest verifies that closing the +// parent's closeCh causes the loop to exit cleanly when there is no buffered +// train request. +func TestTrainLoopCloseChNoRequest(t *testing.T) { + s := newMinimalScorchForTrainer(t) + tr := &vectorTrainer{ + parent: s, + config: map[string]interface{}{}, + trainCh: make(chan *trainRequest, 1), + } + + s.asyncTasks.Add(1) + go tr.trainLoop() + + close(s.closeCh) + if !waitGroupDone(&s.asyncTasks, 2*time.Second) { + t.Fatal("trainLoop did not exit within deadline") + } +} + +// TestTrainLoopCloseChAcksRequest verifies that the goroutine +// exits after closeCh is closed, and that any ackCh buffered in trainCh is +// handled before the goroutine returns (no leaked ackCh). +// +// Due to Go's non-deterministic select, the loop may either: +// +// (a) react to closeCh first → drain the request and ack it with an error, OR +// (b) consume the request first (nil-sample, non-final → close ackCh) and +// then exit via closeCh on the next iteration. +// +// Either outcome is correct. The invariant under test is that ackCh is +// drained/closed before the goroutine exits, preventing a caller goroutine leak. +func TestTrainLoopCloseChAcksRequest(t *testing.T) { + s := newMinimalScorchForTrainer(t) + tr := &vectorTrainer{ + parent: s, + config: map[string]interface{}{}, + trainCh: make(chan *trainRequest, 1), + } + + // Buffered so that the loop's non-blocking write (if it takes closeCh path) + // does not deadlock. + ackCh := make(chan error, 1) + tr.trainCh <- &trainRequest{finalSample: false, sample: nil, ackCh: ackCh} + + s.asyncTasks.Add(1) + go tr.trainLoop() + + close(s.closeCh) + if !waitGroupDone(&s.asyncTasks, 2*time.Second) { + t.Fatal("trainLoop did not exit within deadline") + } + + // ackCh must be drainable without blocking regardless of which select path + // was taken. + select { + case <-ackCh: + // received error (closeCh drain path) or nil (closed by normal processing) + case <-time.After(100 * time.Millisecond): + t.Fatal("ackCh was not handled after trainLoop exited — possible goroutine leak") + } +} + +// TestTrainLoopNilSampleAcks verifies that a nil-sample, +// non-final request is acknowledged without error, and that the loop continues +// running afterwards (cleaned up by closing closeCh). +func TestTrainLoopNilSampleAcks(t *testing.T) { + s := newMinimalScorchForTrainer(t) + tr := &vectorTrainer{ + parent: s, + config: map[string]interface{}{}, + trainCh: make(chan *trainRequest, 1), + } + + ackCh := make(chan error, 1) + tr.trainCh <- &trainRequest{finalSample: false, sample: nil, ackCh: ackCh} + + s.asyncTasks.Add(1) + go tr.trainLoop() + + // The loop should close ackCh with no value written to it. + select { + case err := <-ackCh: + if err != nil { + t.Errorf("expected nil error for nil-sample non-final request, got: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for ackCh to be closed") + } + + // Stop the loop cleanly. + close(s.closeCh) + if !waitGroupDone(&s.asyncTasks, 2*time.Second) { + t.Fatal("trainLoop did not exit within deadline after closeCh") + } +} + +// mockUnpersistedSeg satisfies both segment.Segment (via mockSegmentBase) and +// segment.UnpersistedSegment. persistFn controls the return value of Persist. +type mockUnpersistedSeg struct { + mockSegmentBase + persistFn func(path string) error +} + +func (m *mockUnpersistedSeg) Persist(path string) error { + if m.persistFn != nil { + return m.persistFn(path) + } + return nil +} + +// TestTrainLoopPersistErrorAcks verifies that +// when persist-to-directory fails for a non-nil UnpersistedSegment, the loop: +// 1. sends an error on ackCh and closes it, and +// 2. continues running (does not return), so a subsequent closeCh can stop it. +func TestTrainLoopPersistErrorAcks(t *testing.T) { + s := newMinimalScorchForTrainer(t) + tr := &vectorTrainer{ + parent: s, + config: map[string]interface{}{}, + trainCh: make(chan *trainRequest, 1), + } + + ackCh := make(chan error, 1) + tr.trainCh <- &trainRequest{ + finalSample: false, + sample: &mockUnpersistedSeg{ + persistFn: func(string) error { return fmt.Errorf("disk full") }, + }, + ackCh: ackCh, + } + + s.asyncTasks.Add(1) + go tr.trainLoop() + + // The loop should write an error and close ackCh before continuing. + select { + case err, ok := <-ackCh: + if !ok { + t.Fatal("ackCh was closed before the error value could be read") + } + if err == nil { + t.Fatal("expected non-nil error from failed persist, got nil") + } + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for error on ackCh") + } + + // Loop must still be running; stop it cleanly. + close(s.closeCh) + if !waitGroupDone(&s.asyncTasks, 2*time.Second) { + t.Fatal("trainLoop did not exit within deadline after closeCh") + } +} + +func TestTrainerSampleIngestion(t *testing.T) { + cfg := CreateConfig("TestTrainerSampleIngestion") + cfg[IndexTrainedWithFastMerge] = true + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + analysisQueue := index.NewAnalysisQueue(1) + s, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("failed to create Scorch: %v", err) + } + defer s.Close() + + sc, ok := s.(*Scorch) + if !ok { + t.Fatal("expected Scorch instance") + } + + batch := index.NewBatch() + for i := 0; i < 100; i++ { + doc := document.NewDocument(fmt.Sprintf("doc-%d", i)) + stubVal := float32(i) + doc.AddField(document.NewVectorField("vec", nil, []float32{stubVal, stubVal + 1, stubVal + 2}, 3, "cosine", index.IndexOptimizedForRecall)) + batch.Update(doc) + } + + tr, ok := sc.trainer.(*vectorTrainer) + if !ok { + t.Fatal("expected vectorTrainer instance") + } + if sc.rootBolt == nil { + err := sc.openBolt() + if err != nil { + t.Fatalf("opening bolt failed %v", err) + } + } + sc.asyncTasks.Add(1) + go tr.trainLoop() + + err = sc.Train(batch) + if err != nil { + t.Fatalf("training failed: %v", err) + } + + val, err := tr.getInternal(util.BoltTrainCompleteKey) + if err != nil { + t.Fatalf("failed to get internal value: %v", err) + } + + if string(val) != "false" { + t.Errorf("expected 'false' for training complete key, got %q", val) + } + + if tr.trainedSamples != 100 { + t.Errorf("expected 100 trained samples, got %d", tr.trainedSamples) + } + + b := index.NewBatch() + doc := document.NewDocument("doc-998") + doc.AddField(document.NewVectorField("vec", nil, []float32{998, 998, 998}, 3, "cosine", index.IndexOptimizedForRecall)) + b.Update(doc) + err = sc.Train(b) + if err != nil { + t.Fatalf("training failed: %v", err) + } + val, err = tr.getInternal(util.BoltTrainCompleteKey) + if err != nil { + t.Fatalf("failed to get internal value: %v", err) + } + + if string(val) != "false" { + t.Errorf("expected 'false' for training complete key, got %q", val) + } + + if tr.trainedSamples != 101 { + t.Errorf("expected 101 trained samples, got %d", tr.trainedSamples) + } + + b.Reset() + b.SetInternal(util.BoltTrainCompleteKey, []byte("true")) + sc.Train(b) + if err != nil { + t.Fatalf("training failed: %v", err) + } + + val, err = tr.getInternal(util.BoltTrainCompleteKey) + if err != nil { + t.Fatalf("failed to get internal value: %v", err) + } + + if string(val) != "true" { + t.Errorf("expected 'true' for training complete key, got %q", val) + } + + if tr.trainedSamples != 101 { + t.Errorf("expected 101 trained samples (no increment after training complete), got %d", tr.trainedSamples) + } +} + +func TestTrainerBlocksDataIngestion(t *testing.T) { + cfg := CreateConfig("TestTrainerBlocksDataIngestion") + cfg[IndexTrainedWithFastMerge] = true + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + analysisQueue := index.NewAnalysisQueue(1) + s, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("failed to create Scorch: %v", err) + } + defer s.Close() + + sc, ok := s.(*Scorch) + if !ok { + t.Fatal("expected Scorch instance") + } + + batch := index.NewBatch() + for i := 0; i < 100; i++ { + doc := document.NewDocument(fmt.Sprintf("doc-%d", i)) + stubVal := float32(i) + doc.AddField(document.NewVectorField("vec", nil, []float32{stubVal, stubVal + 1, stubVal + 2}, 3, "cosine", index.IndexOptimizedForRecall)) + batch.Update(doc) + } + + tr, ok := sc.trainer.(*vectorTrainer) + if !ok { + t.Fatal("expected vectorTrainer instance") + } + if sc.rootBolt == nil { + err := sc.openBolt() + if err != nil { + t.Fatalf("opening bolt failed %v", err) + } + } + + // spawn all the routines to check the data ingest as well + err = sc.Open() + if err != nil { + t.Fatalf("failed to open Scorch: %v", err) + } + + err = sc.Train(batch) + if err != nil { + t.Fatalf("training failed: %v", err) + } + + val, err := tr.getInternal(util.BoltTrainCompleteKey) + if err != nil { + t.Fatalf("failed to get internal value: %v", err) + } + + if string(val) != "false" { + t.Errorf("expected 'false' for training complete key, got %q", val) + } + + if tr.trainedSamples != 100 { + t.Errorf("expected 100 trained samples, got %d", tr.trainedSamples) + } + + batch.Reset() + doc := document.NewDocument("doc-998") + doc.AddField(document.NewVectorField("vec", nil, []float32{998, 998, 998}, 3, "cosine", index.IndexOptimizedForRecall)) + batch.Update(doc) + err = sc.Batch(batch) + if err == nil { + t.Fatalf("data ingestion should be blocked during training") + } + + batch.Reset() + batch.SetInternal(util.BoltTrainCompleteKey, []byte("true")) + err = sc.Train(batch) + if err != nil { + t.Fatalf("training failed: %v", err) + } + + val, err = tr.getInternal(util.BoltTrainCompleteKey) + if err != nil { + t.Fatalf("failed to get internal value: %v", err) + } + + if string(val) != "true" { + t.Errorf("expected 'true' for training complete key, got %q", val) + } + + batch.Reset() + doc = document.NewDocument("doc-998") + doc.AddField(document.NewVectorField("vec", nil, []float32{998, 998, 998}, 3, "cosine", index.IndexOptimizedForRecall)) + batch.Update(doc) + err = sc.Batch(batch) + if err != nil { + t.Fatalf("data ingestion expected to suceed after training is complete") + } + + err = sc.Train(batch) + if err == nil { + t.Fatalf("re-training is not allowed on the index") + } + +} + +type dummyFileCopyAPI interface { + CopyFile(fileName string, dst index.IndexDirectory) error + SetPathInBolt(key []byte, value []byte) error +} + +// trainedIndexWriter implements the writer used when copying the trained index +// file from the source partition into this partition's index directory. +type trainedIndexWriter struct { + rootpath string + destIndex dummyFileCopyAPI +} + +func (c *trainedIndexWriter) GetWriter(path string) (io.WriteCloser, error) { + if !(strings.HasSuffix(path, index.TrainedIndexFileName)) { + return nil, fmt.Errorf("write not allowed on path %s", path) + } + return os.OpenFile(filepath.Join(c.rootpath, filepath.Base(path)), os.O_CREATE|os.O_WRONLY, 0600) +} +func (c *trainedIndexWriter) SetPathInBolt(key []byte, value []byte) error { + return c.destIndex.SetPathInBolt(key, value) +} + +func TestTrainerIndexCopy(t *testing.T) { + cfg := CreateConfig("TestTrainerIndexCopy") + cfg[IndexTrainedWithFastMerge] = true + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + + cfg["path"] = filepath.Clean(cfg["path"].(string)) + analysisQueue := index.NewAnalysisQueue(1) + s, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("failed to create Scorch: %v", err) + } + defer s.Close() + + sc, ok := s.(*Scorch) + if !ok { + t.Fatal("expected Scorch instance") + } + + batch := index.NewBatch() + for i := 0; i < 100; i++ { + doc := document.NewDocument(fmt.Sprintf("doc-%d", i)) + stubVal := float32(i) + doc.AddField(document.NewVectorField("vec", nil, []float32{stubVal, stubVal + 1, stubVal + 2}, 3, "cosine", index.IndexOptimizedForRecall)) + batch.Update(doc) + } + + tr, ok := sc.trainer.(*vectorTrainer) + if !ok { + t.Fatal("expected vectorTrainer instance") + } + if sc.rootBolt == nil { + err := sc.openBolt() + if err != nil { + t.Fatalf("opening bolt failed %v", err) + } + } + sc.asyncTasks.Add(1) + go tr.trainLoop() + + err = sc.Train(batch) + if err != nil { + t.Fatalf("training failed: %v", err) + } + + val, err := tr.getInternal(util.BoltTrainCompleteKey) + if err != nil { + t.Fatalf("failed to get internal value: %v", err) + } + + if string(val) != "false" { + t.Errorf("expected 'false' for training complete key, got %q", val) + } + + // ----------------------------------------------------------------------- + // At this point the source trainer has been trained with 100 samples, and the + // trained index file should be available on disk. Now we will create a new + // trainer in a new Scorch instance, and copy the trained index file from the + // source trainer into the destination trainer's index directory, simulating + // the file copy between partitions to avoid re-training. + // ----------------------------------------------------------------------- + dstCfg := CreateConfig("TestTrainerIndexCopyDest") + dstCfg[IndexTrainedWithFastMerge] = true + err = InitTest(dstCfg) + if err != nil { + t.Fatal(err) + } + + dstCfg["path"] = filepath.Clean(dstCfg["path"].(string)) + + dstWriter := &trainedIndexWriter{ + rootpath: dstCfg["path"].(string), + } + + dstScorch, err := NewScorch(Name, dstCfg, analysisQueue) + if err != nil { + t.Fatalf("failed to create new Scorch: %v", err) + } + defer dstScorch.Close() + + dst, ok := dstScorch.(*Scorch) + if !ok { + t.Fatal("expected Scorch instance") + } + dstWriter.destIndex = dstScorch.(dummyFileCopyAPI) + + dstTrainer, ok := dst.trainer.(*vectorTrainer) + if !ok { + t.Fatal("expected vectorTrainer instance") + } + if dst.rootBolt == nil { + err := dst.openBolt() + if err != nil { + t.Fatalf("opening bolt failed %v", err) + } + } + dst.asyncTasks.Add(1) + go dstTrainer.trainLoop() + + src, ok := s.(dummyFileCopyAPI) + if !ok { + t.Fatal("expected dummyFileCopyAPI instance") + } + + err = src.CopyFile(index.TrainedIndexFileName, dstWriter) + if err != nil { + t.Fatalf("CopyFile failed: %v", err) + } + + val, err = dstTrainer.getInternal(util.BoltTrainCompleteKey) + if err != nil { + t.Fatalf("failed to get internal value: %v", err) + } + + if string(val) != "false" { + t.Errorf("expected 'false' for training complete key in destination trainer, got %q", val) + } + + if trainedIndex, err := dstTrainer.getTrainedIndex("vec"); err != nil || trainedIndex == nil { + t.Errorf("expected trained index in destination trainer, got nil") + } + +} + +func TestTrainerFallbackNaiveMerge(t *testing.T) { + cfg := CreateConfig("TestTrainerFallbackNaiveMerge") + cfg[IndexTrainedWithFastMerge] = true + err := InitTest(cfg) + if err != nil { + t.Fatal(err) + } + + cfg["path"] = filepath.Clean(cfg["path"].(string)) + analysisQueue := index.NewAnalysisQueue(1) + s, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("failed to create Scorch: %v", err) + } + + sc, ok := s.(*Scorch) + if !ok { + t.Fatal("expected Scorch instance") + } + + tr, ok := sc.trainer.(*vectorTrainer) + if !ok { + t.Fatal("expected vectorTrainer instance") + } + if sc.rootBolt == nil { + err := sc.openBolt() + if err != nil { + t.Fatalf("opening bolt failed %v", err) + } + } + // spawn all the routines to check the data ingest as well + err = sc.Open() + if err != nil { + t.Fatalf("failed to open Scorch: %v", err) + } + + // mocking the scenario where the application would've failed to create the trained index file + // due to some fail of operation but expects the underlying the indexing layer + // to continue ingestion and fallback to naive merge and complete the training + // without the trained index file. + batch := index.NewBatch() + batch.SetInternal(util.BoltTrainCompleteKey, []byte("true")) + err = sc.Train(batch) + if err != nil { + t.Fatalf("training failed: %v", err) + } + + batch.Reset() + doc := document.NewDocument("doc-998") + doc.AddField(document.NewVectorField("vec", nil, []float32{998, 998, 998}, 3, "cosine", index.IndexOptimizedForRecall)) + batch.Update(doc) + err = sc.Batch(batch) + if err != nil { + t.Fatalf("data ingestion expected to suceed after training is complete") + } + + val, err := tr.getInternal(util.BoltTrainCompleteKey) + if err != nil { + t.Fatalf("failed to get internal value: %v", err) + } + + if string(val) != "true" { + t.Errorf("expected 'true' for training complete key, got %q", val) + } + + s.Close() + + // Re-open the index to verify that it can be loaded without the trained index file. + sn, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatalf("failed to re-open Scorch: %v", err) + } + + err = sn.Open() + if err != nil { + t.Fatalf("failed to open Scorch after fallback: %v", err) + } + defer sn.Close() +} diff --git a/index/scorch/unadorned.go b/index/scorch/unadorned.go new file mode 100644 index 0000000..e73b14b --- /dev/null +++ b/index/scorch/unadorned.go @@ -0,0 +1,219 @@ +// Copyright (c) 2020 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "math" + "reflect" + + "github.com/RoaringBitmap/roaring/v2" + segment "github.com/blevesearch/scorch_segment_api/v2" +) + +var reflectStaticSizeUnadornedPostingsIteratorBitmap int +var reflectStaticSizeUnadornedPostingsIterator1Hit int +var reflectStaticSizeUnadornedPosting int + +func init() { + var pib unadornedPostingsIteratorBitmap + reflectStaticSizeUnadornedPostingsIteratorBitmap = int(reflect.TypeOf(pib).Size()) + var pi1h unadornedPostingsIterator1Hit + reflectStaticSizeUnadornedPostingsIterator1Hit = int(reflect.TypeOf(pi1h).Size()) + var up UnadornedPosting + reflectStaticSizeUnadornedPosting = int(reflect.TypeOf(up).Size()) +} + +type unadornedPostingsIteratorBitmap struct { + actual roaring.IntPeekable + actualBM *roaring.Bitmap + next UnadornedPosting // reused across Next() calls +} + +func (i *unadornedPostingsIteratorBitmap) Next() (segment.Posting, error) { + return i.nextAtOrAfter(0) +} + +func (i *unadornedPostingsIteratorBitmap) Advance(docNum uint64) (segment.Posting, error) { + return i.nextAtOrAfter(docNum) +} + +func (i *unadornedPostingsIteratorBitmap) nextAtOrAfter(atOrAfter uint64) (segment.Posting, error) { + docNum, exists := i.nextDocNumAtOrAfter(atOrAfter) + if !exists { + return nil, nil + } + i.next = UnadornedPosting{} // clear the struct + rv := &i.next + rv.docNum = docNum + return rv, nil +} + +func (i *unadornedPostingsIteratorBitmap) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool) { + if i.actual == nil || !i.actual.HasNext() { + return 0, false + } + i.actual.AdvanceIfNeeded(uint32(atOrAfter)) + + if !i.actual.HasNext() { + return 0, false // couldn't find anything + } + + return uint64(i.actual.Next()), true +} + +func (i *unadornedPostingsIteratorBitmap) Size() int { + return reflectStaticSizeUnadornedPostingsIteratorBitmap +} + +func (i *unadornedPostingsIteratorBitmap) BytesRead() uint64 { + return 0 +} + +func (i *unadornedPostingsIteratorBitmap) BytesWritten() uint64 { + return 0 +} + +func (i *unadornedPostingsIteratorBitmap) ResetBytesRead(uint64) {} + +func (i *unadornedPostingsIteratorBitmap) ActualBitmap() *roaring.Bitmap { + return i.actualBM +} + +func (i *unadornedPostingsIteratorBitmap) DocNum1Hit() (uint64, bool) { + return 0, false +} + +func (i *unadornedPostingsIteratorBitmap) ReplaceActual(actual *roaring.Bitmap) { + i.actualBM = actual + i.actual = actual.Iterator() +} + +// Resets the iterator to the beginning of the postings list. +// by resetting the actual iterator. +func (i *unadornedPostingsIteratorBitmap) ResetIterator() { + i.actual = i.actualBM.Iterator() +} + +func newUnadornedPostingsIteratorFromBitmap(bm *roaring.Bitmap) segment.PostingsIterator { + return &unadornedPostingsIteratorBitmap{ + actualBM: bm, + actual: bm.Iterator(), + } +} + +const docNum1HitFinished = math.MaxUint64 + +type unadornedPostingsIterator1Hit struct { + docNumOrig uint64 // original 1-hit docNum used to create this iterator + docNum uint64 // current docNum + next UnadornedPosting // reused across Next() calls +} + +func (i *unadornedPostingsIterator1Hit) Next() (segment.Posting, error) { + return i.nextAtOrAfter(0) +} + +func (i *unadornedPostingsIterator1Hit) Advance(docNum uint64) (segment.Posting, error) { + return i.nextAtOrAfter(docNum) +} + +func (i *unadornedPostingsIterator1Hit) nextAtOrAfter(atOrAfter uint64) (segment.Posting, error) { + docNum, exists := i.nextDocNumAtOrAfter(atOrAfter) + if !exists { + return nil, nil + } + i.next = UnadornedPosting{} // clear the struct + rv := &i.next + rv.docNum = docNum + return rv, nil +} + +func (i *unadornedPostingsIterator1Hit) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool) { + if i.docNum == docNum1HitFinished { + return 0, false + } + if i.docNum < atOrAfter { + // advanced past our 1-hit + i.docNum = docNum1HitFinished // consume our 1-hit docNum + return 0, false + } + docNum := i.docNum + i.docNum = docNum1HitFinished // consume our 1-hit docNum + return docNum, true +} + +func (i *unadornedPostingsIterator1Hit) Size() int { + return reflectStaticSizeUnadornedPostingsIterator1Hit +} + +func (i *unadornedPostingsIterator1Hit) BytesRead() uint64 { + return 0 +} + +func (i *unadornedPostingsIterator1Hit) BytesWritten() uint64 { + return 0 +} + +func (i *unadornedPostingsIterator1Hit) ResetBytesRead(uint64) {} + +func (i *unadornedPostingsIterator1Hit) ActualBitmap() *roaring.Bitmap { return nil } + +func (i *unadornedPostingsIterator1Hit) DocNum1Hit() (uint64, bool) { + return i.docNumOrig, true +} + +// ReplaceActual is a no-op: a 1-hit iterator has no actual bitmap, and callers +// only invoke ReplaceActual on iterators whose ActualBitmap is non-nil. +func (i *unadornedPostingsIterator1Hit) ReplaceActual(*roaring.Bitmap) {} + +// ResetIterator resets the iterator to the original state. +func (i *unadornedPostingsIterator1Hit) ResetIterator() { + i.docNum = i.docNumOrig +} + +func newUnadornedPostingsIteratorFrom1Hit(docNum1Hit uint64) segment.PostingsIterator { + return &unadornedPostingsIterator1Hit{ + docNumOrig: docNum1Hit, + docNum: docNum1Hit, + } +} + +type ResetablePostingsIterator interface { + ResetIterator() +} + +type UnadornedPosting struct { + docNum uint64 +} + +func (p *UnadornedPosting) Number() uint64 { + return p.docNum +} + +func (p *UnadornedPosting) Frequency() uint64 { + return 0 +} + +func (p *UnadornedPosting) Norm() float64 { + return 0 +} + +func (p *UnadornedPosting) Locations() []segment.Location { + return nil +} + +func (p *UnadornedPosting) Size() int { + return reflectStaticSizeUnadornedPosting +} diff --git a/index/upsidedown/analysis.go b/index/upsidedown/analysis.go new file mode 100644 index 0000000..1ebd191 --- /dev/null +++ b/index/upsidedown/analysis.go @@ -0,0 +1,129 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + index "github.com/blevesearch/bleve_index_api" +) + +type IndexRow interface { + KeySize() int + KeyTo([]byte) (int, error) + Key() []byte + + ValueSize() int + ValueTo([]byte) (int, error) + Value() []byte +} + +type AnalysisResult struct { + DocID string + Rows []IndexRow +} + +func (udc *UpsideDownCouch) Analyze(d index.Document) *AnalysisResult { + return udc.analyze(d) +} + +func (udc *UpsideDownCouch) analyze(d index.Document) *AnalysisResult { + rv := &AnalysisResult{ + DocID: d.ID(), + Rows: make([]IndexRow, 0, 100), + } + + docIDBytes := []byte(d.ID()) + + // track our back index entries + backIndexStoredEntries := make([]*BackIndexStoreEntry, 0) + + // information we collate as we merge fields with same name + fieldTermFreqs := make(map[uint16]index.TokenFrequencies) + fieldLengths := make(map[uint16]int) + fieldIncludeTermVectors := make(map[uint16]bool) + fieldNames := make(map[uint16]string) + + analyzeField := func(field index.Field, storable bool) { + fieldIndex, newFieldRow := udc.fieldIndexOrNewRow(field.Name()) + if newFieldRow != nil { + rv.Rows = append(rv.Rows, newFieldRow) + } + fieldNames[fieldIndex] = field.Name() + + if field.Options().IsIndexed() { + field.Analyze() + fieldLength := field.AnalyzedLength() + tokenFreqs := field.AnalyzedTokenFrequencies() + existingFreqs := fieldTermFreqs[fieldIndex] + if existingFreqs == nil { + fieldTermFreqs[fieldIndex] = tokenFreqs + } else { + existingFreqs.MergeAll(field.Name(), tokenFreqs) + fieldTermFreqs[fieldIndex] = existingFreqs + } + fieldLengths[fieldIndex] += fieldLength + fieldIncludeTermVectors[fieldIndex] = field.Options().IncludeTermVectors() + } + + if storable && field.Options().IsStored() { + rv.Rows, backIndexStoredEntries = udc.storeField(docIDBytes, field, fieldIndex, rv.Rows, backIndexStoredEntries) + } + } + + // walk all the fields, record stored fields now + // place information about indexed fields into map + // this collates information across fields with + // same names (arrays) + d.VisitFields(func(field index.Field) { + analyzeField(field, true) + }) + + if d.HasComposite() { + for fieldIndex, tokenFreqs := range fieldTermFreqs { + // see if any of the composite fields need this + d.VisitComposite(func(field index.CompositeField) { + field.Compose(fieldNames[fieldIndex], fieldLengths[fieldIndex], tokenFreqs) + }) + } + + d.VisitComposite(func(field index.CompositeField) { + analyzeField(field, false) + }) + } + + rowsCapNeeded := len(rv.Rows) + 1 + for _, tokenFreqs := range fieldTermFreqs { + rowsCapNeeded += len(tokenFreqs) + } + + rv.Rows = append(make([]IndexRow, 0, rowsCapNeeded), rv.Rows...) + + backIndexTermsEntries := make([]*BackIndexTermsEntry, 0, len(fieldTermFreqs)) + + // walk through the collated information and process + // once for each indexed field (unique name) + for fieldIndex, tokenFreqs := range fieldTermFreqs { + fieldLength := fieldLengths[fieldIndex] + includeTermVectors := fieldIncludeTermVectors[fieldIndex] + + // encode this field + rv.Rows, backIndexTermsEntries = udc.indexField(docIDBytes, includeTermVectors, fieldIndex, fieldLength, tokenFreqs, rv.Rows, backIndexTermsEntries) + } + + // build the back index row + backIndexRow := NewBackIndexRow(docIDBytes, backIndexTermsEntries, backIndexStoredEntries) + rv.Rows = append(rv.Rows, backIndexRow) + + return rv +} diff --git a/index/upsidedown/analysis_test.go b/index/upsidedown/analysis_test.go new file mode 100644 index 0000000..4255346 --- /dev/null +++ b/index/upsidedown/analysis_test.go @@ -0,0 +1,115 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "testing" + + "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/null" + "github.com/blevesearch/bleve/v2/registry" + index "github.com/blevesearch/bleve_index_api" +) + +func TestAnalysisBug328(t *testing.T) { + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(standard.Name) + if err != nil { + t.Fatal(err) + } + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(null.Name, nil, analysisQueue) + if err != nil { + t.Fatal(err) + } + + d := document.NewDocument("1") + f := document.NewTextFieldCustom("title", nil, []byte("bleve"), index.IndexField|index.IncludeTermVectors, analyzer) + d.AddField(f) + f = document.NewTextFieldCustom("body", nil, []byte("bleve"), index.IndexField|index.IncludeTermVectors, analyzer) + d.AddField(f) + cf := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, []string{}, index.IndexField|index.IncludeTermVectors) + d.AddField(cf) + + rv := idx.(*UpsideDownCouch).analyze(d) + fieldIndexes := make(map[uint16]string) + for _, row := range rv.Rows { + if row, ok := row.(*FieldRow); ok { + fieldIndexes[row.index] = row.name + } + if row, ok := row.(*TermFrequencyRow); ok && string(row.term) == "bleve" { + for _, vec := range row.vectors { + if vec.field != row.field { + if fieldIndexes[row.field] != "_all" { + t.Errorf("row named %s field %d - vector field %d", fieldIndexes[row.field], row.field, vec.field) + } + } + } + } + } +} + +func BenchmarkAnalyze(b *testing.B) { + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(standard.Name) + if err != nil { + b.Fatal(err) + } + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(null.Name, nil, analysisQueue) + if err != nil { + b.Fatal(err) + } + + d := document.NewDocument("1") + f := document.NewTextFieldWithAnalyzer("desc", nil, bleveWikiArticle1K, analyzer) + d.AddField(f) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + rv := idx.(*UpsideDownCouch).analyze(d) + if len(rv.Rows) < 92 || len(rv.Rows) > 93 { + b.Fatalf("expected 512-13 rows, got %d", len(rv.Rows)) + } + } +} + +var bleveWikiArticle1K = []byte(`Boiling liquid expanding vapor explosion +From Wikipedia, the free encyclopedia +See also: Boiler explosion and Steam explosion + +Flames subsequent to a flammable liquid BLEVE from a tanker. BLEVEs do not necessarily involve fire. + +This article's tone or style may not reflect the encyclopedic tone used on Wikipedia. See Wikipedia's guide to writing better articles for suggestions. (July 2013) +A boiling liquid expanding vapor explosion (BLEVE, /ˈblɛviː/ blev-ee) is an explosion caused by the rupture of a vessel containing a pressurized liquid above its boiling point.[1] +Contents [hide] +1 Mechanism +1.1 Water example +1.2 BLEVEs without chemical reactions +2 Fires +3 Incidents +4 Safety measures +5 See also +6 References +7 External links +Mechanism[edit] + +This section needs additional citations for verification. Please help improve this article by adding citations to reliable sources. Unsourced material may be challenged and removed. (July 2013) +There are three characteristics of liquids which are relevant to the discussion of a BLEVE:`) diff --git a/index/upsidedown/benchmark_all.sh b/index/upsidedown/benchmark_all.sh new file mode 100755 index 0000000..079fef1 --- /dev/null +++ b/index/upsidedown/benchmark_all.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +BENCHMARKS=`grep "func Benchmark" *_test.go | sed 's/.*func //' | sed s/\(.*{//` + +for BENCHMARK in $BENCHMARKS +do + go test -v -run=xxx -bench=^$BENCHMARK$ -benchtime=10s -tags 'forestdb leveldb' | grep -v ok | grep -v PASS +done diff --git a/index/upsidedown/benchmark_boltdb_test.go b/index/upsidedown/benchmark_boltdb_test.go new file mode 100644 index 0000000..5fabfa6 --- /dev/null +++ b/index/upsidedown/benchmark_boltdb_test.go @@ -0,0 +1,75 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "testing" + + "github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb" +) + +var boltTestConfig = map[string]interface{}{ + "path": "test", +} + +func BenchmarkBoltDBIndexing1Workers(b *testing.B) { + CommonBenchmarkIndex(b, boltdb.Name, boltTestConfig, DestroyTest, 1) +} + +func BenchmarkBoltDBIndexing2Workers(b *testing.B) { + CommonBenchmarkIndex(b, boltdb.Name, boltTestConfig, DestroyTest, 2) +} + +func BenchmarkBoltDBIndexing4Workers(b *testing.B) { + CommonBenchmarkIndex(b, boltdb.Name, boltTestConfig, DestroyTest, 4) +} + +// batches + +func BenchmarkBoltDBIndexing1Workers10Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 1, 10) +} + +func BenchmarkBoltDBIndexing2Workers10Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 2, 10) +} + +func BenchmarkBoltDBIndexing4Workers10Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 4, 10) +} + +func BenchmarkBoltDBIndexing1Workers100Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 1, 100) +} + +func BenchmarkBoltDBIndexing2Workers100Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 2, 100) +} + +func BenchmarkBoltDBIndexing4Workers100Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 4, 100) +} + +func BenchmarkBoltBIndexing1Workers1000Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 1, 1000) +} + +func BenchmarkBoltBIndexing2Workers1000Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 2, 1000) +} + +func BenchmarkBoltBIndexing4Workers1000Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 4, 1000) +} diff --git a/index/upsidedown/benchmark_common_test.go b/index/upsidedown/benchmark_common_test.go new file mode 100644 index 0000000..758b2a2 --- /dev/null +++ b/index/upsidedown/benchmark_common_test.go @@ -0,0 +1,149 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "os" + "strconv" + "testing" + + _ "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/registry" + index "github.com/blevesearch/bleve_index_api" +) + +var benchmarkDocBodies = []string{ + "A boiling liquid expanding vapor explosion (BLEVE, /ˈblɛviː/ blev-ee) is an explosion caused by the rupture of a vessel containing a pressurized liquid above its boiling point.", + "A boiler explosion is a catastrophic failure of a boiler. As seen today, boiler explosions are of two kinds. One kind is a failure of the pressure parts of the steam and water sides. There can be many different causes, such as failure of the safety valve, corrosion of critical parts of the boiler, or low water level. Corrosion along the edges of lap joints was a common cause of early boiler explosions.", + "A boiler is a closed vessel in which water or other fluid is heated. The fluid does not necessarily boil. (In North America the term \"furnace\" is normally used if the purpose is not actually to boil the fluid.) The heated or vaporized fluid exits the boiler for use in various processes or heating applications,[1][2] including central heating, boiler-based power generation, cooking, and sanitation.", + "A pressure vessel is a closed container designed to hold gases or liquids at a pressure substantially different from the ambient pressure.", + "Pressure (symbol: p or P) is the ratio of force to the area over which that force is distributed.", + "Liquid is one of the four fundamental states of matter (the others being solid, gas, and plasma), and is the only state with a definite volume but no fixed shape.", + "The boiling point of a substance is the temperature at which the vapor pressure of the liquid equals the pressure surrounding the liquid[1][2] and the liquid changes into a vapor.", + "Vapor pressure or equilibrium vapor pressure is defined as the pressure exerted by a vapor in thermodynamic equilibrium with its condensed phases (solid or liquid) at a given temperature in a closed system.", + "Industrial gases are a group of gases that are specifically manufactured for use in a wide range of industries, which include oil and gas, petrochemicals, chemicals, power, mining, steelmaking, metals, environmental protection, medicine, pharmaceuticals, biotechnology, food, water, fertilizers, nuclear power, electronics and aerospace.", + "The expansion ratio of a liquefied and cryogenic substance is the volume of a given amount of that substance in liquid form compared to the volume of the same amount of substance in gaseous form, at room temperature and normal atmospheric pressure.", +} + +type KVStoreDestroy func() error + +func DestroyTest() error { + return os.RemoveAll("test") +} + +func CommonBenchmarkIndex(b *testing.B, storeName string, storeConfig map[string]interface{}, destroy KVStoreDestroy, analysisWorkers int) { + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed("standard") + if err != nil { + b.Fatal(err) + } + + indexDocument := document.NewDocument(""). + AddField(document.NewTextFieldWithAnalyzer("body", []uint64{}, []byte(benchmarkDocBodies[0]), analyzer)) + + b.ResetTimer() + b.StopTimer() + for i := 0; i < b.N; i++ { + analysisQueue := index.NewAnalysisQueue(analysisWorkers) + idx, err := NewUpsideDownCouch(storeName, storeConfig, analysisQueue) + if err != nil { + b.Fatal(err) + } + + err = idx.Open() + if err != nil { + b.Fatal(err) + } + indexDocument.SetID(strconv.Itoa(i)) + // just time the indexing portion + b.StartTimer() + err = idx.Update(indexDocument) + if err != nil { + b.Fatal(err) + } + b.StopTimer() + err = idx.Close() + if err != nil { + b.Fatal(err) + } + err = destroy() + if err != nil { + b.Fatal(err) + } + analysisQueue.Close() + } +} + +func CommonBenchmarkIndexBatch(b *testing.B, storeName string, storeConfig map[string]interface{}, destroy KVStoreDestroy, analysisWorkers, batchSize int) { + + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed("standard") + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + b.StopTimer() + for i := 0; i < b.N; i++ { + + analysisQueue := index.NewAnalysisQueue(analysisWorkers) + idx, err := NewUpsideDownCouch(storeName, storeConfig, analysisQueue) + if err != nil { + b.Fatal(err) + } + + err = idx.Open() + if err != nil { + b.Fatal(err) + } + + b.StartTimer() + batch := index.NewBatch() + for j := 0; j < 1000; j++ { + if j%batchSize == 0 { + if len(batch.IndexOps) > 0 { + err := idx.Batch(batch) + if err != nil { + b.Fatal(err) + } + } + batch = index.NewBatch() + } + indexDocument := document.NewDocument(""). + AddField(document.NewTextFieldWithAnalyzer("body", []uint64{}, []byte(benchmarkDocBodies[j%10]), analyzer)) + indexDocument.SetID(strconv.Itoa(i) + "-" + strconv.Itoa(j)) + batch.Update(indexDocument) + } + // close last batch + if len(batch.IndexOps) > 0 { + err := idx.Batch(batch) + if err != nil { + b.Fatal(err) + } + } + b.StopTimer() + err = idx.Close() + if err != nil { + b.Fatal(err) + } + err = destroy() + if err != nil { + b.Fatal(err) + } + analysisQueue.Close() + } +} diff --git a/index/upsidedown/benchmark_gtreap_test.go b/index/upsidedown/benchmark_gtreap_test.go new file mode 100644 index 0000000..fb508fd --- /dev/null +++ b/index/upsidedown/benchmark_gtreap_test.go @@ -0,0 +1,71 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "testing" + + "github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap" +) + +func BenchmarkGTreapIndexing1Workers(b *testing.B) { + CommonBenchmarkIndex(b, gtreap.Name, nil, DestroyTest, 1) +} + +func BenchmarkGTreapIndexing2Workers(b *testing.B) { + CommonBenchmarkIndex(b, gtreap.Name, nil, DestroyTest, 2) +} + +func BenchmarkGTreapIndexing4Workers(b *testing.B) { + CommonBenchmarkIndex(b, gtreap.Name, nil, DestroyTest, 4) +} + +// batches + +func BenchmarkGTreapIndexing1Workers10Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 1, 10) +} + +func BenchmarkGTreapIndexing2Workers10Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 2, 10) +} + +func BenchmarkGTreapIndexing4Workers10Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 4, 10) +} + +func BenchmarkGTreapIndexing1Workers100Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 1, 100) +} + +func BenchmarkGTreapIndexing2Workers100Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 2, 100) +} + +func BenchmarkGTreapIndexing4Workers100Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 4, 100) +} + +func BenchmarkGTreapIndexing1Workers1000Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 1, 1000) +} + +func BenchmarkGTreapIndexing2Workers1000Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 2, 1000) +} + +func BenchmarkGTreapIndexing4Workers1000Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 4, 1000) +} diff --git a/index/upsidedown/benchmark_null_test.go b/index/upsidedown/benchmark_null_test.go new file mode 100644 index 0000000..1576aae --- /dev/null +++ b/index/upsidedown/benchmark_null_test.go @@ -0,0 +1,71 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "testing" + + "github.com/blevesearch/bleve/v2/index/upsidedown/store/null" +) + +func BenchmarkNullIndexing1Workers(b *testing.B) { + CommonBenchmarkIndex(b, null.Name, nil, DestroyTest, 1) +} + +func BenchmarkNullIndexing2Workers(b *testing.B) { + CommonBenchmarkIndex(b, null.Name, nil, DestroyTest, 2) +} + +func BenchmarkNullIndexing4Workers(b *testing.B) { + CommonBenchmarkIndex(b, null.Name, nil, DestroyTest, 4) +} + +// batches + +func BenchmarkNullIndexing1Workers10Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 1, 10) +} + +func BenchmarkNullIndexing2Workers10Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 2, 10) +} + +func BenchmarkNullIndexing4Workers10Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 4, 10) +} + +func BenchmarkNullIndexing1Workers100Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 1, 100) +} + +func BenchmarkNullIndexing2Workers100Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 2, 100) +} + +func BenchmarkNullIndexing4Workers100Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 4, 100) +} + +func BenchmarkNullIndexing1Workers1000Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 1, 1000) +} + +func BenchmarkNullIndexing2Workers1000Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 2, 1000) +} + +func BenchmarkNullIndexing4Workers1000Batch(b *testing.B) { + CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 4, 1000) +} diff --git a/index/upsidedown/dump.go b/index/upsidedown/dump.go new file mode 100644 index 0000000..64ebb1b --- /dev/null +++ b/index/upsidedown/dump.go @@ -0,0 +1,174 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "bytes" + "sort" + + "github.com/blevesearch/upsidedown_store_api" +) + +// the functions in this file are only intended to be used by +// the bleve_dump utility and the debug http handlers +// if your application relies on them, you're doing something wrong +// they may change or be removed at any time + +func dumpPrefix(kvreader store.KVReader, rv chan interface{}, prefix []byte) { + start := prefix + if start == nil { + start = []byte{0} + } + it := kvreader.PrefixIterator(start) + defer func() { + cerr := it.Close() + if cerr != nil { + rv <- cerr + } + }() + key, val, valid := it.Current() + for valid { + ck := make([]byte, len(key)) + copy(ck, key) + cv := make([]byte, len(val)) + copy(cv, val) + row, err := ParseFromKeyValue(ck, cv) + if err != nil { + rv <- err + return + } + rv <- row + + it.Next() + key, val, valid = it.Current() + } +} + +func dumpRange(kvreader store.KVReader, rv chan interface{}, start, end []byte) { + it := kvreader.RangeIterator(start, end) + defer func() { + cerr := it.Close() + if cerr != nil { + rv <- cerr + } + }() + key, val, valid := it.Current() + for valid { + ck := make([]byte, len(key)) + copy(ck, key) + cv := make([]byte, len(val)) + copy(cv, val) + row, err := ParseFromKeyValue(ck, cv) + if err != nil { + rv <- err + return + } + rv <- row + + it.Next() + key, val, valid = it.Current() + } +} + +func (i *IndexReader) DumpAll() chan interface{} { + rv := make(chan interface{}) + go func() { + defer close(rv) + dumpRange(i.kvreader, rv, nil, nil) + }() + return rv +} + +func (i *IndexReader) DumpFields() chan interface{} { + rv := make(chan interface{}) + go func() { + defer close(rv) + dumpPrefix(i.kvreader, rv, []byte{'f'}) + }() + return rv +} + +type keyset [][]byte + +func (k keyset) Len() int { return len(k) } +func (k keyset) Swap(i, j int) { k[i], k[j] = k[j], k[i] } +func (k keyset) Less(i, j int) bool { return bytes.Compare(k[i], k[j]) < 0 } + +// DumpDoc returns all rows in the index related to this doc id +func (i *IndexReader) DumpDoc(id string) chan interface{} { + idBytes := []byte(id) + + rv := make(chan interface{}) + + go func() { + defer close(rv) + + back, err := backIndexRowForDoc(i.kvreader, []byte(id)) + if err != nil { + rv <- err + return + } + + // no such doc + if back == nil { + return + } + // build sorted list of term keys + keys := make(keyset, 0) + for _, entry := range back.termsEntries { + for i := range entry.Terms { + tfr := NewTermFrequencyRow([]byte(entry.Terms[i]), uint16(*entry.Field), idBytes, 0, 0) + key := tfr.Key() + keys = append(keys, key) + } + } + sort.Sort(keys) + + // first add all the stored rows + storedRowPrefix := NewStoredRow(idBytes, 0, []uint64{}, 'x', []byte{}).ScanPrefixForDoc() + dumpPrefix(i.kvreader, rv, storedRowPrefix) + + // now walk term keys in order and add them as well + if len(keys) > 0 { + it := i.kvreader.RangeIterator(keys[0], nil) + defer func() { + cerr := it.Close() + if cerr != nil { + rv <- cerr + } + }() + + for _, key := range keys { + it.Seek(key) + rkey, rval, valid := it.Current() + if !valid { + break + } + rck := make([]byte, len(rkey)) + copy(rck, key) + rcv := make([]byte, len(rval)) + copy(rcv, rval) + row, err := ParseFromKeyValue(rck, rcv) + if err != nil { + rv <- err + return + } + rv <- row + } + } + }() + + return rv +} diff --git a/index/upsidedown/dump_test.go b/index/upsidedown/dump_test.go new file mode 100644 index 0000000..35e00df --- /dev/null +++ b/index/upsidedown/dump_test.go @@ -0,0 +1,155 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "testing" + "time" + + "github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb" + index "github.com/blevesearch/bleve_index_api" + + "github.com/blevesearch/bleve/v2/document" +) + +func TestDump(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + var expectedCount uint64 + reader, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err := reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + doc := document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), index.IndexField|index.StoreField)) + doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, index.IndexField|index.StoreField)) + dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), time.RFC3339, index.IndexField|index.StoreField) + if err != nil { + t.Error(err) + } + doc.AddField(dateField) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + doc = document.NewDocument("2") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test2"), index.IndexField|index.StoreField)) + doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, index.IndexField|index.StoreField)) + dateField, err = document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), time.RFC3339, index.IndexField|index.StoreField) + if err != nil { + t.Error(err) + } + doc.AddField(dateField) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + fieldsCount := 0 + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + upsideDownReader, ok := reader.(*IndexReader) + if !ok { + t.Fatal("dump is only supported by index type upsidedown") + } + fieldsRows := upsideDownReader.DumpFields() + for range fieldsRows { + fieldsCount++ + } + if fieldsCount != 3 { + t.Errorf("expected 3 fields, got %d", fieldsCount) + } + + // 1 text term + // 16 numeric terms + // 16 date terms + // 3 stored fields + expectedDocRowCount := int(1 + (2 * (64 / document.DefaultPrecisionStep)) + 3) + docRowCount := 0 + docRows := upsideDownReader.DumpDoc("1") + for range docRows { + docRowCount++ + } + if docRowCount != expectedDocRowCount { + t.Errorf("expected %d rows for document, got %d", expectedDocRowCount, docRowCount) + } + + docRowCount = 0 + docRows = upsideDownReader.DumpDoc("2") + for range docRows { + docRowCount++ + } + if docRowCount != expectedDocRowCount { + t.Errorf("expected %d rows for document, got %d", expectedDocRowCount, docRowCount) + } + + // 1 version + // fieldsCount field rows + // 2 docs * expectedDocRowCount + // 2 back index rows + // 2 text term row count (2 different text terms) + // 16 numeric term row counts (shared for both docs, same numeric value) + // 16 date term row counts (shared for both docs, same date value) + expectedAllRowCount := int(1 + fieldsCount + (2 * expectedDocRowCount) + 2 + 2 + int((2 * (64 / document.DefaultPrecisionStep)))) + allRowCount := 0 + allRows := upsideDownReader.DumpAll() + for range allRows { + allRowCount++ + } + if allRowCount != expectedAllRowCount { + t.Errorf("expected %d rows for all, got %d", expectedAllRowCount, allRowCount) + } + + err = reader.Close() + if err != nil { + t.Fatal(err) + } +} diff --git a/index/upsidedown/field_cache.go b/index/upsidedown/field_cache.go new file mode 100644 index 0000000..1f68b71 --- /dev/null +++ b/index/upsidedown/field_cache.go @@ -0,0 +1,88 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "sync" +) + +type FieldCache struct { + fieldIndexes map[string]uint16 + indexFields []string + lastFieldIndex int + mutex sync.RWMutex +} + +func NewFieldCache() *FieldCache { + return &FieldCache{ + fieldIndexes: make(map[string]uint16), + lastFieldIndex: -1, + } +} + +func (f *FieldCache) AddExisting(field string, index uint16) { + f.mutex.Lock() + f.addLOCKED(field, index) + f.mutex.Unlock() +} + +func (f *FieldCache) addLOCKED(field string, index uint16) uint16 { + f.fieldIndexes[field] = index + if len(f.indexFields) < int(index)+1 { + prevIndexFields := f.indexFields + f.indexFields = make([]string, int(index)+16) + copy(f.indexFields, prevIndexFields) + } + f.indexFields[int(index)] = field + if int(index) > f.lastFieldIndex { + f.lastFieldIndex = int(index) + } + return index +} + +// FieldNamed returns the index of the field, and whether or not it existed +// before this call. if createIfMissing is true, and new field index is assigned +// but the second return value will still be false +func (f *FieldCache) FieldNamed(field string, createIfMissing bool) (uint16, bool) { + f.mutex.RLock() + if index, ok := f.fieldIndexes[field]; ok { + f.mutex.RUnlock() + return index, true + } else if !createIfMissing { + f.mutex.RUnlock() + return 0, false + } + // trade read lock for write lock + f.mutex.RUnlock() + f.mutex.Lock() + // need to check again with write lock + if index, ok := f.fieldIndexes[field]; ok { + f.mutex.Unlock() + return index, true + } + // assign next field id + index := f.addLOCKED(field, uint16(f.lastFieldIndex+1)) + f.mutex.Unlock() + return index, false +} + +func (f *FieldCache) FieldIndexed(index uint16) (field string) { + f.mutex.RLock() + if int(index) < len(f.indexFields) { + field = f.indexFields[int(index)] + } + f.mutex.RUnlock() + return field +} diff --git a/index/upsidedown/field_dict.go b/index/upsidedown/field_dict.go new file mode 100644 index 0000000..c990fd4 --- /dev/null +++ b/index/upsidedown/field_dict.go @@ -0,0 +1,86 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "fmt" + + index "github.com/blevesearch/bleve_index_api" + store "github.com/blevesearch/upsidedown_store_api" +) + +type UpsideDownCouchFieldDict struct { + indexReader *IndexReader + iterator store.KVIterator + dictRow *DictionaryRow + dictEntry *index.DictEntry + field uint16 +} + +func newUpsideDownCouchFieldDict(indexReader *IndexReader, field uint16, startTerm, endTerm []byte) (*UpsideDownCouchFieldDict, error) { + + startKey := NewDictionaryRow(startTerm, field, 0).Key() + if endTerm == nil { + endTerm = []byte{ByteSeparator} + } else { + endTerm = incrementBytes(endTerm) + } + endKey := NewDictionaryRow(endTerm, field, 0).Key() + + it := indexReader.kvreader.RangeIterator(startKey, endKey) + + return &UpsideDownCouchFieldDict{ + indexReader: indexReader, + iterator: it, + dictRow: &DictionaryRow{}, // Pre-alloced, reused row. + dictEntry: &index.DictEntry{}, // Pre-alloced, reused entry. + field: field, + }, nil + +} + +func (r *UpsideDownCouchFieldDict) BytesRead() uint64 { + return 0 +} + +func (r *UpsideDownCouchFieldDict) Next() (*index.DictEntry, error) { + key, val, valid := r.iterator.Current() + if !valid { + return nil, nil + } + + err := r.dictRow.parseDictionaryK(key) + if err != nil { + return nil, fmt.Errorf("unexpected error parsing dictionary row key: %v", err) + } + err = r.dictRow.parseDictionaryV(val) + if err != nil { + return nil, fmt.Errorf("unexpected error parsing dictionary row val: %v", err) + } + r.dictEntry.Term = string(r.dictRow.term) + r.dictEntry.Count = r.dictRow.count + // advance the iterator to the next term + r.iterator.Next() + return r.dictEntry, nil + +} + +func (r *UpsideDownCouchFieldDict) Cardinality() int { + return 0 +} + +func (r *UpsideDownCouchFieldDict) Close() error { + return r.iterator.Close() +} diff --git a/index/upsidedown/field_dict_test.go b/index/upsidedown/field_dict_test.go new file mode 100644 index 0000000..d796d7e --- /dev/null +++ b/index/upsidedown/field_dict_test.go @@ -0,0 +1,183 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb" + index "github.com/blevesearch/bleve_index_api" +) + +func TestIndexFieldDict(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + doc = document.NewDocument("2") + doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test test test"), testAnalyzer)) + doc.AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("eat more rice"), index.IndexField|index.IncludeTermVectors, testAnalyzer)) + doc.AddField(document.NewTextFieldCustom("prefix", []uint64{}, []byte("bob cat cats catting dog doggy zoo"), index.IndexField|index.IncludeTermVectors, testAnalyzer)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + dict, err := indexReader.FieldDict("name") + if err != nil { + t.Errorf("error creating reader: %v", err) + } + defer func() { + err := dict.Close() + if err != nil { + t.Fatal(err) + } + }() + + termCount := 0 + curr, err := dict.Next() + for err == nil && curr != nil { + termCount++ + if curr.Term != "test" { + t.Errorf("expected term to be 'test', got '%s'", curr.Term) + } + curr, err = dict.Next() + } + if termCount != 1 { + t.Errorf("expected 1 term for this field, got %d", termCount) + } + + dict2, err := indexReader.FieldDict("desc") + if err != nil { + t.Errorf("error creating reader: %v", err) + } + defer func() { + err := dict2.Close() + if err != nil { + t.Fatal(err) + } + }() + + termCount = 0 + terms := make([]string, 0) + curr, err = dict2.Next() + for err == nil && curr != nil { + termCount++ + terms = append(terms, curr.Term) + curr, err = dict2.Next() + } + if termCount != 3 { + t.Errorf("expected 3 term for this field, got %d", termCount) + } + expectedTerms := []string{"eat", "more", "rice"} + if !reflect.DeepEqual(expectedTerms, terms) { + t.Errorf("expected %#v, got %#v", expectedTerms, terms) + } + + // test start and end range + dict3, err := indexReader.FieldDictRange("desc", []byte("fun"), []byte("nice")) + if err != nil { + t.Errorf("error creating reader: %v", err) + } + defer func() { + err := dict3.Close() + if err != nil { + t.Fatal(err) + } + }() + + termCount = 0 + terms = make([]string, 0) + curr, err = dict3.Next() + for err == nil && curr != nil { + termCount++ + terms = append(terms, curr.Term) + curr, err = dict3.Next() + } + if termCount != 1 { + t.Errorf("expected 1 term for this field, got %d", termCount) + } + expectedTerms = []string{"more"} + if !reflect.DeepEqual(expectedTerms, terms) { + t.Errorf("expected %#v, got %#v", expectedTerms, terms) + } + + // test use case for prefix + dict4, err := indexReader.FieldDictPrefix("prefix", []byte("cat")) + if err != nil { + t.Errorf("error creating reader: %v", err) + } + defer func() { + err := dict4.Close() + if err != nil { + t.Fatal(err) + } + }() + + termCount = 0 + terms = make([]string, 0) + curr, err = dict4.Next() + for err == nil && curr != nil { + termCount++ + terms = append(terms, curr.Term) + curr, err = dict4.Next() + } + if termCount != 3 { + t.Errorf("expected 3 term for this field, got %d", termCount) + } + expectedTerms = []string{"cat", "cats", "catting"} + if !reflect.DeepEqual(expectedTerms, terms) { + t.Errorf("expected %#v, got %#v", expectedTerms, terms) + } +} diff --git a/index/upsidedown/index_reader.go b/index/upsidedown/index_reader.go new file mode 100644 index 0000000..0b91739 --- /dev/null +++ b/index/upsidedown/index_reader.go @@ -0,0 +1,228 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "context" + "reflect" + + "github.com/blevesearch/bleve/v2/document" + index "github.com/blevesearch/bleve_index_api" + store "github.com/blevesearch/upsidedown_store_api" +) + +var reflectStaticSizeIndexReader int + +func init() { + var ir IndexReader + reflectStaticSizeIndexReader = int(reflect.TypeOf(ir).Size()) +} + +type IndexReader struct { + index *UpsideDownCouch + kvreader store.KVReader + docCount uint64 +} + +func (i *IndexReader) TermFieldReader(ctx context.Context, term []byte, fieldName string, includeFreq, includeNorm, includeTermVectors bool) (index.TermFieldReader, error) { + fieldIndex, fieldExists := i.index.fieldCache.FieldNamed(fieldName, false) + if fieldExists { + return newUpsideDownCouchTermFieldReader(i, term, uint16(fieldIndex), includeFreq, includeNorm, includeTermVectors) + } + return newUpsideDownCouchTermFieldReader(i, []byte{ByteSeparator}, ^uint16(0), includeFreq, includeNorm, includeTermVectors) +} + +func (i *IndexReader) FieldDict(fieldName string) (index.FieldDict, error) { + return i.FieldDictRange(fieldName, nil, nil) +} + +func (i *IndexReader) FieldDictRange(fieldName string, startTerm []byte, endTerm []byte) (index.FieldDict, error) { + fieldIndex, fieldExists := i.index.fieldCache.FieldNamed(fieldName, false) + if fieldExists { + return newUpsideDownCouchFieldDict(i, uint16(fieldIndex), startTerm, endTerm) + } + return newUpsideDownCouchFieldDict(i, ^uint16(0), []byte{ByteSeparator}, []byte{}) +} + +func (i *IndexReader) FieldDictPrefix(fieldName string, termPrefix []byte) (index.FieldDict, error) { + return i.FieldDictRange(fieldName, termPrefix, termPrefix) +} + +func (i *IndexReader) DocIDReaderAll() (index.DocIDReader, error) { + return newUpsideDownCouchDocIDReader(i) +} + +func (i *IndexReader) DocIDReaderOnly(ids []string) (index.DocIDReader, error) { + return newUpsideDownCouchDocIDReaderOnly(i, ids) +} + +func (i *IndexReader) Document(id string) (doc index.Document, err error) { + // first hit the back index to confirm doc exists + var backIndexRow *BackIndexRow + backIndexRow, err = backIndexRowForDoc(i.kvreader, []byte(id)) + if err != nil { + return + } + if backIndexRow == nil { + return + } + rvd := document.NewDocument(id) + storedRow := NewStoredRow([]byte(id), 0, []uint64{}, 'x', nil) + storedRowScanPrefix := storedRow.ScanPrefixForDoc() + it := i.kvreader.PrefixIterator(storedRowScanPrefix) + defer func() { + if cerr := it.Close(); err == nil && cerr != nil { + err = cerr + } + }() + key, val, valid := it.Current() + for valid { + safeVal := make([]byte, len(val)) + copy(safeVal, val) + var row *StoredRow + row, err = NewStoredRowKV(key, safeVal) + if err != nil { + return nil, err + } + if row != nil { + fieldName := i.index.fieldCache.FieldIndexed(row.field) + field := decodeFieldType(row.typ, fieldName, row.arrayPositions, row.value) + if field != nil { + rvd.AddField(field) + } + } + + it.Next() + key, val, valid = it.Current() + } + return rvd, nil +} + +func (i *IndexReader) documentVisitFieldTerms(id index.IndexInternalID, fields []string, visitor index.DocValueVisitor) error { + fieldsMap := make(map[uint16]string, len(fields)) + for _, f := range fields { + id, ok := i.index.fieldCache.FieldNamed(f, false) + if ok { + fieldsMap[id] = f + } + } + + tempRow := BackIndexRow{ + doc: id, + } + + keyBuf := GetRowBuffer() + if tempRow.KeySize() > len(keyBuf.buf) { + keyBuf.buf = make([]byte, 2*tempRow.KeySize()) + } + defer PutRowBuffer(keyBuf) + keySize, err := tempRow.KeyTo(keyBuf.buf) + if err != nil { + return err + } + + value, err := i.kvreader.Get(keyBuf.buf[:keySize]) + if err != nil { + return err + } + if value == nil { + return nil + } + + return visitBackIndexRow(value, func(field uint32, term []byte) { + if field, ok := fieldsMap[uint16(field)]; ok { + visitor(field, term) + } + }) +} + +func (i *IndexReader) Fields() (fields []string, err error) { + fields = make([]string, 0) + it := i.kvreader.PrefixIterator([]byte{'f'}) + defer func() { + if cerr := it.Close(); err == nil && cerr != nil { + err = cerr + } + }() + key, val, valid := it.Current() + for valid { + var row UpsideDownCouchRow + row, err = ParseFromKeyValue(key, val) + if err != nil { + fields = nil + return + } + if row != nil { + fieldRow, ok := row.(*FieldRow) + if ok { + fields = append(fields, fieldRow.name) + } + } + + it.Next() + key, val, valid = it.Current() + } + return +} + +func (i *IndexReader) GetInternal(key []byte) ([]byte, error) { + internalRow := NewInternalRow(key, nil) + return i.kvreader.Get(internalRow.Key()) +} + +func (i *IndexReader) DocCount() (uint64, error) { + return i.docCount, nil +} + +func (i *IndexReader) Close() error { + return i.kvreader.Close() +} + +func (i *IndexReader) ExternalID(id index.IndexInternalID) (string, error) { + return string(id), nil +} + +func (i *IndexReader) InternalID(id string) (index.IndexInternalID, error) { + return index.IndexInternalID(id), nil +} + +func incrementBytes(in []byte) []byte { + rv := make([]byte, len(in)) + copy(rv, in) + for i := len(rv) - 1; i >= 0; i-- { + rv[i] = rv[i] + 1 + if rv[i] != 0 { + // didn't overflow, so stop + break + } + } + return rv +} + +func (i *IndexReader) DocValueReader(fields []string) (index.DocValueReader, error) { + return &DocValueReader{i: i, fields: fields}, nil +} + +type DocValueReader struct { + i *IndexReader + fields []string +} + +func (dvr *DocValueReader) VisitDocValues(id index.IndexInternalID, + visitor index.DocValueVisitor) error { + return dvr.i.documentVisitFieldTerms(id, dvr.fields, visitor) +} + +func (dvr *DocValueReader) BytesRead() uint64 { return 0 } diff --git a/index/upsidedown/protoc-README.md b/index/upsidedown/protoc-README.md new file mode 100644 index 0000000..367a6b0 --- /dev/null +++ b/index/upsidedown/protoc-README.md @@ -0,0 +1,13 @@ +## Instructions for generating new go stubs using upsidedown.proto + +1. Download latest of protoc-gen-go +``` +go install google.golang.org/protobuf/cmd/protoc-gen-go@latest +``` + +2. To generate `upsidedown.pb.go` using upsdidedown.proto: +``` +protoc --go_out=. --go_opt=Mindex/upsidedown/upsidedown.proto=index/upsidedown/ index/upsidedown/upsidedown.proto +``` + +3. Manually add back Size and MarshalTo methods for BackIndexRowValue, BackIndexTermsEntry, BackIndexStoreEntry to support upside_down. diff --git a/index/upsidedown/reader.go b/index/upsidedown/reader.go new file mode 100644 index 0000000..6b68fe7 --- /dev/null +++ b/index/upsidedown/reader.go @@ -0,0 +1,376 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "bytes" + "reflect" + "sort" + "sync/atomic" + + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/upsidedown_store_api" +) + +var reflectStaticSizeUpsideDownCouchTermFieldReader int +var reflectStaticSizeUpsideDownCouchDocIDReader int + +func init() { + var tfr UpsideDownCouchTermFieldReader + reflectStaticSizeUpsideDownCouchTermFieldReader = + int(reflect.TypeOf(tfr).Size()) + var cdr UpsideDownCouchDocIDReader + reflectStaticSizeUpsideDownCouchDocIDReader = + int(reflect.TypeOf(cdr).Size()) +} + +type UpsideDownCouchTermFieldReader struct { + count uint64 + indexReader *IndexReader + iterator store.KVIterator + term []byte + tfrNext *TermFrequencyRow + tfrPrealloc TermFrequencyRow + keyBuf []byte + field uint16 + includeTermVectors bool +} + +func (r *UpsideDownCouchTermFieldReader) Size() int { + sizeInBytes := reflectStaticSizeUpsideDownCouchTermFieldReader + size.SizeOfPtr + + len(r.term) + + r.tfrPrealloc.Size() + + len(r.keyBuf) + + if r.tfrNext != nil { + sizeInBytes += r.tfrNext.Size() + } + + return sizeInBytes +} + +func newUpsideDownCouchTermFieldReader(indexReader *IndexReader, term []byte, field uint16, includeFreq, includeNorm, includeTermVectors bool) (*UpsideDownCouchTermFieldReader, error) { + bufNeeded := termFrequencyRowKeySize(term, nil) + if bufNeeded < dictionaryRowKeySize(term) { + bufNeeded = dictionaryRowKeySize(term) + } + buf := make([]byte, bufNeeded) + + bufUsed := dictionaryRowKeyTo(buf, field, term) + val, err := indexReader.kvreader.Get(buf[:bufUsed]) + if err != nil { + return nil, err + } + if val == nil { + atomic.AddUint64(&indexReader.index.stats.termSearchersStarted, uint64(1)) + rv := &UpsideDownCouchTermFieldReader{ + count: 0, + term: term, + field: field, + includeTermVectors: includeTermVectors, + } + rv.tfrNext = &rv.tfrPrealloc + return rv, nil + } + + count, err := dictionaryRowParseV(val) + if err != nil { + return nil, err + } + + bufUsed = termFrequencyRowKeyTo(buf, field, term, nil) + it := indexReader.kvreader.PrefixIterator(buf[:bufUsed]) + + atomic.AddUint64(&indexReader.index.stats.termSearchersStarted, uint64(1)) + return &UpsideDownCouchTermFieldReader{ + indexReader: indexReader, + iterator: it, + count: count, + term: term, + field: field, + includeTermVectors: includeTermVectors, + }, nil +} + +func (r *UpsideDownCouchTermFieldReader) Count() uint64 { + return r.count +} + +func (r *UpsideDownCouchTermFieldReader) Next(preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) { + if r.iterator != nil { + // We treat tfrNext also like an initialization flag, which + // tells us whether we need to invoke the underlying + // iterator.Next(). The first time, don't call iterator.Next(). + if r.tfrNext != nil { + r.iterator.Next() + } else { + r.tfrNext = &r.tfrPrealloc + } + key, val, valid := r.iterator.Current() + if valid { + tfr := r.tfrNext + err := tfr.parseKDoc(key, r.term) + if err != nil { + return nil, err + } + err = tfr.parseV(val, r.includeTermVectors) + if err != nil { + return nil, err + } + rv := preAlloced + if rv == nil { + rv = &index.TermFieldDoc{} + } + rv.ID = append(rv.ID, tfr.doc...) + rv.Freq = tfr.freq + rv.Norm = float64(tfr.norm) + if tfr.vectors != nil { + rv.Vectors = r.indexReader.index.termFieldVectorsFromTermVectors(tfr.vectors) + } + return rv, nil + } + } + return nil, nil +} + +func (r *UpsideDownCouchTermFieldReader) Advance(docID index.IndexInternalID, preAlloced *index.TermFieldDoc) (rv *index.TermFieldDoc, err error) { + if r.iterator != nil { + if r.tfrNext == nil { + r.tfrNext = &TermFrequencyRow{} + } + tfr := InitTermFrequencyRow(r.tfrNext, r.term, r.field, docID, 0, 0) + r.keyBuf, err = tfr.KeyAppendTo(r.keyBuf[:0]) + if err != nil { + return nil, err + } + r.iterator.Seek(r.keyBuf) + key, val, valid := r.iterator.Current() + if valid { + err := tfr.parseKDoc(key, r.term) + if err != nil { + return nil, err + } + err = tfr.parseV(val, r.includeTermVectors) + if err != nil { + return nil, err + } + rv = preAlloced + if rv == nil { + rv = &index.TermFieldDoc{} + } + rv.ID = append(rv.ID, tfr.doc...) + rv.Freq = tfr.freq + rv.Norm = float64(tfr.norm) + if tfr.vectors != nil { + rv.Vectors = r.indexReader.index.termFieldVectorsFromTermVectors(tfr.vectors) + } + return rv, nil + } + } + return nil, nil +} + +func (r *UpsideDownCouchTermFieldReader) Close() error { + if r.indexReader != nil { + atomic.AddUint64(&r.indexReader.index.stats.termSearchersFinished, uint64(1)) + } + if r.iterator != nil { + return r.iterator.Close() + } + return nil +} + +type UpsideDownCouchDocIDReader struct { + indexReader *IndexReader + iterator store.KVIterator + only []string + onlyPos int + onlyMode bool +} + +func (r *UpsideDownCouchDocIDReader) Size() int { + sizeInBytes := reflectStaticSizeUpsideDownCouchDocIDReader + + reflectStaticSizeIndexReader + size.SizeOfPtr + + for _, entry := range r.only { + sizeInBytes += size.SizeOfString + len(entry) + } + + return sizeInBytes +} + +func newUpsideDownCouchDocIDReader(indexReader *IndexReader) (*UpsideDownCouchDocIDReader, error) { + startBytes := []byte{0x0} + endBytes := []byte{0xff} + + bisr := NewBackIndexRow(startBytes, nil, nil) + bier := NewBackIndexRow(endBytes, nil, nil) + it := indexReader.kvreader.RangeIterator(bisr.Key(), bier.Key()) + + return &UpsideDownCouchDocIDReader{ + indexReader: indexReader, + iterator: it, + }, nil +} + +func newUpsideDownCouchDocIDReaderOnly(indexReader *IndexReader, ids []string) (*UpsideDownCouchDocIDReader, error) { + // we don't actually own the list of ids, so if before we sort we must copy + idsCopy := make([]string, len(ids)) + copy(idsCopy, ids) + // ensure ids are sorted + sort.Strings(idsCopy) + startBytes := []byte{0x0} + if len(idsCopy) > 0 { + startBytes = []byte(idsCopy[0]) + } + endBytes := []byte{0xff} + if len(idsCopy) > 0 { + endBytes = incrementBytes([]byte(idsCopy[len(idsCopy)-1])) + } + bisr := NewBackIndexRow(startBytes, nil, nil) + bier := NewBackIndexRow(endBytes, nil, nil) + it := indexReader.kvreader.RangeIterator(bisr.Key(), bier.Key()) + + return &UpsideDownCouchDocIDReader{ + indexReader: indexReader, + iterator: it, + only: idsCopy, + onlyMode: true, + }, nil +} + +func (r *UpsideDownCouchDocIDReader) Next() (index.IndexInternalID, error) { + key, val, valid := r.iterator.Current() + + if r.onlyMode { + var rv index.IndexInternalID + for valid && r.onlyPos < len(r.only) { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + if !bytes.Equal(br.doc, []byte(r.only[r.onlyPos])) { + ok := r.nextOnly() + if !ok { + return nil, nil + } + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + key, val, valid = r.iterator.Current() + continue + } else { + rv = append([]byte(nil), br.doc...) + break + } + } + if valid && r.onlyPos < len(r.only) { + ok := r.nextOnly() + if ok { + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + } + return rv, nil + } + + } else { + if valid { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + rv := append([]byte(nil), br.doc...) + r.iterator.Next() + return rv, nil + } + } + return nil, nil +} + +func (r *UpsideDownCouchDocIDReader) Advance(docID index.IndexInternalID) (index.IndexInternalID, error) { + + if r.onlyMode { + r.onlyPos = sort.SearchStrings(r.only, string(docID)) + if r.onlyPos >= len(r.only) { + // advanced to key after our last only key + return nil, nil + } + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + key, val, valid := r.iterator.Current() + + var rv index.IndexInternalID + for valid && r.onlyPos < len(r.only) { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + if !bytes.Equal(br.doc, []byte(r.only[r.onlyPos])) { + // the only key we seek'd to didn't exist + // now look for the closest key that did exist in only + r.onlyPos = sort.SearchStrings(r.only, string(br.doc)) + if r.onlyPos >= len(r.only) { + // advanced to key after our last only key + return nil, nil + } + // now seek to this new only key + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + key, val, valid = r.iterator.Current() + continue + } else { + rv = append([]byte(nil), br.doc...) + break + } + } + if valid && r.onlyPos < len(r.only) { + ok := r.nextOnly() + if ok { + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + } + return rv, nil + } + } else { + bir := NewBackIndexRow(docID, nil, nil) + r.iterator.Seek(bir.Key()) + key, val, valid := r.iterator.Current() + if valid { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + rv := append([]byte(nil), br.doc...) + r.iterator.Next() + return rv, nil + } + } + return nil, nil +} + +func (r *UpsideDownCouchDocIDReader) Close() error { + return r.iterator.Close() +} + +// move the r.only pos forward one, skipping duplicates +// return true if there is more data, or false if we got to the end of the list +func (r *UpsideDownCouchDocIDReader) nextOnly() bool { + + // advance 1 position, until we see a different key + // it's already sorted, so this skips duplicates + start := r.onlyPos + r.onlyPos++ + for r.onlyPos < len(r.only) && r.only[r.onlyPos] == r.only[start] { + start = r.onlyPos + r.onlyPos++ + } + // indicate if we got to the end of the list + return r.onlyPos < len(r.only) +} diff --git a/index/upsidedown/reader_test.go b/index/upsidedown/reader_test.go new file mode 100644 index 0000000..cec2eca --- /dev/null +++ b/index/upsidedown/reader_test.go @@ -0,0 +1,548 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "context" + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb" + index "github.com/blevesearch/bleve_index_api" +) + +func TestIndexReader(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + var expectedCount uint64 + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + doc = document.NewDocument("2") + doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test test test"), testAnalyzer)) + doc.AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("eat more rice"), index.IndexField|index.IncludeTermVectors, testAnalyzer)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + // first look for a term that doesn't exist + reader, err := indexReader.TermFieldReader(context.TODO(), []byte("nope"), "name", true, true, true) + if err != nil { + t.Errorf("Error accessing term field reader: %v", err) + } + count := reader.Count() + if count != 0 { + t.Errorf("Expected doc count to be: %d got: %d", 0, count) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + reader, err = indexReader.TermFieldReader(context.TODO(), []byte("test"), "name", true, true, true) + if err != nil { + t.Errorf("Error accessing term field reader: %v", err) + } + + count = reader.Count() + if count != expectedCount { + t.Errorf("Expected doc count to be: %d got: %d", expectedCount, count) + } + + var match *index.TermFieldDoc + var actualCount uint64 + match, err = reader.Next(nil) + for err == nil && match != nil { + match, err = reader.Next(nil) + if err != nil { + t.Errorf("unexpected error reading next") + } + actualCount++ + } + if actualCount != count { + t.Errorf("count was 2, but only saw %d", actualCount) + } + + expectedMatch := &index.TermFieldDoc{ + ID: index.IndexInternalID("2"), + Freq: 1, + Norm: 0.5773502588272095, + Vectors: []*index.TermFieldVector{ + { + Field: "desc", + Pos: 3, + Start: 9, + End: 13, + }, + }, + } + tfr, err := indexReader.TermFieldReader(context.TODO(), []byte("rice"), "desc", true, true, true) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + match, err = tfr.Next(nil) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if !reflect.DeepEqual(expectedMatch, match) { + t.Errorf("got %#v, expected %#v", match, expectedMatch) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + // now test usage of advance + reader, err = indexReader.TermFieldReader(context.TODO(), []byte("test"), "name", true, true, true) + if err != nil { + t.Errorf("Error accessing term field reader: %v", err) + } + + match, err = reader.Advance(index.IndexInternalID("2"), nil) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if match == nil { + t.Fatalf("Expected match, got nil") + } + if !match.ID.Equals(index.IndexInternalID("2")) { + t.Errorf("Expected ID '2', got '%s'", match.ID) + } + match, err = reader.Advance(index.IndexInternalID("3"), nil) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if match != nil { + t.Errorf("expected nil, got %v", match) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + // now test creating a reader for a field that doesn't exist + reader, err = indexReader.TermFieldReader(context.TODO(), []byte("water"), "doesnotexist", true, true, true) + if err != nil { + t.Errorf("Error accessing term field reader: %v", err) + } + count = reader.Count() + if count != 0 { + t.Errorf("expected count 0 for reader of non-existent field") + } + match, err = reader.Next(nil) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if match != nil { + t.Errorf("expected nil, got %v", match) + } + match, err = reader.Advance(index.IndexInternalID("anywhere"), nil) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if match != nil { + t.Errorf("expected nil, got %v", match) + } +} + +func TestIndexDocIdReader(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + var expectedCount uint64 + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + doc = document.NewDocument("2") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test test test"))) + doc.AddField(document.NewTextFieldWithIndexingOptions("desc", []uint64{}, []byte("eat more rice"), index.IndexField|index.IncludeTermVectors)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Error(err) + } + }() + + // first get all doc ids + reader, err := indexReader.DocIDReaderAll() + if err != nil { + t.Errorf("Error accessing doc id reader: %v", err) + } + defer func() { + err := reader.Close() + if err != nil { + t.Fatal(err) + } + }() + + id, err := reader.Next() + if err != nil { + t.Fatal(err) + } + + count := uint64(0) + for id != nil { + count++ + id, err = reader.Next() + if err != nil { + t.Fatal(err) + } + } + if count != expectedCount { + t.Errorf("expected %d, got %d", expectedCount, count) + } + + // try it again, but jump to the second doc this time + reader2, err := indexReader.DocIDReaderAll() + if err != nil { + t.Errorf("Error accessing doc id reader: %v", err) + } + defer func() { + err := reader2.Close() + if err != nil { + t.Error(err) + } + }() + + id, err = reader2.Advance(index.IndexInternalID("2")) + if err != nil { + t.Error(err) + } + if !id.Equals(index.IndexInternalID("2")) { + t.Errorf("expected to find id '2', got '%s'", id) + } + + id, err = reader2.Advance(index.IndexInternalID("3")) + if err != nil { + t.Error(err) + } + if id != nil { + t.Errorf("expected to find id '', got '%s'", id) + } +} + +func TestCrashBadBackIndexRow(t *testing.T) { + br, err := NewBackIndexRowKV([]byte{byte('b'), byte('a'), ByteSeparator}, []byte{}) + if err != nil { + t.Fatal(err) + } + if string(br.doc) != "a" { + t.Fatal(err) + } +} + +func TestIndexDocIdOnlyReader(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + doc := document.NewDocument("1") + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + doc = document.NewDocument("3") + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + doc = document.NewDocument("5") + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + doc = document.NewDocument("7") + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + doc = document.NewDocument("9") + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Error(err) + } + }() + + onlyIds := []string{"1", "5", "9"} + reader, err := indexReader.DocIDReaderOnly(onlyIds) + if err != nil { + t.Errorf("Error accessing doc id reader: %v", err) + } + defer func() { + err := reader.Close() + if err != nil { + t.Fatal(err) + } + }() + + id, err := reader.Next() + if err != nil { + t.Fatal(err) + } + + count := uint64(0) + for id != nil { + count++ + id, err = reader.Next() + if err != nil { + t.Fatal(err) + } + } + if count != 3 { + t.Errorf("expected 3, got %d", count) + } + + // try it again, but jump + reader2, err := indexReader.DocIDReaderOnly(onlyIds) + if err != nil { + t.Errorf("Error accessing doc id reader: %v", err) + } + defer func() { + err := reader2.Close() + if err != nil { + t.Error(err) + } + }() + + id, err = reader2.Advance(index.IndexInternalID("5")) + if err != nil { + t.Error(err) + } + if !id.Equals(index.IndexInternalID("5")) { + t.Errorf("expected to find id '5', got '%s'", id) + } + + id, err = reader2.Advance(index.IndexInternalID("a")) + if err != nil { + t.Error(err) + } + if id != nil { + t.Errorf("expected to find id '', got '%s'", id) + } + + // some keys aren't actually there + onlyIds = []string{"0", "2", "4", "5", "6", "8", "a"} + reader3, err := indexReader.DocIDReaderOnly(onlyIds) + if err != nil { + t.Errorf("Error accessing doc id reader: %v", err) + } + defer func() { + err := reader3.Close() + if err != nil { + t.Error(err) + } + }() + + id, err = reader3.Next() + if err != nil { + t.Fatal(err) + } + + count = uint64(0) + for id != nil { + count++ + id, err = reader3.Next() + if err != nil { + t.Fatal(err) + } + } + if count != 1 { + t.Errorf("expected 1, got %d", count) + } + + // mix advance and next + onlyIds = []string{"0", "1", "3", "5", "6", "9"} + reader4, err := indexReader.DocIDReaderOnly(onlyIds) + if err != nil { + t.Errorf("Error accessing doc id reader: %v", err) + } + defer func() { + err := reader4.Close() + if err != nil { + t.Error(err) + } + }() + + // first key is "1" + id, err = reader4.Next() + if err != nil { + t.Error(err) + } + if !id.Equals(index.IndexInternalID("1")) { + t.Errorf("expected to find id '1', got '%s'", id) + } + + // advancing to key we dont have gives next + id, err = reader4.Advance(index.IndexInternalID("2")) + if err != nil { + t.Error(err) + } + if !id.Equals(index.IndexInternalID("3")) { + t.Errorf("expected to find id '3', got '%s'", id) + } + + // next after advance works + id, err = reader4.Next() + if err != nil { + t.Error(err) + } + if !id.Equals(index.IndexInternalID("5")) { + t.Errorf("expected to find id '5', got '%s'", id) + } + + // advancing to key we do have works + id, err = reader4.Advance(index.IndexInternalID("9")) + if err != nil { + t.Error(err) + } + if !id.Equals(index.IndexInternalID("9")) { + t.Errorf("expected to find id '9', got '%s'", id) + } + + // advance backwards at end + id, err = reader4.Advance(index.IndexInternalID("4")) + if err != nil { + t.Error(err) + } + if !id.Equals(index.IndexInternalID("5")) { + t.Errorf("expected to find id '5', got '%s'", id) + } + + // next after advance works + id, err = reader4.Next() + if err != nil { + t.Error(err) + } + if !id.Equals(index.IndexInternalID("9")) { + t.Errorf("expected to find id '9', got '%s'", id) + } + + // advance backwards to key that exists, but not in only set + id, err = reader4.Advance(index.IndexInternalID("7")) + if err != nil { + t.Error(err) + } + if !id.Equals(index.IndexInternalID("9")) { + t.Errorf("expected to find id '9', got '%s'", id) + } +} diff --git a/index/upsidedown/row.go b/index/upsidedown/row.go new file mode 100644 index 0000000..b6d2c3d --- /dev/null +++ b/index/upsidedown/row.go @@ -0,0 +1,1144 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "math" + "reflect" + + "github.com/blevesearch/bleve/v2/size" + "google.golang.org/protobuf/proto" +) + +var ( + reflectStaticSizeTermFrequencyRow int + reflectStaticSizeTermVector int +) + +func init() { + var tfr TermFrequencyRow + reflectStaticSizeTermFrequencyRow = int(reflect.TypeOf(tfr).Size()) + var tv TermVector + reflectStaticSizeTermVector = int(reflect.TypeOf(tv).Size()) +} + +const ByteSeparator byte = 0xff + +type UpsideDownCouchRowStream chan UpsideDownCouchRow + +type UpsideDownCouchRow interface { + KeySize() int + KeyTo([]byte) (int, error) + Key() []byte + Value() []byte + ValueSize() int + ValueTo([]byte) (int, error) +} + +func ParseFromKeyValue(key, value []byte) (UpsideDownCouchRow, error) { + if len(key) > 0 { + switch key[0] { + case 'v': + return NewVersionRowKV(key, value) + case 'f': + return NewFieldRowKV(key, value) + case 'd': + return NewDictionaryRowKV(key, value) + case 't': + return NewTermFrequencyRowKV(key, value) + case 'b': + return NewBackIndexRowKV(key, value) + case 's': + return NewStoredRowKV(key, value) + case 'i': + return NewInternalRowKV(key, value) + } + return nil, fmt.Errorf("Unknown field type '%s'", string(key[0])) + } + return nil, fmt.Errorf("Invalid empty key") +} + +// VERSION + +type VersionRow struct { + version uint8 +} + +func (v *VersionRow) Key() []byte { + return []byte{'v'} +} + +func (v *VersionRow) KeySize() int { + return 1 +} + +func (v *VersionRow) KeyTo(buf []byte) (int, error) { + buf[0] = 'v' + return 1, nil +} + +func (v *VersionRow) Value() []byte { + return []byte{byte(v.version)} +} + +func (v *VersionRow) ValueSize() int { + return 1 +} + +func (v *VersionRow) ValueTo(buf []byte) (int, error) { + buf[0] = v.version + return 1, nil +} + +func (v *VersionRow) String() string { + return fmt.Sprintf("Version: %d", v.version) +} + +func NewVersionRow(version uint8) *VersionRow { + return &VersionRow{ + version: version, + } +} + +func NewVersionRowKV(key, value []byte) (*VersionRow, error) { + rv := VersionRow{} + buf := bytes.NewBuffer(value) + err := binary.Read(buf, binary.LittleEndian, &rv.version) + if err != nil { + return nil, err + } + return &rv, nil +} + +// INTERNAL STORAGE + +type InternalRow struct { + key []byte + val []byte +} + +func (i *InternalRow) Key() []byte { + buf := make([]byte, i.KeySize()) + size, _ := i.KeyTo(buf) + return buf[:size] +} + +func (i *InternalRow) KeySize() int { + return len(i.key) + 1 +} + +func (i *InternalRow) KeyTo(buf []byte) (int, error) { + buf[0] = 'i' + actual := copy(buf[1:], i.key) + return 1 + actual, nil +} + +func (i *InternalRow) Value() []byte { + return i.val +} + +func (i *InternalRow) ValueSize() int { + return len(i.val) +} + +func (i *InternalRow) ValueTo(buf []byte) (int, error) { + actual := copy(buf, i.val) + return actual, nil +} + +func (i *InternalRow) String() string { + return fmt.Sprintf("InternalStore - Key: %s (% x) Val: %s (% x)", i.key, i.key, i.val, i.val) +} + +func NewInternalRow(key, val []byte) *InternalRow { + return &InternalRow{ + key: key, + val: val, + } +} + +func NewInternalRowKV(key, value []byte) (*InternalRow, error) { + rv := InternalRow{} + rv.key = key[1:] + rv.val = value + return &rv, nil +} + +// FIELD definition + +type FieldRow struct { + index uint16 + name string +} + +func (f *FieldRow) Key() []byte { + buf := make([]byte, f.KeySize()) + size, _ := f.KeyTo(buf) + return buf[:size] +} + +func (f *FieldRow) KeySize() int { + return 3 +} + +func (f *FieldRow) KeyTo(buf []byte) (int, error) { + buf[0] = 'f' + binary.LittleEndian.PutUint16(buf[1:3], f.index) + return 3, nil +} + +func (f *FieldRow) Value() []byte { + return append([]byte(f.name), ByteSeparator) +} + +func (f *FieldRow) ValueSize() int { + return len(f.name) + 1 +} + +func (f *FieldRow) ValueTo(buf []byte) (int, error) { + size := copy(buf, f.name) + buf[size] = ByteSeparator + return size + 1, nil +} + +func (f *FieldRow) String() string { + return fmt.Sprintf("Field: %d Name: %s", f.index, f.name) +} + +func NewFieldRow(index uint16, name string) *FieldRow { + return &FieldRow{ + index: index, + name: name, + } +} + +func NewFieldRowKV(key, value []byte) (*FieldRow, error) { + rv := FieldRow{} + + buf := bytes.NewBuffer(key) + _, err := buf.ReadByte() // type + if err != nil { + return nil, err + } + err = binary.Read(buf, binary.LittleEndian, &rv.index) + if err != nil { + return nil, err + } + + buf = bytes.NewBuffer(value) + rv.name, err = buf.ReadString(ByteSeparator) + if err != nil { + return nil, err + } + rv.name = rv.name[:len(rv.name)-1] // trim off separator byte + + return &rv, nil +} + +// DICTIONARY + +const DictionaryRowMaxValueSize = binary.MaxVarintLen64 + +type DictionaryRow struct { + term []byte + count uint64 + field uint16 +} + +func (dr *DictionaryRow) Key() []byte { + buf := make([]byte, dr.KeySize()) + size, _ := dr.KeyTo(buf) + return buf[:size] +} + +func (dr *DictionaryRow) KeySize() int { + return dictionaryRowKeySize(dr.term) +} + +func dictionaryRowKeySize(term []byte) int { + return len(term) + 3 +} + +func (dr *DictionaryRow) KeyTo(buf []byte) (int, error) { + return dictionaryRowKeyTo(buf, dr.field, dr.term), nil +} + +func dictionaryRowKeyTo(buf []byte, field uint16, term []byte) int { + buf[0] = 'd' + binary.LittleEndian.PutUint16(buf[1:3], field) + size := copy(buf[3:], term) + return size + 3 +} + +func (dr *DictionaryRow) Value() []byte { + buf := make([]byte, dr.ValueSize()) + size, _ := dr.ValueTo(buf) + return buf[:size] +} + +func (dr *DictionaryRow) ValueSize() int { + return DictionaryRowMaxValueSize +} + +func (dr *DictionaryRow) ValueTo(buf []byte) (int, error) { + used := binary.PutUvarint(buf, dr.count) + return used, nil +} + +func (dr *DictionaryRow) String() string { + return fmt.Sprintf("Dictionary Term: `%s` Field: %d Count: %d ", string(dr.term), dr.field, dr.count) +} + +func NewDictionaryRow(term []byte, field uint16, count uint64) *DictionaryRow { + return &DictionaryRow{ + term: term, + field: field, + count: count, + } +} + +func NewDictionaryRowKV(key, value []byte) (*DictionaryRow, error) { + rv, err := NewDictionaryRowK(key) + if err != nil { + return nil, err + } + + err = rv.parseDictionaryV(value) + if err != nil { + return nil, err + } + return rv, nil +} + +func NewDictionaryRowK(key []byte) (*DictionaryRow, error) { + rv := &DictionaryRow{} + err := rv.parseDictionaryK(key) + if err != nil { + return nil, err + } + return rv, nil +} + +func (dr *DictionaryRow) parseDictionaryK(key []byte) error { + dr.field = binary.LittleEndian.Uint16(key[1:3]) + if dr.term != nil { + dr.term = dr.term[:0] + } + dr.term = append(dr.term, key[3:]...) + return nil +} + +func (dr *DictionaryRow) parseDictionaryV(value []byte) error { + count, err := dictionaryRowParseV(value) + if err != nil { + return err + } + dr.count = count + return nil +} + +func dictionaryRowParseV(value []byte) (uint64, error) { + count, nread := binary.Uvarint(value) + if nread <= 0 { + return 0, fmt.Errorf("DictionaryRow parse Uvarint error, nread: %d", nread) + } + return count, nil +} + +// TERM FIELD FREQUENCY + +type TermVector struct { + field uint16 + arrayPositions []uint64 + pos uint64 + start uint64 + end uint64 +} + +func (tv *TermVector) Size() int { + return reflectStaticSizeTermVector + size.SizeOfPtr + + len(tv.arrayPositions)*size.SizeOfUint64 +} + +func (tv *TermVector) String() string { + return fmt.Sprintf("Field: %d Pos: %d Start: %d End %d ArrayPositions: %#v", tv.field, tv.pos, tv.start, tv.end, tv.arrayPositions) +} + +type TermFrequencyRow struct { + term []byte + doc []byte + freq uint64 + vectors []*TermVector + norm float32 + field uint16 +} + +func (tfr *TermFrequencyRow) Size() int { + sizeInBytes := reflectStaticSizeTermFrequencyRow + + len(tfr.term) + + len(tfr.doc) + + for _, entry := range tfr.vectors { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} + +func (tfr *TermFrequencyRow) Term() []byte { + return tfr.term +} + +func (tfr *TermFrequencyRow) Freq() uint64 { + return tfr.freq +} + +func (tfr *TermFrequencyRow) ScanPrefixForField() []byte { + buf := make([]byte, 3) + buf[0] = 't' + binary.LittleEndian.PutUint16(buf[1:3], tfr.field) + return buf +} + +func (tfr *TermFrequencyRow) ScanPrefixForFieldTermPrefix() []byte { + buf := make([]byte, 3+len(tfr.term)) + buf[0] = 't' + binary.LittleEndian.PutUint16(buf[1:3], tfr.field) + copy(buf[3:], tfr.term) + return buf +} + +func (tfr *TermFrequencyRow) ScanPrefixForFieldTerm() []byte { + buf := make([]byte, 3+len(tfr.term)+1) + buf[0] = 't' + binary.LittleEndian.PutUint16(buf[1:3], tfr.field) + termLen := copy(buf[3:], tfr.term) + buf[3+termLen] = ByteSeparator + return buf +} + +func (tfr *TermFrequencyRow) Key() []byte { + buf := make([]byte, tfr.KeySize()) + size, _ := tfr.KeyTo(buf) + return buf[:size] +} + +func (tfr *TermFrequencyRow) KeySize() int { + return termFrequencyRowKeySize(tfr.term, tfr.doc) +} + +func termFrequencyRowKeySize(term, doc []byte) int { + return 3 + len(term) + 1 + len(doc) +} + +func (tfr *TermFrequencyRow) KeyTo(buf []byte) (int, error) { + return termFrequencyRowKeyTo(buf, tfr.field, tfr.term, tfr.doc), nil +} + +func termFrequencyRowKeyTo(buf []byte, field uint16, term, doc []byte) int { + buf[0] = 't' + binary.LittleEndian.PutUint16(buf[1:3], field) + termLen := copy(buf[3:], term) + buf[3+termLen] = ByteSeparator + docLen := copy(buf[3+termLen+1:], doc) + return 3 + termLen + 1 + docLen +} + +func (tfr *TermFrequencyRow) KeyAppendTo(buf []byte) ([]byte, error) { + keySize := tfr.KeySize() + if cap(buf) < keySize { + buf = make([]byte, keySize) + } + actualSize, err := tfr.KeyTo(buf[0:keySize]) + return buf[0:actualSize], err +} + +func (tfr *TermFrequencyRow) DictionaryRowKey() []byte { + dr := NewDictionaryRow(tfr.term, tfr.field, 0) + return dr.Key() +} + +func (tfr *TermFrequencyRow) DictionaryRowKeySize() int { + dr := NewDictionaryRow(tfr.term, tfr.field, 0) + return dr.KeySize() +} + +func (tfr *TermFrequencyRow) DictionaryRowKeyTo(buf []byte) (int, error) { + dr := NewDictionaryRow(tfr.term, tfr.field, 0) + return dr.KeyTo(buf) +} + +func (tfr *TermFrequencyRow) Value() []byte { + buf := make([]byte, tfr.ValueSize()) + size, _ := tfr.ValueTo(buf) + return buf[:size] +} + +func (tfr *TermFrequencyRow) ValueSize() int { + bufLen := binary.MaxVarintLen64 + binary.MaxVarintLen64 + for _, vector := range tfr.vectors { + bufLen += (binary.MaxVarintLen64 * 4) + (1+len(vector.arrayPositions))*binary.MaxVarintLen64 + } + return bufLen +} + +func (tfr *TermFrequencyRow) ValueTo(buf []byte) (int, error) { + used := binary.PutUvarint(buf[:binary.MaxVarintLen64], tfr.freq) + + normuint32 := math.Float32bits(tfr.norm) + newbuf := buf[used : used+binary.MaxVarintLen64] + used += binary.PutUvarint(newbuf, uint64(normuint32)) + + for _, vector := range tfr.vectors { + used += binary.PutUvarint(buf[used:used+binary.MaxVarintLen64], uint64(vector.field)) + used += binary.PutUvarint(buf[used:used+binary.MaxVarintLen64], vector.pos) + used += binary.PutUvarint(buf[used:used+binary.MaxVarintLen64], vector.start) + used += binary.PutUvarint(buf[used:used+binary.MaxVarintLen64], vector.end) + used += binary.PutUvarint(buf[used:used+binary.MaxVarintLen64], uint64(len(vector.arrayPositions))) + for _, arrayPosition := range vector.arrayPositions { + used += binary.PutUvarint(buf[used:used+binary.MaxVarintLen64], arrayPosition) + } + } + return used, nil +} + +func (tfr *TermFrequencyRow) String() string { + return fmt.Sprintf("Term: `%s` Field: %d DocId: `%s` Frequency: %d Norm: %f Vectors: %v", string(tfr.term), tfr.field, string(tfr.doc), tfr.freq, tfr.norm, tfr.vectors) +} + +func InitTermFrequencyRow(tfr *TermFrequencyRow, term []byte, field uint16, docID []byte, freq uint64, norm float32) *TermFrequencyRow { + tfr.term = term + tfr.field = field + tfr.doc = docID + tfr.freq = freq + tfr.norm = norm + return tfr +} + +func NewTermFrequencyRow(term []byte, field uint16, docID []byte, freq uint64, norm float32) *TermFrequencyRow { + return &TermFrequencyRow{ + term: term, + field: field, + doc: docID, + freq: freq, + norm: norm, + } +} + +func NewTermFrequencyRowWithTermVectors(term []byte, field uint16, docID []byte, freq uint64, norm float32, vectors []*TermVector) *TermFrequencyRow { + return &TermFrequencyRow{ + term: term, + field: field, + doc: docID, + freq: freq, + norm: norm, + vectors: vectors, + } +} + +func NewTermFrequencyRowK(key []byte) (*TermFrequencyRow, error) { + rv := &TermFrequencyRow{} + err := rv.parseK(key) + if err != nil { + return nil, err + } + return rv, nil +} + +func (tfr *TermFrequencyRow) parseK(key []byte) error { + keyLen := len(key) + if keyLen < 3 { + return fmt.Errorf("invalid term frequency key, no valid field") + } + tfr.field = binary.LittleEndian.Uint16(key[1:3]) + + termEndPos := bytes.IndexByte(key[3:], ByteSeparator) + if termEndPos < 0 { + return fmt.Errorf("invalid term frequency key, no byte separator terminating term") + } + tfr.term = key[3 : 3+termEndPos] + + docLen := keyLen - (3 + termEndPos + 1) + if docLen < 1 { + return fmt.Errorf("invalid term frequency key, empty docid") + } + tfr.doc = key[3+termEndPos+1:] + + return nil +} + +func (tfr *TermFrequencyRow) parseKDoc(key []byte, term []byte) error { + tfr.doc = key[3+len(term)+1:] + if len(tfr.doc) == 0 { + return fmt.Errorf("invalid term frequency key, empty docid") + } + + return nil +} + +func (tfr *TermFrequencyRow) parseV(value []byte, includeTermVectors bool) error { + var bytesRead int + tfr.freq, bytesRead = binary.Uvarint(value) + if bytesRead <= 0 { + return fmt.Errorf("invalid term frequency value, invalid frequency") + } + currOffset := bytesRead + + var norm uint64 + norm, bytesRead = binary.Uvarint(value[currOffset:]) + if bytesRead <= 0 { + return fmt.Errorf("invalid term frequency value, no norm") + } + currOffset += bytesRead + + tfr.norm = math.Float32frombits(uint32(norm)) + + tfr.vectors = nil + if !includeTermVectors { + return nil + } + + var field uint64 + field, bytesRead = binary.Uvarint(value[currOffset:]) + for bytesRead > 0 { + currOffset += bytesRead + tv := TermVector{} + tv.field = uint16(field) + // at this point we expect at least one term vector + if tfr.vectors == nil { + tfr.vectors = make([]*TermVector, 0) + } + + tv.pos, bytesRead = binary.Uvarint(value[currOffset:]) + if bytesRead <= 0 { + return fmt.Errorf("invalid term frequency value, vector contains no position") + } + currOffset += bytesRead + + tv.start, bytesRead = binary.Uvarint(value[currOffset:]) + if bytesRead <= 0 { + return fmt.Errorf("invalid term frequency value, vector contains no start") + } + currOffset += bytesRead + + tv.end, bytesRead = binary.Uvarint(value[currOffset:]) + if bytesRead <= 0 { + return fmt.Errorf("invalid term frequency value, vector contains no end") + } + currOffset += bytesRead + + var arrayPositionsLen uint64 + arrayPositionsLen, bytesRead = binary.Uvarint(value[currOffset:]) + if bytesRead <= 0 { + return fmt.Errorf("invalid term frequency value, vector contains no arrayPositionLen") + } + currOffset += bytesRead + + if arrayPositionsLen > 0 { + tv.arrayPositions = make([]uint64, arrayPositionsLen) + for i := 0; uint64(i) < arrayPositionsLen; i++ { + tv.arrayPositions[i], bytesRead = binary.Uvarint(value[currOffset:]) + if bytesRead <= 0 { + return fmt.Errorf("invalid term frequency value, vector contains no arrayPosition of index %d", i) + } + currOffset += bytesRead + } + } + + tfr.vectors = append(tfr.vectors, &tv) + // try to read next record (may not exist) + field, bytesRead = binary.Uvarint(value[currOffset:]) + } + if len(value[currOffset:]) > 0 && bytesRead <= 0 { + return fmt.Errorf("invalid term frequency value, vector field invalid") + } + + return nil +} + +func NewTermFrequencyRowKV(key, value []byte) (*TermFrequencyRow, error) { + rv, err := NewTermFrequencyRowK(key) + if err != nil { + return nil, err + } + + err = rv.parseV(value, true) + if err != nil { + return nil, err + } + return rv, nil +} + +type BackIndexRow struct { + doc []byte + termsEntries []*BackIndexTermsEntry + storedEntries []*BackIndexStoreEntry +} + +func (br *BackIndexRow) AllTermKeys() [][]byte { + if br == nil { + return nil + } + rv := make([][]byte, 0, len(br.termsEntries)) // FIXME this underestimates severely + for _, termsEntry := range br.termsEntries { + for i := range termsEntry.Terms { + termRow := NewTermFrequencyRow([]byte(termsEntry.Terms[i]), uint16(termsEntry.GetField()), br.doc, 0, 0) + rv = append(rv, termRow.Key()) + } + } + return rv +} + +func (br *BackIndexRow) AllStoredKeys() [][]byte { + if br == nil { + return nil + } + rv := make([][]byte, len(br.storedEntries)) + for i, storedEntry := range br.storedEntries { + storedRow := NewStoredRow(br.doc, uint16(storedEntry.GetField()), storedEntry.GetArrayPositions(), 'x', []byte{}) + rv[i] = storedRow.Key() + } + return rv +} + +func (br *BackIndexRow) Key() []byte { + buf := make([]byte, br.KeySize()) + size, _ := br.KeyTo(buf) + return buf[:size] +} + +func (br *BackIndexRow) KeySize() int { + return len(br.doc) + 1 +} + +func (br *BackIndexRow) KeyTo(buf []byte) (int, error) { + buf[0] = 'b' + used := copy(buf[1:], br.doc) + return used + 1, nil +} + +func (br *BackIndexRow) Value() []byte { + buf := make([]byte, br.ValueSize()) + size, _ := br.ValueTo(buf) + return buf[:size] +} + +func (br *BackIndexRow) ValueSize() int { + birv := &BackIndexRowValue{ + TermsEntries: br.termsEntries, + StoredEntries: br.storedEntries, + } + return birv.Size() +} + +func (br *BackIndexRow) ValueTo(buf []byte) (int, error) { + birv := &BackIndexRowValue{ + TermsEntries: br.termsEntries, + StoredEntries: br.storedEntries, + } + return birv.MarshalTo(buf) +} + +func (br *BackIndexRow) String() string { + return fmt.Sprintf("Backindex DocId: `%s` Terms Entries: %v, Stored Entries: %v", string(br.doc), br.termsEntries, br.storedEntries) +} + +func NewBackIndexRow(docID []byte, entries []*BackIndexTermsEntry, storedFields []*BackIndexStoreEntry) *BackIndexRow { + return &BackIndexRow{ + doc: docID, + termsEntries: entries, + storedEntries: storedFields, + } +} + +func NewBackIndexRowKV(key, value []byte) (*BackIndexRow, error) { + rv := BackIndexRow{} + + buf := bytes.NewBuffer(key) + _, err := buf.ReadByte() // type + if err != nil { + return nil, err + } + + rv.doc, err = buf.ReadBytes(ByteSeparator) + if err == io.EOF && len(rv.doc) < 1 { + err = fmt.Errorf("invalid doc length 0 - % x", key) + } + if err != nil && err != io.EOF { + return nil, err + } else if err == nil { + rv.doc = rv.doc[:len(rv.doc)-1] // trim off separator byte + } + + var birv BackIndexRowValue + err = proto.Unmarshal(value, &birv) + if err != nil { + return nil, err + } + rv.termsEntries = birv.TermsEntries + rv.storedEntries = birv.StoredEntries + + return &rv, nil +} + +// STORED + +type StoredRow struct { + doc []byte + field uint16 + arrayPositions []uint64 + typ byte + value []byte +} + +func (s *StoredRow) Key() []byte { + buf := make([]byte, s.KeySize()) + size, _ := s.KeyTo(buf) + return buf[0:size] +} + +func (s *StoredRow) KeySize() int { + return 1 + len(s.doc) + 1 + 2 + (binary.MaxVarintLen64 * len(s.arrayPositions)) +} + +func (s *StoredRow) KeyTo(buf []byte) (int, error) { + docLen := len(s.doc) + buf[0] = 's' + copy(buf[1:], s.doc) + buf[1+docLen] = ByteSeparator + binary.LittleEndian.PutUint16(buf[1+docLen+1:], s.field) + bytesUsed := 1 + docLen + 1 + 2 + for _, arrayPosition := range s.arrayPositions { + varbytes := binary.PutUvarint(buf[bytesUsed:], arrayPosition) + bytesUsed += varbytes + } + return bytesUsed, nil +} + +func (s *StoredRow) Value() []byte { + buf := make([]byte, s.ValueSize()) + size, _ := s.ValueTo(buf) + return buf[:size] +} + +func (s *StoredRow) ValueSize() int { + return len(s.value) + 1 +} + +func (s *StoredRow) ValueTo(buf []byte) (int, error) { + buf[0] = s.typ + used := copy(buf[1:], s.value) + return used + 1, nil +} + +func (s *StoredRow) String() string { + return fmt.Sprintf("Document: %s Field %d, Array Positions: %v, Type: %s Value: %s", s.doc, s.field, s.arrayPositions, string(s.typ), s.value) +} + +func (s *StoredRow) ScanPrefixForDoc() []byte { + docLen := len(s.doc) + buf := make([]byte, 1+docLen+1) + buf[0] = 's' + copy(buf[1:], s.doc) + buf[1+docLen] = ByteSeparator + return buf +} + +func NewStoredRow(docID []byte, field uint16, arrayPositions []uint64, typ byte, value []byte) *StoredRow { + return &StoredRow{ + doc: docID, + field: field, + arrayPositions: arrayPositions, + typ: typ, + value: value, + } +} + +func NewStoredRowK(key []byte) (*StoredRow, error) { + rv := StoredRow{} + + buf := bytes.NewBuffer(key) + _, err := buf.ReadByte() // type + if err != nil { + return nil, err + } + + rv.doc, err = buf.ReadBytes(ByteSeparator) + if err != nil { + return nil, err + } + if len(rv.doc) < 2 { // 1 for min doc id length, 1 for separator + err = fmt.Errorf("invalid doc length 0") + return nil, err + } + + rv.doc = rv.doc[:len(rv.doc)-1] // trim off separator byte + + err = binary.Read(buf, binary.LittleEndian, &rv.field) + if err != nil { + return nil, err + } + + rv.arrayPositions = make([]uint64, 0) + nextArrayPos, err := binary.ReadUvarint(buf) + for err == nil { + rv.arrayPositions = append(rv.arrayPositions, nextArrayPos) + nextArrayPos, err = binary.ReadUvarint(buf) + } + return &rv, nil +} + +func NewStoredRowKV(key, value []byte) (*StoredRow, error) { + rv, err := NewStoredRowK(key) + if err != nil { + return nil, err + } + rv.typ = value[0] + rv.value = value[1:] + return rv, nil +} + +type backIndexFieldTermVisitor func(field uint32, term []byte) + +// visitBackIndexRow is designed to process a protobuf encoded +// value, without creating unnecessary garbage. Instead values are passed +// to a callback, inspected first, and only copied if necessary. +// Due to the fact that this borrows from generated code, it must be marnually +// updated if the protobuf definition changes. +// +// This code originates from: +// func (m *BackIndexRowValue) Unmarshal(data []byte) error +// the sections which create garbage or parse uninteresting sections +// have been commented out. This was done by design to allow for easier +// merging in the future if that original function is regenerated +func visitBackIndexRow(data []byte, callback backIndexFieldTermVisitor) error { + l := len(data) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TermsEntries", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + postIndex := iNdEx + msglen + if msglen < 0 { + return ErrInvalidLengthUpsidedown + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + // dont parse term entries + // m.TermsEntries = append(m.TermsEntries, &BackIndexTermsEntry{}) + // if err := m.TermsEntries[len(m.TermsEntries)-1].Unmarshal(data[iNdEx:postIndex]); err != nil { + // return err + // } + // instead, inspect them + if err := visitBackIndexRowFieldTerms(data[iNdEx:postIndex], callback); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field StoredEntries", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + postIndex := iNdEx + msglen + if msglen < 0 { + return ErrInvalidLengthUpsidedown + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + // don't parse stored entries + // m.StoredEntries = append(m.StoredEntries, &BackIndexStoreEntry{}) + // if err := m.StoredEntries[len(m.StoredEntries)-1].Unmarshal(data[iNdEx:postIndex]); err != nil { + // return err + // } + iNdEx = postIndex + default: + var sizeOfWire int + for { + sizeOfWire++ + wire >>= 7 + if wire == 0 { + break + } + } + iNdEx -= sizeOfWire + skippy, err := skipUpsidedown(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthUpsidedown + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + // don't track unrecognized data + // m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + return nil +} + +// visitBackIndexRowFieldTerms is designed to process a protobuf encoded +// sub-value within the BackIndexRowValue, without creating unnecessary garbage. +// Instead values are passed to a callback, inspected first, and only copied if +// necessary. Due to the fact that this borrows from generated code, it must +// be marnually updated if the protobuf definition changes. +// +// This code originates from: +// func (m *BackIndexTermsEntry) Unmarshal(data []byte) error { +// the sections which create garbage or parse uninteresting sections +// have been commented out. This was done by design to allow for easier +// merging in the future if that original function is regenerated +func visitBackIndexRowFieldTerms(data []byte, callback backIndexFieldTermVisitor) error { + var theField uint32 + + var hasFields [1]uint64 + l := len(data) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + } + var v uint32 + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + v |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + // m.Field = &v + theField = v + hasFields[0] |= uint64(0x00000001) + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Terms", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + postIndex := iNdEx + int(stringLen) + if postIndex > l { + return io.ErrUnexpectedEOF + } + // m.Terms = append(m.Terms, string(data[iNdEx:postIndex])) + callback(theField, data[iNdEx:postIndex]) + iNdEx = postIndex + default: + var sizeOfWire int + for { + sizeOfWire++ + wire >>= 7 + if wire == 0 { + break + } + } + iNdEx -= sizeOfWire + skippy, err := skipUpsidedown(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthUpsidedown + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + // m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + // if hasFields[0]&uint64(0x00000001) == 0 { + // return new(github_com_golang_protobuf_proto.RequiredNotSetError) + // } + + return nil +} diff --git a/index/upsidedown/row_merge.go b/index/upsidedown/row_merge.go new file mode 100644 index 0000000..39172ad --- /dev/null +++ b/index/upsidedown/row_merge.go @@ -0,0 +1,76 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "encoding/binary" +) + +var mergeOperator upsideDownMerge + +var dictionaryTermIncr []byte +var dictionaryTermDecr []byte + +func init() { + dictionaryTermIncr = make([]byte, 8) + binary.LittleEndian.PutUint64(dictionaryTermIncr, uint64(1)) + dictionaryTermDecr = make([]byte, 8) + var negOne = int64(-1) + binary.LittleEndian.PutUint64(dictionaryTermDecr, uint64(negOne)) +} + +type upsideDownMerge struct{} + +func (m *upsideDownMerge) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) { + // set up record based on key + dr, err := NewDictionaryRowK(key) + if err != nil { + return nil, false + } + if len(existingValue) > 0 { + // if existing value, parse it + err = dr.parseDictionaryV(existingValue) + if err != nil { + return nil, false + } + } + + // now process operands + for _, operand := range operands { + next := int64(binary.LittleEndian.Uint64(operand)) + if next < 0 && uint64(-next) > dr.count { + // subtracting next from existing would overflow + dr.count = 0 + } else if next < 0 { + dr.count -= uint64(-next) + } else { + dr.count += uint64(next) + } + } + + return dr.Value(), true +} + +func (m *upsideDownMerge) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) { + left := int64(binary.LittleEndian.Uint64(leftOperand)) + right := int64(binary.LittleEndian.Uint64(rightOperand)) + rv := make([]byte, 8) + binary.LittleEndian.PutUint64(rv, uint64(left+right)) + return rv, true +} + +func (m *upsideDownMerge) Name() string { + return "upsideDownMerge" +} diff --git a/index/upsidedown/row_merge_test.go b/index/upsidedown/row_merge_test.go new file mode 100644 index 0000000..66eb9cb --- /dev/null +++ b/index/upsidedown/row_merge_test.go @@ -0,0 +1,57 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "bytes" + "encoding/binary" + "testing" +) + +func TestPartialMerge(t *testing.T) { + + tests := []struct { + in [][]byte + out uint64 + }{ + { + in: [][]byte{dictionaryTermIncr, dictionaryTermIncr, dictionaryTermIncr, dictionaryTermIncr, dictionaryTermIncr}, + out: 5, + }, + } + + mo := &upsideDownMerge{} + for _, test := range tests { + curr := test.in[0] + for _, next := range test.in[1:] { + var ok bool + curr, ok = mo.PartialMerge([]byte("key"), curr, next) + if !ok { + t.Errorf("expected partial merge ok") + } + } + actual := decodeCount(curr) + if actual != test.out { + t.Errorf("expected %d, got %d", test.out, actual) + } + } + +} + +func decodeCount(in []byte) uint64 { + buf := bytes.NewBuffer(in) + count, _ := binary.ReadUvarint(buf) + return count +} diff --git a/index/upsidedown/row_test.go b/index/upsidedown/row_test.go new file mode 100644 index 0000000..2221134 --- /dev/null +++ b/index/upsidedown/row_test.go @@ -0,0 +1,382 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "math" + "reflect" + "testing" + + "google.golang.org/protobuf/proto" +) + +func TestRows(t *testing.T) { + tests := []struct { + input UpsideDownCouchRow + outKey []byte + outVal []byte + }{ + { + NewVersionRow(1), + []byte{'v'}, + []byte{0x1}, + }, + { + NewFieldRow(0, "name"), + []byte{'f', 0, 0}, + []byte{'n', 'a', 'm', 'e', ByteSeparator}, + }, + { + NewFieldRow(1, "desc"), + []byte{'f', 1, 0}, + []byte{'d', 'e', 's', 'c', ByteSeparator}, + }, + { + NewFieldRow(513, "style"), + []byte{'f', 1, 2}, + []byte{'s', 't', 'y', 'l', 'e', ByteSeparator}, + }, + { + NewDictionaryRow([]byte{'b', 'e', 'e', 'r'}, 0, 27), + []byte{'d', 0, 0, 'b', 'e', 'e', 'r'}, + []byte{27}, + }, + { + NewTermFrequencyRow([]byte{'b', 'e', 'e', 'r'}, 0, []byte("catz"), 3, 3.14), + []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'c', 'a', 't', 'z'}, + []byte{3, 195, 235, 163, 130, 4}, + }, + { + NewTermFrequencyRow([]byte{'b', 'e', 'e', 'r'}, 0, []byte("budweiser"), 3, 3.14), + []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, + []byte{3, 195, 235, 163, 130, 4}, + }, + { + NewTermFrequencyRowWithTermVectors([]byte{'b', 'e', 'e', 'r'}, 0, []byte("budweiser"), 3, 3.14, []*TermVector{{field: 0, pos: 1, start: 3, end: 11}, {field: 0, pos: 2, start: 23, end: 31}, {field: 0, pos: 3, start: 43, end: 51}}), + []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, + []byte{3, 195, 235, 163, 130, 4, 0, 1, 3, 11, 0, 0, 2, 23, 31, 0, 0, 3, 43, 51, 0}, + }, + // test larger varints + { + NewTermFrequencyRowWithTermVectors([]byte{'b', 'e', 'e', 'r'}, 0, []byte("budweiser"), 25896, 3.14, []*TermVector{{field: 255, pos: 1, start: 3, end: 11}, {field: 0, pos: 2198, start: 23, end: 31}, {field: 0, pos: 3, start: 43, end: 51}}), + []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, + []byte{168, 202, 1, 195, 235, 163, 130, 4, 255, 1, 1, 3, 11, 0, 0, 150, 17, 23, 31, 0, 0, 3, 43, 51, 0}, + }, + // test vectors with arrayPositions + { + NewTermFrequencyRowWithTermVectors([]byte{'b', 'e', 'e', 'r'}, 0, []byte("budweiser"), 25896, 3.14, []*TermVector{{field: 255, pos: 1, start: 3, end: 11, arrayPositions: []uint64{0}}, {field: 0, pos: 2198, start: 23, end: 31, arrayPositions: []uint64{1, 2}}, {field: 0, pos: 3, start: 43, end: 51, arrayPositions: []uint64{3, 4, 5}}}), + []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, + []byte{168, 202, 1, 195, 235, 163, 130, 4, 255, 1, 1, 3, 11, 1, 0, 0, 150, 17, 23, 31, 2, 1, 2, 0, 3, 43, 51, 3, 3, 4, 5}, + }, + { + NewBackIndexRow([]byte("budweiser"), []*BackIndexTermsEntry{{Field: proto.Uint32(0), Terms: []string{"beer"}}}, nil), + []byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, + []byte{10, 8, 8, 0, 18, 4, 'b', 'e', 'e', 'r'}, + }, + { + NewBackIndexRow([]byte("budweiser"), []*BackIndexTermsEntry{{Field: proto.Uint32(0), Terms: []string{"beer"}}, {Field: proto.Uint32(1), Terms: []string{"beat"}}}, nil), + []byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, + []byte{10, 8, 8, 0, 18, 4, 'b', 'e', 'e', 'r', 10, 8, 8, 1, 18, 4, 'b', 'e', 'a', 't'}, + }, + { + NewBackIndexRow([]byte("budweiser"), []*BackIndexTermsEntry{{Field: proto.Uint32(0), Terms: []string{"beer"}}, {Field: proto.Uint32(1), Terms: []string{"beat"}}}, []*BackIndexStoreEntry{{Field: proto.Uint32(3)}, {Field: proto.Uint32(4)}, {Field: proto.Uint32(5)}}), + []byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, + []byte{10, 8, 8, 0, 18, 4, 'b', 'e', 'e', 'r', 10, 8, 8, 1, 18, 4, 'b', 'e', 'a', 't', 18, 2, 8, 3, 18, 2, 8, 4, 18, 2, 8, 5}, + }, + { + NewStoredRow([]byte("budweiser"), 0, []uint64{}, byte('t'), []byte("an american beer")), + []byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator, 0, 0}, + []byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'}, + }, + { + NewStoredRow([]byte("budweiser"), 0, []uint64{2, 294, 3078}, byte('t'), []byte("an american beer")), + []byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator, 0, 0, 2, 166, 2, 134, 24}, + []byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'}, + }, + { + NewInternalRow([]byte("mapping"), []byte(`{"mapping":"json content"}`)), + []byte{'i', 'm', 'a', 'p', 'p', 'i', 'n', 'g'}, + []byte{'{', '"', 'm', 'a', 'p', 'p', 'i', 'n', 'g', '"', ':', '"', 'j', 's', 'o', 'n', ' ', 'c', 'o', 'n', 't', 'e', 'n', 't', '"', '}'}, + }, + } + + // test going from struct to k/v bytes + for i, test := range tests { + rk := test.input.Key() + if !reflect.DeepEqual(rk, test.outKey) { + t.Errorf("Expected key to be %v got: %v", test.outKey, rk) + } + rv := test.input.Value() + if !reflect.DeepEqual(rv, test.outVal) { + t.Errorf("Expected value to be %v got: %v for %d", test.outVal, rv, i) + } + } + + // now test going back from k/v bytes to struct + for i, test := range tests { + row, err := ParseFromKeyValue(test.outKey, test.outVal) + if err != nil { + t.Errorf("error parsking key/value: %v", err) + } + if !reflect.DeepEqual(row, test.input) { + t.Errorf("Expected: %#v got: %#v for %d", test.input, row, i) + } + } + +} + +func TestInvalidRows(t *testing.T) { + tests := []struct { + key []byte + val []byte + }{ + // empty key + { + []byte{}, + []byte{}, + }, + // no such type q + { + []byte{'q'}, + []byte{}, + }, + // type v, invalid empty value + { + []byte{'v'}, + []byte{}, + }, + // type f, invalid key + { + []byte{'f'}, + []byte{}, + }, + // type f, valid key, invalid value + { + []byte{'f', 0, 0}, + []byte{}, + }, + // type t, invalid key (missing field) + { + []byte{'t'}, + []byte{}, + }, + // type t, invalid key (missing term) + { + []byte{'t', 0, 0}, + []byte{}, + }, + // type t, invalid key (missing id) + { + []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator}, + []byte{}, + }, + // type t, invalid val (missing freq) + { + []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, + []byte{}, + }, + // type t, invalid val (missing norm) + { + []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, + []byte{3}, + }, + // type t, invalid val (half missing tv field, full missing is valid (no term vectors)) + { + []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, + []byte{3, 25, 255}, + }, + // type t, invalid val (missing tv pos) + { + []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, + []byte{3, 25, 0}, + }, + // type t, invalid val (missing tv start) + { + []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, + []byte{3, 25, 0, 0}, + }, + // type t, invalid val (missing tv end) + { + []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, + []byte{3, 25, 0, 0, 0}, + }, + // type b, invalid key (missing id) + { + []byte{'b'}, + []byte{'b', 'e', 'e', 'r', ByteSeparator, 0, 0}, + }, + // type b, invalid val (missing field) + { + []byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, + []byte{'g', 'a', 'r', 'b', 'a', 'g', 'e'}, + }, + // type s, invalid key (missing id) + { + []byte{'s'}, + []byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'}, + }, + // type b, invalid val (missing field) + { + []byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator}, + []byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'}, + }, + } + + for _, test := range tests { + _, err := ParseFromKeyValue(test.key, test.val) + if err == nil { + t.Errorf("expected error, got nil") + } + } +} + +func TestDictionaryRowValueBug197(t *testing.T) { + // this was the smallest value that would trigger a crash + dr := &DictionaryRow{ + field: 0, + term: []byte("marty"), + count: 72057594037927936, + } + dr.Value() + // this is the maximum possible value + dr = &DictionaryRow{ + field: 0, + term: []byte("marty"), + count: math.MaxUint64, + } + dr.Value() + // neither of these should panic +} + +func BenchmarkTermFrequencyRowEncode(b *testing.B) { + row := NewTermFrequencyRowWithTermVectors( + []byte{'b', 'e', 'e', 'r'}, + 0, + []byte("budweiser"), + 3, + 3.14, + []*TermVector{ + { + field: 0, + pos: 1, + start: 3, + end: 11, + }, + { + field: 0, + pos: 2, + start: 23, + end: 31, + }, + { + field: 0, + pos: 3, + start: 43, + end: 51, + }, + }) + b.ResetTimer() + for i := 0; i < b.N; i++ { + row.Key() + row.Value() + } +} + +func BenchmarkTermFrequencyRowDecode(b *testing.B) { + k := []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'} + v := []byte{3, 195, 235, 163, 130, 4, 0, 1, 3, 11, 0, 0, 2, 23, 31, 0, 0, 3, 43, 51, 0} + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := NewTermFrequencyRowKV(k, v) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkBackIndexRowEncode(b *testing.B) { + field := uint32(1) + t1 := "term1" + row := NewBackIndexRow([]byte("beername"), + []*BackIndexTermsEntry{ + { + Field: &field, + Terms: []string{t1}, + }, + }, + []*BackIndexStoreEntry{ + { + Field: &field, + }, + }) + b.ResetTimer() + for i := 0; i < b.N; i++ { + row.Key() + row.Value() + b.Logf("%#v", row.Value()) + } +} + +func BenchmarkBackIndexRowDecode(b *testing.B) { + k := []byte{0x62, 0x62, 0x65, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65} + v := []byte{0xa, 0x9, 0x8, 0x1, 0x12, 0x5, 0x74, 0x65, 0x72, 0x6d, 0x31, 0x12, 0x2, 0x8, 0x1} + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := NewBackIndexRowKV(k, v) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkStoredRowEncode(b *testing.B) { + row := NewStoredRow([]byte("budweiser"), 0, []uint64{}, byte('t'), []byte("an american beer")) + b.ResetTimer() + for i := 0; i < b.N; i++ { + row.Key() + row.Value() + } +} + +func BenchmarkStoredRowDecode(b *testing.B) { + k := []byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator, 0, 0} + v := []byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'} + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := NewStoredRowKV(k, v) + if err != nil { + b.Fatal(err) + } + } +} + +func TestVisitBackIndexRow(t *testing.T) { + expected := map[uint32][]byte{ + 0: []byte("beer"), + 1: []byte("beat"), + } + val := []byte{10, 8, 8, 0, 18, 4, 'b', 'e', 'e', 'r', 10, 8, 8, 1, 18, 4, 'b', 'e', 'a', 't', 18, 2, 8, 3, 18, 2, 8, 4, 18, 2, 8, 5} + err := visitBackIndexRow(val, func(field uint32, term []byte) { + if reflect.DeepEqual(expected[field], term) { + delete(expected, field) + } + }) + if err != nil { + t.Fatal(err) + } + if len(expected) > 0 { + t.Errorf("expected visitor to see these but did not %v", expected) + } +} diff --git a/index/upsidedown/stats.go b/index/upsidedown/stats.go new file mode 100644 index 0000000..5fea7c1 --- /dev/null +++ b/index/upsidedown/stats.go @@ -0,0 +1,55 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "sync/atomic" + + "github.com/blevesearch/bleve/v2/util" + "github.com/blevesearch/upsidedown_store_api" +) + +type indexStat struct { + updates, deletes, batches, errors uint64 + analysisTime, indexTime uint64 + termSearchersStarted uint64 + termSearchersFinished uint64 + numPlainTextBytesIndexed uint64 + i *UpsideDownCouch +} + +func (i *indexStat) statsMap() map[string]interface{} { + m := map[string]interface{}{} + m["updates"] = atomic.LoadUint64(&i.updates) + m["deletes"] = atomic.LoadUint64(&i.deletes) + m["batches"] = atomic.LoadUint64(&i.batches) + m["errors"] = atomic.LoadUint64(&i.errors) + m["analysis_time"] = atomic.LoadUint64(&i.analysisTime) + m["index_time"] = atomic.LoadUint64(&i.indexTime) + m["term_searchers_started"] = atomic.LoadUint64(&i.termSearchersStarted) + m["term_searchers_finished"] = atomic.LoadUint64(&i.termSearchersFinished) + m["num_plain_text_bytes_indexed"] = atomic.LoadUint64(&i.numPlainTextBytesIndexed) + + if o, ok := i.i.store.(store.KVStoreStats); ok { + m["kv"] = o.StatsMap() + } + + return m +} + +func (i *indexStat) MarshalJSON() ([]byte, error) { + m := i.statsMap() + return util.MarshalJSON(m) +} diff --git a/index/upsidedown/store/boltdb/iterator.go b/index/upsidedown/store/boltdb/iterator.go new file mode 100644 index 0000000..cf4da87 --- /dev/null +++ b/index/upsidedown/store/boltdb/iterator.go @@ -0,0 +1,85 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package boltdb + +import ( + "bytes" + + bolt "go.etcd.io/bbolt" +) + +type Iterator struct { + store *Store + tx *bolt.Tx + cursor *bolt.Cursor + prefix []byte + start []byte + end []byte + valid bool + key []byte + val []byte +} + +func (i *Iterator) updateValid() { + i.valid = (i.key != nil) + if i.valid { + if i.prefix != nil { + i.valid = bytes.HasPrefix(i.key, i.prefix) + } else if i.end != nil { + i.valid = bytes.Compare(i.key, i.end) < 0 + } + } +} + +func (i *Iterator) Seek(k []byte) { + if i.start != nil && bytes.Compare(k, i.start) < 0 { + k = i.start + } + if i.prefix != nil && !bytes.HasPrefix(k, i.prefix) { + if bytes.Compare(k, i.prefix) < 0 { + k = i.prefix + } else { + i.valid = false + return + } + } + i.key, i.val = i.cursor.Seek(k) + i.updateValid() +} + +func (i *Iterator) Next() { + i.key, i.val = i.cursor.Next() + i.updateValid() +} + +func (i *Iterator) Current() ([]byte, []byte, bool) { + return i.key, i.val, i.valid +} + +func (i *Iterator) Key() []byte { + return i.key +} + +func (i *Iterator) Value() []byte { + return i.val +} + +func (i *Iterator) Valid() bool { + return i.valid +} + +func (i *Iterator) Close() error { + return nil +} diff --git a/index/upsidedown/store/boltdb/reader.go b/index/upsidedown/store/boltdb/reader.go new file mode 100644 index 0000000..79513f6 --- /dev/null +++ b/index/upsidedown/store/boltdb/reader.go @@ -0,0 +1,73 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package boltdb + +import ( + store "github.com/blevesearch/upsidedown_store_api" + bolt "go.etcd.io/bbolt" +) + +type Reader struct { + store *Store + tx *bolt.Tx + bucket *bolt.Bucket +} + +func (r *Reader) Get(key []byte) ([]byte, error) { + var rv []byte + v := r.bucket.Get(key) + if v != nil { + rv = make([]byte, len(v)) + copy(rv, v) + } + return rv, nil +} + +func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) { + return store.MultiGet(r, keys) +} + +func (r *Reader) PrefixIterator(prefix []byte) store.KVIterator { + cursor := r.bucket.Cursor() + + rv := &Iterator{ + store: r.store, + tx: r.tx, + cursor: cursor, + prefix: prefix, + } + + rv.Seek(prefix) + return rv +} + +func (r *Reader) RangeIterator(start, end []byte) store.KVIterator { + cursor := r.bucket.Cursor() + + rv := &Iterator{ + store: r.store, + tx: r.tx, + cursor: cursor, + start: start, + end: end, + } + + rv.Seek(start) + return rv +} + +func (r *Reader) Close() error { + return r.tx.Rollback() +} diff --git a/index/upsidedown/store/boltdb/stats.go b/index/upsidedown/store/boltdb/stats.go new file mode 100644 index 0000000..d896183 --- /dev/null +++ b/index/upsidedown/store/boltdb/stats.go @@ -0,0 +1,28 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package boltdb + +import ( + "github.com/blevesearch/bleve/v2/util" +) + +type stats struct { + s *Store +} + +func (s *stats) MarshalJSON() ([]byte, error) { + bs := s.s.db.Stats() + return util.MarshalJSON(bs) +} diff --git a/index/upsidedown/store/boltdb/store.go b/index/upsidedown/store/boltdb/store.go new file mode 100644 index 0000000..2ebe9d2 --- /dev/null +++ b/index/upsidedown/store/boltdb/store.go @@ -0,0 +1,184 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package boltdb implements a store.KVStore on top of BoltDB. It supports the +// following options: +// +// "bucket" (string): the name of BoltDB bucket to use, defaults to "bleve". +// +// "nosync" (bool): if true, set boltdb.DB.NoSync to true. It speeds up index +// operations in exchange of losing integrity guarantees if indexation aborts +// without closing the index. Use it when rebuilding indexes from zero. +package boltdb + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + + "github.com/blevesearch/bleve/v2/registry" + store "github.com/blevesearch/upsidedown_store_api" + bolt "go.etcd.io/bbolt" +) + +const ( + Name = "boltdb" + defaultCompactBatchSize = 100 +) + +type Store struct { + path string + bucket string + db *bolt.DB + noSync bool + fillPercent float64 + mo store.MergeOperator +} + +func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) { + path, ok := config["path"].(string) + if !ok { + return nil, fmt.Errorf("must specify path") + } + if path == "" { + return nil, os.ErrInvalid + } + + bucket, ok := config["bucket"].(string) + if !ok { + bucket = "bleve" + } + + noSync, _ := config["nosync"].(bool) + + fillPercent, ok := config["fillPercent"].(float64) + if !ok { + fillPercent = bolt.DefaultFillPercent + } + + bo := &bolt.Options{} + ro, ok := config["read_only"].(bool) + if ok { + bo.ReadOnly = ro + } + + if initialMmapSize, ok := config["initialMmapSize"].(int); ok { + bo.InitialMmapSize = initialMmapSize + } else if initialMmapSize, ok := config["initialMmapSize"].(float64); ok { + bo.InitialMmapSize = int(initialMmapSize) + } + + db, err := bolt.Open(path, 0600, bo) + if err != nil { + return nil, err + } + db.NoSync = noSync + + if !bo.ReadOnly { + err = db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte(bucket)) + + return err + }) + if err != nil { + return nil, err + } + } + + rv := Store{ + path: path, + bucket: bucket, + db: db, + mo: mo, + noSync: noSync, + fillPercent: fillPercent, + } + return &rv, nil +} + +func (bs *Store) Close() error { + return bs.db.Close() +} + +func (bs *Store) Reader() (store.KVReader, error) { + tx, err := bs.db.Begin(false) + if err != nil { + return nil, err + } + return &Reader{ + store: bs, + tx: tx, + bucket: tx.Bucket([]byte(bs.bucket)), + }, nil +} + +func (bs *Store) Writer() (store.KVWriter, error) { + return &Writer{ + store: bs, + }, nil +} + +func (bs *Store) Stats() json.Marshaler { + return &stats{ + s: bs, + } +} + +// CompactWithBatchSize removes DictionaryTerm entries with a count of zero (in batchSize batches) +// Removing entries is a workaround for github issue #374. +func (bs *Store) CompactWithBatchSize(batchSize int) error { + for { + cnt := 0 + err := bs.db.Batch(func(tx *bolt.Tx) error { + c := tx.Bucket([]byte(bs.bucket)).Cursor() + prefix := []byte("d") + + for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() { + if bytes.Equal(v, []byte{0}) { + cnt++ + if err := c.Delete(); err != nil { + return err + } + if cnt == batchSize { + break + } + } + + } + return nil + }) + if err != nil { + return err + } + + if cnt == 0 { + break + } + } + return nil +} + +// Compact calls CompactWithBatchSize with a default batch size of 100. This is a workaround +// for github issue #374. +func (bs *Store) Compact() error { + return bs.CompactWithBatchSize(defaultCompactBatchSize) +} + +func init() { + err := registry.RegisterKVStore(Name, New) + if err != nil { + panic(err) + } +} diff --git a/index/upsidedown/store/boltdb/store_test.go b/index/upsidedown/store/boltdb/store_test.go new file mode 100644 index 0000000..6011701 --- /dev/null +++ b/index/upsidedown/store/boltdb/store_test.go @@ -0,0 +1,148 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !darwin || !arm64 + +package boltdb + +import ( + "os" + "testing" + + store "github.com/blevesearch/upsidedown_store_api" + "github.com/blevesearch/upsidedown_store_api/test" + bolt "go.etcd.io/bbolt" +) + +func open(t *testing.T, mo store.MergeOperator) store.KVStore { + rv, err := New(mo, map[string]interface{}{"path": "test"}) + if err != nil { + t.Fatal(err) + } + return rv +} + +func cleanup(t *testing.T, s store.KVStore) { + err := s.Close() + if err != nil { + t.Fatal(err) + } + err = os.RemoveAll("test") + if err != nil { + t.Fatal(err) + } +} + +func TestBoltDBKVCrud(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestKVCrud(t, s) +} + +func TestBoltDBReaderIsolation(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestReaderIsolation(t, s) +} + +func TestBoltDBReaderOwnsGetBytes(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestReaderOwnsGetBytes(t, s) +} + +func TestBoltDBWriterOwnsBytes(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestWriterOwnsBytes(t, s) +} + +func TestBoltDBPrefixIterator(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestPrefixIterator(t, s) +} + +func TestBoltDBPrefixIteratorSeek(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestPrefixIteratorSeek(t, s) +} + +func TestBoltDBRangeIterator(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestRangeIterator(t, s) +} + +func TestBoltDBRangeIteratorSeek(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestRangeIteratorSeek(t, s) +} + +func TestBoltDBMerge(t *testing.T) { + s := open(t, &test.TestMergeCounter{}) + defer cleanup(t, s) + test.CommonTestMerge(t, s) +} + +func TestBoltDBConfig(t *testing.T) { + var tests = []struct { + in map[string]interface{} + path string + bucket string + noSync bool + fillPercent float64 + }{ + { + map[string]interface{}{"path": "test", "bucket": "mybucket", "nosync": true, "fillPercent": 0.75}, + "test", + "mybucket", + true, + 0.75, + }, + { + map[string]interface{}{"path": "test"}, + "test", + "bleve", + false, + bolt.DefaultFillPercent, + }, + } + + for _, test := range tests { + kv, err := New(nil, test.in) + if err != nil { + t.Fatal(err) + } + bs, ok := kv.(*Store) + if !ok { + t.Fatal("failed type assertion to *boltdb.Store") + } + if bs.path != test.path { + t.Fatalf("path: expected %q, got %q", test.path, bs.path) + } + if bs.bucket != test.bucket { + t.Fatalf("bucket: expected %q, got %q", test.bucket, bs.bucket) + } + if bs.noSync != test.noSync { + t.Fatalf("noSync: expected %t, got %t", test.noSync, bs.noSync) + } + if bs.fillPercent != test.fillPercent { + t.Fatalf("fillPercent: expected %f, got %f", test.fillPercent, bs.fillPercent) + } + cleanup(t, kv) + } +} diff --git a/index/upsidedown/store/boltdb/writer.go b/index/upsidedown/store/boltdb/writer.go new file mode 100644 index 0000000..c25583c --- /dev/null +++ b/index/upsidedown/store/boltdb/writer.go @@ -0,0 +1,95 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package boltdb + +import ( + "fmt" + + store "github.com/blevesearch/upsidedown_store_api" +) + +type Writer struct { + store *Store +} + +func (w *Writer) NewBatch() store.KVBatch { + return store.NewEmulatedBatch(w.store.mo) +} + +func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) { + return make([]byte, options.TotalBytes), w.NewBatch(), nil +} + +func (w *Writer) ExecuteBatch(batch store.KVBatch) (err error) { + + emulatedBatch, ok := batch.(*store.EmulatedBatch) + if !ok { + return fmt.Errorf("wrong type of batch") + } + + tx, err := w.store.db.Begin(true) + if err != nil { + return + } + // defer function to ensure that once started, + // we either Commit tx or Rollback + defer func() { + // if nothing went wrong, commit + if err == nil { + // careful to catch error here too + err = tx.Commit() + } else { + // caller should see error that caused abort, + // not success or failure of Rollback itself + _ = tx.Rollback() + } + }() + + bucket := tx.Bucket([]byte(w.store.bucket)) + bucket.FillPercent = w.store.fillPercent + + for k, mergeOps := range emulatedBatch.Merger.Merges { + kb := []byte(k) + existingVal := bucket.Get(kb) + mergedVal, fullMergeOk := w.store.mo.FullMerge(kb, existingVal, mergeOps) + if !fullMergeOk { + err = fmt.Errorf("merge operator returned failure") + return + } + err = bucket.Put(kb, mergedVal) + if err != nil { + return + } + } + + for _, op := range emulatedBatch.Ops { + if op.V != nil { + err = bucket.Put(op.K, op.V) + if err != nil { + return + } + } else { + err = bucket.Delete(op.K) + if err != nil { + return + } + } + } + return +} + +func (w *Writer) Close() error { + return nil +} diff --git a/index/upsidedown/store/goleveldb/batch.go b/index/upsidedown/store/goleveldb/batch.go new file mode 100644 index 0000000..1536588 --- /dev/null +++ b/index/upsidedown/store/goleveldb/batch.go @@ -0,0 +1,50 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package goleveldb + +import ( + "github.com/blevesearch/goleveldb/leveldb" + store "github.com/blevesearch/upsidedown_store_api" +) + +type Batch struct { + store *Store + merge *store.EmulatedMerge + batch *leveldb.Batch +} + +func (b *Batch) Set(key, val []byte) { + b.batch.Put(key, val) +} + +func (b *Batch) Delete(key []byte) { + b.batch.Delete(key) +} + +func (b *Batch) Merge(key, val []byte) { + b.merge.Merge(key, val) +} + +func (b *Batch) Reset() { + b.batch.Reset() + b.merge = store.NewEmulatedMerge(b.store.mo) +} + +func (b *Batch) Close() error { + b.batch.Reset() + b.batch = nil + b.merge = nil + return nil +} diff --git a/index/upsidedown/store/goleveldb/config.go b/index/upsidedown/store/goleveldb/config.go new file mode 100644 index 0000000..376db83 --- /dev/null +++ b/index/upsidedown/store/goleveldb/config.go @@ -0,0 +1,66 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package goleveldb + +import ( + "github.com/blevesearch/goleveldb/leveldb/filter" + "github.com/blevesearch/goleveldb/leveldb/opt" +) + +func applyConfig(o *opt.Options, config map[string]interface{}) (*opt.Options, error) { + + ro, ok := config["read_only"].(bool) + if ok { + o.ReadOnly = ro + } + + cim, ok := config["create_if_missing"].(bool) + if ok { + o.ErrorIfMissing = !cim + } + + eie, ok := config["error_if_exists"].(bool) + if ok { + o.ErrorIfExist = eie + } + + wbs, ok := config["write_buffer_size"].(float64) + if ok { + o.WriteBuffer = int(wbs) + } + + bs, ok := config["block_size"].(float64) + if ok { + o.BlockSize = int(bs) + } + + bri, ok := config["block_restart_interval"].(float64) + if ok { + o.BlockRestartInterval = int(bri) + } + + lcc, ok := config["lru_cache_capacity"].(float64) + if ok { + o.BlockCacheCapacity = int(lcc) + } + + bfbpk, ok := config["bloom_filter_bits_per_key"].(float64) + if ok { + bf := filter.NewBloomFilter(int(bfbpk)) + o.Filter = bf + } + + return o, nil +} diff --git a/index/upsidedown/store/goleveldb/iterator.go b/index/upsidedown/store/goleveldb/iterator.go new file mode 100644 index 0000000..e8e002d --- /dev/null +++ b/index/upsidedown/store/goleveldb/iterator.go @@ -0,0 +1,54 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package goleveldb + +import "github.com/blevesearch/goleveldb/leveldb/iterator" + +type Iterator struct { + store *Store + iterator iterator.Iterator +} + +func (ldi *Iterator) Seek(key []byte) { + ldi.iterator.Seek(key) +} + +func (ldi *Iterator) Next() { + ldi.iterator.Next() +} + +func (ldi *Iterator) Current() ([]byte, []byte, bool) { + if ldi.Valid() { + return ldi.Key(), ldi.Value(), true + } + return nil, nil, false +} + +func (ldi *Iterator) Key() []byte { + return ldi.iterator.Key() +} + +func (ldi *Iterator) Value() []byte { + return ldi.iterator.Value() +} + +func (ldi *Iterator) Valid() bool { + return ldi.iterator.Valid() +} + +func (ldi *Iterator) Close() error { + ldi.iterator.Release() + return nil +} diff --git a/index/upsidedown/store/goleveldb/reader.go b/index/upsidedown/store/goleveldb/reader.go new file mode 100644 index 0000000..9fa3a0e --- /dev/null +++ b/index/upsidedown/store/goleveldb/reader.go @@ -0,0 +1,68 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package goleveldb + +import ( + "github.com/blevesearch/goleveldb/leveldb" + "github.com/blevesearch/goleveldb/leveldb/util" + store "github.com/blevesearch/upsidedown_store_api" +) + +type Reader struct { + store *Store + snapshot *leveldb.Snapshot +} + +func (r *Reader) Get(key []byte) ([]byte, error) { + b, err := r.snapshot.Get(key, r.store.defaultReadOptions) + if err == leveldb.ErrNotFound { + return nil, nil + } + return b, err +} + +func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) { + return store.MultiGet(r, keys) +} + +func (r *Reader) PrefixIterator(prefix []byte) store.KVIterator { + byteRange := util.BytesPrefix(prefix) + iter := r.snapshot.NewIterator(byteRange, r.store.defaultReadOptions) + iter.First() + rv := Iterator{ + store: r.store, + iterator: iter, + } + return &rv +} + +func (r *Reader) RangeIterator(start, end []byte) store.KVIterator { + byteRange := &util.Range{ + Start: start, + Limit: end, + } + iter := r.snapshot.NewIterator(byteRange, r.store.defaultReadOptions) + iter.First() + rv := Iterator{ + store: r.store, + iterator: iter, + } + return &rv +} + +func (r *Reader) Close() error { + r.snapshot.Release() + return nil +} diff --git a/index/upsidedown/store/goleveldb/store.go b/index/upsidedown/store/goleveldb/store.go new file mode 100644 index 0000000..3607c6d --- /dev/null +++ b/index/upsidedown/store/goleveldb/store.go @@ -0,0 +1,152 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package goleveldb + +import ( + "bytes" + "fmt" + "os" + + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/goleveldb/leveldb" + "github.com/blevesearch/goleveldb/leveldb/opt" + "github.com/blevesearch/goleveldb/leveldb/util" + store "github.com/blevesearch/upsidedown_store_api" +) + +const ( + Name = "goleveldb" + defaultCompactBatchSize = 250 +) + +type Store struct { + path string + opts *opt.Options + db *leveldb.DB + mo store.MergeOperator + + defaultWriteOptions *opt.WriteOptions + defaultReadOptions *opt.ReadOptions +} + +func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) { + + path, ok := config["path"].(string) + if !ok { + return nil, fmt.Errorf("must specify path") + } + if path == "" { + return nil, os.ErrInvalid + } + + opts, err := applyConfig(&opt.Options{}, config) + if err != nil { + return nil, err + } + + db, err := leveldb.OpenFile(path, opts) + if err != nil { + return nil, err + } + + rv := Store{ + path: path, + opts: opts, + db: db, + mo: mo, + defaultReadOptions: &opt.ReadOptions{}, + defaultWriteOptions: &opt.WriteOptions{}, + } + rv.defaultWriteOptions.Sync = true + return &rv, nil +} + +func (ldbs *Store) Close() error { + return ldbs.db.Close() +} + +func (ldbs *Store) Reader() (store.KVReader, error) { + snapshot, _ := ldbs.db.GetSnapshot() + return &Reader{ + store: ldbs, + snapshot: snapshot, + }, nil +} + +func (ldbs *Store) Writer() (store.KVWriter, error) { + return &Writer{ + store: ldbs, + }, nil +} + +// CompactWithBatchSize removes DictionaryTerm entries with a count of zero (in batchSize batches), then +// compacts the underlying goleveldb store. Removing entries is a workaround for github issue #374. +func (ldbs *Store) CompactWithBatchSize(batchSize int) error { + // workaround for github issue #374 - remove DictionaryTerm keys with count=0 + batch := &leveldb.Batch{} + for { + t, err := ldbs.db.OpenTransaction() + if err != nil { + return err + } + iter := t.NewIterator(util.BytesPrefix([]byte("d")), ldbs.defaultReadOptions) + + for iter.Next() { + if bytes.Equal(iter.Value(), []byte{0}) { + k := append([]byte{}, iter.Key()...) + batch.Delete(k) + } + if batch.Len() == batchSize { + break + } + } + iter.Release() + if iter.Error() != nil { + t.Discard() + return iter.Error() + } + + if batch.Len() > 0 { + err := t.Write(batch, ldbs.defaultWriteOptions) + if err != nil { + t.Discard() + return err + } + err = t.Commit() + if err != nil { + return err + } + } else { + t.Discard() + break + } + batch.Reset() + } + + return ldbs.db.CompactRange(util.Range{Start: nil, Limit: nil}) +} + +// Compact compacts the underlying goleveldb store. The current implementation includes a workaround +// for github issue #374 (see CompactWithBatchSize). +func (ldbs *Store) Compact() error { + return ldbs.CompactWithBatchSize(defaultCompactBatchSize) +} + +func init() { + err := registry.RegisterKVStore(Name, New) + if err != nil { + panic(err) + } +} diff --git a/index/upsidedown/store/goleveldb/store_test.go b/index/upsidedown/store/goleveldb/store_test.go new file mode 100644 index 0000000..c9e5d4d --- /dev/null +++ b/index/upsidedown/store/goleveldb/store_test.go @@ -0,0 +1,99 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package goleveldb + +import ( + "os" + "testing" + + store "github.com/blevesearch/upsidedown_store_api" + "github.com/blevesearch/upsidedown_store_api/test" +) + +func open(t *testing.T, mo store.MergeOperator) store.KVStore { + rv, err := New(mo, map[string]interface{}{ + "path": "test", + "create_if_missing": true, + }) + if err != nil { + t.Fatal(err) + } + return rv +} + +func cleanup(t *testing.T, s store.KVStore) { + err := s.Close() + if err != nil { + t.Fatal(err) + } + err = os.RemoveAll("test") + if err != nil { + t.Fatal(err) + } +} + +func TestGoLevelDBKVCrud(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestKVCrud(t, s) +} + +func TestGoLevelDBReaderIsolation(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestReaderIsolation(t, s) +} + +func TestGoLevelDBReaderOwnsGetBytes(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestReaderOwnsGetBytes(t, s) +} + +func TestGoLevelDBWriterOwnsBytes(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestWriterOwnsBytes(t, s) +} + +func TestGoLevelDBPrefixIterator(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestPrefixIterator(t, s) +} + +func TestGoLevelDBPrefixIteratorSeek(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestPrefixIteratorSeek(t, s) +} + +func TestGoLevelDBRangeIterator(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestRangeIterator(t, s) +} + +func TestGoLevelDBRangeIteratorSeek(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestRangeIteratorSeek(t, s) +} + +func TestGoLevelDBMerge(t *testing.T) { + s := open(t, &test.TestMergeCounter{}) + defer cleanup(t, s) + test.CommonTestMerge(t, s) +} diff --git a/index/upsidedown/store/goleveldb/writer.go b/index/upsidedown/store/goleveldb/writer.go new file mode 100644 index 0000000..8dfd30c --- /dev/null +++ b/index/upsidedown/store/goleveldb/writer.go @@ -0,0 +1,68 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package goleveldb + +import ( + "fmt" + + "github.com/blevesearch/goleveldb/leveldb" + store "github.com/blevesearch/upsidedown_store_api" +) + +type Writer struct { + store *Store +} + +func (w *Writer) NewBatch() store.KVBatch { + rv := Batch{ + store: w.store, + merge: store.NewEmulatedMerge(w.store.mo), + batch: new(leveldb.Batch), + } + return &rv +} + +func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) { + return make([]byte, options.TotalBytes), w.NewBatch(), nil +} + +func (w *Writer) ExecuteBatch(b store.KVBatch) error { + batch, ok := b.(*Batch) + if !ok { + return fmt.Errorf("wrong type of batch") + } + + // first process merges + for k, mergeOps := range batch.merge.Merges { + kb := []byte(k) + existingVal, err := w.store.db.Get(kb, w.store.defaultReadOptions) + if err != nil && err != leveldb.ErrNotFound { + return err + } + mergedVal, fullMergeOk := w.store.mo.FullMerge(kb, existingVal, mergeOps) + if !fullMergeOk { + return fmt.Errorf("merge operator returned failure") + } + // add the final merge to this batch + batch.batch.Put(kb, mergedVal) + } + + // now execute the batch + return w.store.db.Write(batch.batch, w.store.defaultWriteOptions) +} + +func (w *Writer) Close() error { + return nil +} diff --git a/index/upsidedown/store/gtreap/iterator.go b/index/upsidedown/store/gtreap/iterator.go new file mode 100644 index 0000000..c03f75c --- /dev/null +++ b/index/upsidedown/store/gtreap/iterator.go @@ -0,0 +1,152 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gtreap provides an in-memory implementation of the +// KVStore interfaces using the gtreap balanced-binary treap, +// copy-on-write data structure. +package gtreap + +import ( + "bytes" + "sync" + + "github.com/blevesearch/gtreap" +) + +type Iterator struct { + t *gtreap.Treap + + m sync.Mutex + cancelCh chan struct{} + nextCh chan *Item + curr *Item + currOk bool + + prefix []byte + start []byte + end []byte +} + +func (w *Iterator) Seek(k []byte) { + if w.start != nil && bytes.Compare(k, w.start) < 0 { + k = w.start + } + if w.prefix != nil && !bytes.HasPrefix(k, w.prefix) { + if bytes.Compare(k, w.prefix) < 0 { + k = w.prefix + } else { + var end []byte + for i := len(w.prefix) - 1; i >= 0; i-- { + c := w.prefix[i] + if c < 0xff { + end = make([]byte, i+1) + copy(end, w.prefix) + end[i] = c + 1 + break + } + } + k = end + } + } + w.restart(&Item{k: k}) +} + +func (w *Iterator) restart(start *Item) *Iterator { + cancelCh := make(chan struct{}) + nextCh := make(chan *Item, 1) + + w.m.Lock() + if w.cancelCh != nil { + close(w.cancelCh) + } + w.cancelCh = cancelCh + w.nextCh = nextCh + w.curr = nil + w.currOk = false + w.m.Unlock() + + go func() { + if start != nil { + w.t.VisitAscend(start, func(itm gtreap.Item) bool { + select { + case <-cancelCh: + return false + case nextCh <- itm.(*Item): + return true + } + }) + } + close(nextCh) + }() + + w.Next() + + return w +} + +func (w *Iterator) Next() { + w.m.Lock() + nextCh := w.nextCh + w.m.Unlock() + w.curr, w.currOk = <-nextCh +} + +func (w *Iterator) Current() ([]byte, []byte, bool) { + w.m.Lock() + defer w.m.Unlock() + if !w.currOk || w.curr == nil { + return nil, nil, false + } + if w.prefix != nil && !bytes.HasPrefix(w.curr.k, w.prefix) { + return nil, nil, false + } else if w.end != nil && bytes.Compare(w.curr.k, w.end) >= 0 { + return nil, nil, false + } + return w.curr.k, w.curr.v, w.currOk +} + +func (w *Iterator) Key() []byte { + k, _, ok := w.Current() + if !ok { + return nil + } + return k +} + +func (w *Iterator) Value() []byte { + _, v, ok := w.Current() + if !ok { + return nil + } + return v +} + +func (w *Iterator) Valid() bool { + _, _, ok := w.Current() + return ok +} + +func (w *Iterator) Close() error { + w.m.Lock() + if w.cancelCh != nil { + close(w.cancelCh) + } + w.cancelCh = nil + w.nextCh = nil + w.curr = nil + w.currOk = false + w.m.Unlock() + + return nil +} diff --git a/index/upsidedown/store/gtreap/reader.go b/index/upsidedown/store/gtreap/reader.go new file mode 100644 index 0000000..52b05d7 --- /dev/null +++ b/index/upsidedown/store/gtreap/reader.go @@ -0,0 +1,66 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gtreap provides an in-memory implementation of the +// KVStore interfaces using the gtreap balanced-binary treap, +// copy-on-write data structure. +package gtreap + +import ( + "github.com/blevesearch/upsidedown_store_api" + + "github.com/blevesearch/gtreap" +) + +type Reader struct { + t *gtreap.Treap +} + +func (w *Reader) Get(k []byte) (v []byte, err error) { + var rv []byte + itm := w.t.Get(&Item{k: k}) + if itm != nil { + rv = make([]byte, len(itm.(*Item).v)) + copy(rv, itm.(*Item).v) + return rv, nil + } + return nil, nil +} + +func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) { + return store.MultiGet(r, keys) +} + +func (w *Reader) PrefixIterator(k []byte) store.KVIterator { + rv := Iterator{ + t: w.t, + prefix: k, + } + rv.restart(&Item{k: k}) + return &rv +} + +func (w *Reader) RangeIterator(start, end []byte) store.KVIterator { + rv := Iterator{ + t: w.t, + start: start, + end: end, + } + rv.restart(&Item{k: start}) + return &rv +} + +func (w *Reader) Close() error { + return nil +} diff --git a/index/upsidedown/store/gtreap/store.go b/index/upsidedown/store/gtreap/store.go new file mode 100644 index 0000000..8050e4d --- /dev/null +++ b/index/upsidedown/store/gtreap/store.go @@ -0,0 +1,85 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gtreap provides an in-memory implementation of the +// KVStore interfaces using the gtreap balanced-binary treap, +// copy-on-write data structure. + +package gtreap + +import ( + "bytes" + "fmt" + "os" + "sync" + + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/gtreap" + store "github.com/blevesearch/upsidedown_store_api" +) + +const Name = "gtreap" + +type Store struct { + m sync.Mutex + t *gtreap.Treap + mo store.MergeOperator +} + +type Item struct { + k []byte + v []byte +} + +func itemCompare(a, b interface{}) int { + return bytes.Compare(a.(*Item).k, b.(*Item).k) +} + +func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) { + path, ok := config["path"].(string) + if !ok { + return nil, fmt.Errorf("must specify path") + } + if path != "" { + return nil, os.ErrInvalid + } + + rv := Store{ + t: gtreap.NewTreap(itemCompare), + mo: mo, + } + return &rv, nil +} + +func (s *Store) Close() error { + return nil +} + +func (s *Store) Reader() (store.KVReader, error) { + s.m.Lock() + t := s.t + s.m.Unlock() + return &Reader{t: t}, nil +} + +func (s *Store) Writer() (store.KVWriter, error) { + return &Writer{s: s}, nil +} + +func init() { + err := registry.RegisterKVStore(Name, New) + if err != nil { + panic(err) + } +} diff --git a/index/upsidedown/store/gtreap/store_test.go b/index/upsidedown/store/gtreap/store_test.go new file mode 100644 index 0000000..2fc3aa8 --- /dev/null +++ b/index/upsidedown/store/gtreap/store_test.go @@ -0,0 +1,93 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gtreap + +import ( + "testing" + + store "github.com/blevesearch/upsidedown_store_api" + "github.com/blevesearch/upsidedown_store_api/test" +) + +func open(t *testing.T, mo store.MergeOperator) store.KVStore { + rv, err := New(mo, map[string]interface{}{ + "path": "", + }) + if err != nil { + t.Fatal(err) + } + return rv +} + +func cleanup(t *testing.T, s store.KVStore) { + err := s.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestGTreapKVCrud(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestKVCrud(t, s) +} + +func TestGTreapReaderIsolation(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestReaderIsolation(t, s) +} + +func TestGTreapReaderOwnsGetBytes(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestReaderOwnsGetBytes(t, s) +} + +func TestGTreapWriterOwnsBytes(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestWriterOwnsBytes(t, s) +} + +func TestGTreapPrefixIterator(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestPrefixIterator(t, s) +} + +func TestGTreapPrefixIteratorSeek(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestPrefixIteratorSeek(t, s) +} + +func TestGTreapRangeIterator(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestRangeIterator(t, s) +} + +func TestGTreapRangeIteratorSeek(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestRangeIteratorSeek(t, s) +} + +func TestGTreapMerge(t *testing.T) { + s := open(t, &test.TestMergeCounter{}) + defer cleanup(t, s) + test.CommonTestMerge(t, s) +} diff --git a/index/upsidedown/store/gtreap/writer.go b/index/upsidedown/store/gtreap/writer.go new file mode 100644 index 0000000..80aa15b --- /dev/null +++ b/index/upsidedown/store/gtreap/writer.go @@ -0,0 +1,76 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gtreap provides an in-memory implementation of the +// KVStore interfaces using the gtreap balanced-binary treap, +// copy-on-write data structure. +package gtreap + +import ( + "fmt" + "math/rand" + + "github.com/blevesearch/upsidedown_store_api" +) + +type Writer struct { + s *Store +} + +func (w *Writer) NewBatch() store.KVBatch { + return store.NewEmulatedBatch(w.s.mo) +} + +func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) { + return make([]byte, options.TotalBytes), w.NewBatch(), nil +} + +func (w *Writer) ExecuteBatch(batch store.KVBatch) error { + + emulatedBatch, ok := batch.(*store.EmulatedBatch) + if !ok { + return fmt.Errorf("wrong type of batch") + } + + w.s.m.Lock() + for k, mergeOps := range emulatedBatch.Merger.Merges { + kb := []byte(k) + var existingVal []byte + existingItem := w.s.t.Get(&Item{k: kb}) + if existingItem != nil { + existingVal = w.s.t.Get(&Item{k: kb}).(*Item).v + } + mergedVal, fullMergeOk := w.s.mo.FullMerge(kb, existingVal, mergeOps) + if !fullMergeOk { + return fmt.Errorf("merge operator returned failure") + } + w.s.t = w.s.t.Upsert(&Item{k: kb, v: mergedVal}, rand.Int()) + } + + for _, op := range emulatedBatch.Ops { + if op.V != nil { + w.s.t = w.s.t.Upsert(&Item{k: op.K, v: op.V}, rand.Int()) + } else { + w.s.t = w.s.t.Delete(&Item{k: op.K}) + } + } + w.s.m.Unlock() + + return nil +} + +func (w *Writer) Close() error { + w.s = nil + return nil +} diff --git a/index/upsidedown/store/metrics/batch.go b/index/upsidedown/store/metrics/batch.go new file mode 100644 index 0000000..8293509 --- /dev/null +++ b/index/upsidedown/store/metrics/batch.go @@ -0,0 +1,46 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import store "github.com/blevesearch/upsidedown_store_api" + +type Batch struct { + s *Store + o store.KVBatch +} + +func (b *Batch) Set(key, val []byte) { + b.o.Set(key, val) +} + +func (b *Batch) Delete(key []byte) { + b.o.Delete(key) +} + +func (b *Batch) Merge(key, val []byte) { + b.s.timerBatchMerge.Time(func() { + b.o.Merge(key, val) + }) +} + +func (b *Batch) Reset() { + b.o.Reset() +} + +func (b *Batch) Close() error { + err := b.o.Close() + b.o = nil + return err +} diff --git a/index/upsidedown/store/metrics/iterator.go b/index/upsidedown/store/metrics/iterator.go new file mode 100644 index 0000000..c55d2c3 --- /dev/null +++ b/index/upsidedown/store/metrics/iterator.go @@ -0,0 +1,58 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import store "github.com/blevesearch/upsidedown_store_api" + +type Iterator struct { + s *Store + o store.KVIterator +} + +func (i *Iterator) Seek(x []byte) { + i.s.timerIteratorSeek.Time(func() { + i.o.Seek(x) + }) +} + +func (i *Iterator) Next() { + i.s.timerIteratorNext.Time(func() { + i.o.Next() + }) +} + +func (i *Iterator) Current() ([]byte, []byte, bool) { + return i.o.Current() +} + +func (i *Iterator) Key() []byte { + return i.o.Key() +} + +func (i *Iterator) Value() []byte { + return i.o.Value() +} + +func (i *Iterator) Valid() bool { + return i.o.Valid() +} + +func (i *Iterator) Close() error { + err := i.o.Close() + if err != nil { + i.s.AddError("Iterator.Close", err, nil) + } + return err +} diff --git a/index/upsidedown/store/metrics/metrics_test.go b/index/upsidedown/store/metrics/metrics_test.go new file mode 100644 index 0000000..5ca5fb3 --- /dev/null +++ b/index/upsidedown/store/metrics/metrics_test.go @@ -0,0 +1,141 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "bytes" + "encoding/json" + "fmt" + "testing" + + "github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap" +) + +func TestMetricsStore(t *testing.T) { + _, err := New(nil, map[string]interface{}{}) + if err == nil { + t.Errorf("expected err when bad config") + } + + _, err = New(nil, map[string]interface{}{ + "kvStoreName_actual": "some-invalid-kvstore-name", + }) + if err == nil { + t.Errorf("expected err when unknown kvStoreName_actual") + } + + s, err := New(nil, map[string]interface{}{ + "kvStoreName_actual": gtreap.Name, + "path": "", + }) + if err != nil { + t.Fatal(err) + } + + b := bytes.NewBuffer(nil) + err = s.(*Store).WriteJSON(b) + if err != nil { + t.Fatal(err) + } + if b.Len() <= 0 { + t.Errorf("expected some output from WriteJSON") + } + var m map[string]interface{} + err = json.Unmarshal(b.Bytes(), &m) + if err != nil { + t.Errorf("expected WriteJSON to be unmarshallable") + } + if len(m) == 0 { + t.Errorf("expected some entries") + } + + b = bytes.NewBuffer(nil) + s.(*Store).WriteCSVHeader(b) + if b.Len() <= 0 { + t.Errorf("expected some output from WriteCSVHeader") + } + + b = bytes.NewBuffer(nil) + s.(*Store).WriteCSV(b) + if b.Len() <= 0 { + t.Errorf("expected some output from WriteCSV") + } +} + +func TestErrors(t *testing.T) { + s, err := New(nil, map[string]interface{}{ + "kvStoreName_actual": gtreap.Name, + "path": "", + }) + if err != nil { + t.Fatal(err) + } + + x, ok := s.(*Store) + if !ok { + t.Errorf("expecting a Store") + } + + x.AddError("foo", fmt.Errorf("Foo"), []byte("fooKey")) + x.AddError("bar", fmt.Errorf("Bar"), nil) + x.AddError("baz", fmt.Errorf("Baz"), []byte("bazKey")) + + b := bytes.NewBuffer(nil) + err = x.WriteJSON(b) + if err != nil { + t.Fatal(err) + } + + var m map[string]interface{} + err = json.Unmarshal(b.Bytes(), &m) + if err != nil { + t.Errorf("expected unmarshallable writeJSON, err: %v, b: %s", + err, b.Bytes()) + } + + errorsi, ok := m["Errors"] + if !ok || errorsi == nil { + t.Errorf("expected errorsi") + } + errors, ok := errorsi.([]interface{}) + if !ok || errors == nil { + t.Errorf("expected errorsi is array") + } + if len(errors) != 3 { + t.Errorf("expected errors len 3") + } + + e := errors[0].(map[string]interface{}) + if e["Op"].(string) != "foo" || + e["Err"].(string) != "Foo" || + len(e["Time"].(string)) < 10 || + e["Key"].(string) != "fooKey" { + t.Errorf("expected foo, %#v", e) + } + e = errors[1].(map[string]interface{}) + if e["Op"].(string) != "bar" || + e["Err"].(string) != "Bar" || + len(e["Time"].(string)) < 10 || + e["Key"].(string) != "" { + t.Errorf("expected bar, %#v", e) + } + e = errors[2].(map[string]interface{}) + if e["Op"].(string) != "baz" || + e["Err"].(string) != "Baz" || + len(e["Time"].(string)) < 10 || + e["Key"].(string) != "bazKey" { + t.Errorf("expected baz, %#v", e) + } +} diff --git a/index/upsidedown/store/metrics/reader.go b/index/upsidedown/store/metrics/reader.go new file mode 100644 index 0000000..ad1c25a --- /dev/null +++ b/index/upsidedown/store/metrics/reader.go @@ -0,0 +1,64 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import store "github.com/blevesearch/upsidedown_store_api" + +type Reader struct { + s *Store + o store.KVReader +} + +func (r *Reader) Get(key []byte) (v []byte, err error) { + r.s.timerReaderGet.Time(func() { + v, err = r.o.Get(key) + if err != nil { + r.s.AddError("Reader.Get", err, key) + } + }) + return +} + +func (r *Reader) MultiGet(keys [][]byte) (vals [][]byte, err error) { + r.s.timerReaderMultiGet.Time(func() { + vals, err = r.o.MultiGet(keys) + if err != nil { + r.s.AddError("Reader.MultiGet", err, nil) + } + }) + return +} + +func (r *Reader) PrefixIterator(prefix []byte) (i store.KVIterator) { + r.s.timerReaderPrefixIterator.Time(func() { + i = &Iterator{s: r.s, o: r.o.PrefixIterator(prefix)} + }) + return +} + +func (r *Reader) RangeIterator(start, end []byte) (i store.KVIterator) { + r.s.timerReaderRangeIterator.Time(func() { + i = &Iterator{s: r.s, o: r.o.RangeIterator(start, end)} + }) + return +} + +func (r *Reader) Close() error { + err := r.o.Close() + if err != nil { + r.s.AddError("Reader.Close", err, nil) + } + return err +} diff --git a/index/upsidedown/store/metrics/stats.go b/index/upsidedown/store/metrics/stats.go new file mode 100644 index 0000000..4888192 --- /dev/null +++ b/index/upsidedown/store/metrics/stats.go @@ -0,0 +1,50 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "github.com/blevesearch/bleve/v2/util" + store "github.com/blevesearch/upsidedown_store_api" +) + +type stats struct { + s *Store +} + +func (s *stats) statsMap() map[string]interface{} { + ms := map[string]interface{}{} + + ms["metrics"] = map[string]interface{}{ + "reader_get": TimerMap(s.s.timerReaderGet), + "reader_multi_get": TimerMap(s.s.timerReaderMultiGet), + "reader_prefix_iterator": TimerMap(s.s.timerReaderPrefixIterator), + "reader_range_iterator": TimerMap(s.s.timerReaderRangeIterator), + "writer_execute_batch": TimerMap(s.s.timerWriterExecuteBatch), + "iterator_seek": TimerMap(s.s.timerIteratorSeek), + "iterator_next": TimerMap(s.s.timerIteratorNext), + "batch_merge": TimerMap(s.s.timerBatchMerge), + } + + if o, ok := s.s.o.(store.KVStoreStats); ok { + ms["kv"] = o.StatsMap() + } + + return ms +} + +func (s *stats) MarshalJSON() ([]byte, error) { + m := s.statsMap() + return util.MarshalJSON(m) +} diff --git a/index/upsidedown/store/metrics/store.go b/index/upsidedown/store/metrics/store.go new file mode 100644 index 0000000..9a89756 --- /dev/null +++ b/index/upsidedown/store/metrics/store.go @@ -0,0 +1,277 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package metrics provides a bleve.store.KVStore implementation that +// wraps another, real KVStore implementation, and uses go-metrics to +// track runtime performance metrics. +package metrics + +import ( + "container/list" + "encoding/json" + "fmt" + "io" + "sync" + "time" + + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/util" + "github.com/blevesearch/go-metrics" + store "github.com/blevesearch/upsidedown_store_api" +) + +const Name = "metrics" + +type Store struct { + o store.KVStore + + timerReaderGet metrics.Timer + timerReaderMultiGet metrics.Timer + timerReaderPrefixIterator metrics.Timer + timerReaderRangeIterator metrics.Timer + timerWriterExecuteBatch metrics.Timer + timerIteratorSeek metrics.Timer + timerIteratorNext metrics.Timer + timerBatchMerge metrics.Timer + + m sync.Mutex // Protects the fields that follow. + errors *list.List // Capped list of StoreError's. + + s *stats +} + +func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) { + + name, ok := config["kvStoreName_actual"].(string) + if !ok || name == "" { + return nil, fmt.Errorf("metrics: missing kvStoreName_actual,"+ + " config: %#v", config) + } + + if name == Name { + return nil, fmt.Errorf("metrics: circular kvStoreName_actual") + } + + ctr := registry.KVStoreConstructorByName(name) + if ctr == nil { + return nil, fmt.Errorf("metrics: no kv store constructor,"+ + " kvStoreName_actual: %s", name) + } + + kvs, err := ctr(mo, config) + if err != nil { + return nil, err + } + + rv := &Store{ + o: kvs, + + timerReaderGet: metrics.NewTimer(), + timerReaderMultiGet: metrics.NewTimer(), + timerReaderPrefixIterator: metrics.NewTimer(), + timerReaderRangeIterator: metrics.NewTimer(), + timerWriterExecuteBatch: metrics.NewTimer(), + timerIteratorSeek: metrics.NewTimer(), + timerIteratorNext: metrics.NewTimer(), + timerBatchMerge: metrics.NewTimer(), + + errors: list.New(), + } + + rv.s = &stats{s: rv} + + return rv, nil +} + +func init() { + err := registry.RegisterKVStore(Name, New) + if err != nil { + panic(err) + } +} + +func (s *Store) Close() error { + return s.o.Close() +} + +func (s *Store) Reader() (store.KVReader, error) { + o, err := s.o.Reader() + if err != nil { + s.AddError("Reader", err, nil) + return nil, err + } + return &Reader{s: s, o: o}, nil +} + +func (s *Store) Writer() (store.KVWriter, error) { + o, err := s.o.Writer() + if err != nil { + s.AddError("Writer", err, nil) + return nil, err + } + return &Writer{s: s, o: o}, nil +} + +// Metric specific code below: + +const MaxErrors = 100 + +type StoreError struct { + Time string + Op string + Err string + Key string +} + +func (s *Store) AddError(op string, err error, key []byte) { + e := &StoreError{ + Time: time.Now().Format(time.RFC3339Nano), + Op: op, + Err: fmt.Sprintf("%v", err), + Key: string(key), + } + + s.m.Lock() + for s.errors.Len() >= MaxErrors { + s.errors.Remove(s.errors.Front()) + } + s.errors.PushBack(e) + s.m.Unlock() +} + +func (s *Store) WriteJSON(w io.Writer) (err error) { + _, err = w.Write([]byte(`{"TimerReaderGet":`)) + if err != nil { + return + } + WriteTimerJSON(w, s.timerReaderGet) + _, err = w.Write([]byte(`,"TimerReaderMultiGet":`)) + if err != nil { + return + } + WriteTimerJSON(w, s.timerReaderMultiGet) + _, err = w.Write([]byte(`,"TimerReaderPrefixIterator":`)) + if err != nil { + return + } + WriteTimerJSON(w, s.timerReaderPrefixIterator) + _, err = w.Write([]byte(`,"TimerReaderRangeIterator":`)) + if err != nil { + return + } + WriteTimerJSON(w, s.timerReaderRangeIterator) + _, err = w.Write([]byte(`,"TimerWriterExecuteBatch":`)) + if err != nil { + return + } + WriteTimerJSON(w, s.timerWriterExecuteBatch) + _, err = w.Write([]byte(`,"TimerIteratorSeek":`)) + if err != nil { + return + } + WriteTimerJSON(w, s.timerIteratorSeek) + _, err = w.Write([]byte(`,"TimerIteratorNext":`)) + if err != nil { + return + } + WriteTimerJSON(w, s.timerIteratorNext) + _, err = w.Write([]byte(`,"TimerBatchMerge":`)) + if err != nil { + return + } + WriteTimerJSON(w, s.timerBatchMerge) + + _, err = w.Write([]byte(`,"Errors":[`)) + if err != nil { + return + } + s.m.Lock() + defer s.m.Unlock() + e := s.errors.Front() + i := 0 + for e != nil { + se, ok := e.Value.(*StoreError) + if ok && se != nil { + if i > 0 { + _, err = w.Write([]byte(",")) + if err != nil { + return + } + } + var buf []byte + buf, err = util.MarshalJSON(se) + if err == nil { + _, err = w.Write(buf) + if err != nil { + return + } + } + } + e = e.Next() + i = i + 1 + } + _, err = w.Write([]byte(`]`)) + if err != nil { + return + } + + // see if the underlying implementation has its own stats + if o, ok := s.o.(store.KVStoreStats); ok { + storeStats := o.Stats() + var storeBytes []byte + storeBytes, err = util.MarshalJSON(storeStats) + if err != nil { + return + } + _, err = fmt.Fprintf(w, `, "store": %s`, string(storeBytes)) + if err != nil { + return + } + } + + _, err = w.Write([]byte(`}`)) + if err != nil { + return + } + + return +} + +func (s *Store) WriteCSVHeader(w io.Writer) { + WriteTimerCSVHeader(w, "TimerReaderGet") + WriteTimerCSVHeader(w, "TimerReaderPrefixIterator") + WriteTimerCSVHeader(w, "TimerReaderRangeIterator") + WriteTimerCSVHeader(w, "TimerWtierExecuteBatch") + WriteTimerCSVHeader(w, "TimerIteratorSeek") + WriteTimerCSVHeader(w, "TimerIteratorNext") + WriteTimerCSVHeader(w, "TimerBatchMerge") +} + +func (s *Store) WriteCSV(w io.Writer) { + WriteTimerCSV(w, s.timerReaderGet) + WriteTimerCSV(w, s.timerReaderPrefixIterator) + WriteTimerCSV(w, s.timerReaderRangeIterator) + WriteTimerCSV(w, s.timerWriterExecuteBatch) + WriteTimerCSV(w, s.timerIteratorSeek) + WriteTimerCSV(w, s.timerIteratorNext) + WriteTimerCSV(w, s.timerBatchMerge) +} + +func (s *Store) Stats() json.Marshaler { + return s.s +} + +func (s *Store) StatsMap() map[string]interface{} { + return s.s.statsMap() +} diff --git a/index/upsidedown/store/metrics/store_test.go b/index/upsidedown/store/metrics/store_test.go new file mode 100644 index 0000000..bdcf8eb --- /dev/null +++ b/index/upsidedown/store/metrics/store_test.go @@ -0,0 +1,95 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "testing" + + "github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap" + store "github.com/blevesearch/upsidedown_store_api" + "github.com/blevesearch/upsidedown_store_api/test" +) + +func open(t *testing.T, mo store.MergeOperator) store.KVStore { + rv, err := New(mo, map[string]interface{}{ + "kvStoreName_actual": gtreap.Name, + "path": "", + }) + if err != nil { + t.Fatal(err) + } + return rv +} + +func cleanup(t *testing.T, s store.KVStore) { + err := s.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestMetricsKVCrud(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestKVCrud(t, s) +} + +func TestMetricsReaderIsolation(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestReaderIsolation(t, s) +} + +func TestMetricsReaderOwnsGetBytes(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestReaderOwnsGetBytes(t, s) +} + +func TestMetricsWriterOwnsBytes(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestWriterOwnsBytes(t, s) +} + +func TestMetricsPrefixIterator(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestPrefixIterator(t, s) +} + +func TestMetricsPrefixIteratorSeek(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestPrefixIteratorSeek(t, s) +} + +func TestMetricsRangeIterator(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestRangeIterator(t, s) +} + +func TestMetricsRangeIteratorSeek(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestRangeIteratorSeek(t, s) +} + +func TestMetricsMerge(t *testing.T) { + s := open(t, &test.TestMergeCounter{}) + defer cleanup(t, s) + test.CommonTestMerge(t, s) +} diff --git a/index/upsidedown/store/metrics/util.go b/index/upsidedown/store/metrics/util.go new file mode 100644 index 0000000..6b41018 --- /dev/null +++ b/index/upsidedown/store/metrics/util.go @@ -0,0 +1,135 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "fmt" + "io" + "math" + + "github.com/blevesearch/go-metrics" +) + +// NOTE: This is copy & pasted from cbft as otherwise there +// would be an import cycle. + +var timerPercentiles = []float64{0.5, 0.75, 0.95, 0.99, 0.999} + +func TimerMap(timer metrics.Timer) map[string]interface{} { + + rv := make(map[string]interface{}) + t := timer.Snapshot() + p := t.Percentiles(timerPercentiles) + + percentileKeys := []string{"median", "75%", "95%", "99%", "99.9%"} + percentiles := make(map[string]interface{}) + for i, pi := range p { + if !isNanOrInf(pi) { + percentileKey := percentileKeys[i] + percentiles[percentileKey] = pi + } + } + + rateKeys := []string{"1-min", "5-min", "15-min", "mean"} + rates := make(map[string]interface{}) + for i, ri := range []float64{t.Rate1(), t.Rate5(), t.Rate15(), t.RateMean()} { + if !isNanOrInf(ri) { + rateKey := rateKeys[i] + rates[rateKey] = ri + } + } + + rv["count"] = t.Count() + rv["min"] = t.Min() + rv["max"] = t.Max() + mean := t.Mean() + if !isNanOrInf(mean) { + rv["mean"] = mean + } + stddev := t.StdDev() + if !isNanOrInf(stddev) { + rv["stddev"] = stddev + } + rv["percentiles"] = percentiles + rv["rates"] = rates + + return rv +} + +func isNanOrInf(v float64) bool { + if math.IsNaN(v) || math.IsInf(v, 0) { + return true + } + return false +} + +func WriteTimerJSON(w io.Writer, timer metrics.Timer) { + t := timer.Snapshot() + p := t.Percentiles(timerPercentiles) + + fmt.Fprintf(w, `{"count":%9d,`, t.Count()) + fmt.Fprintf(w, `"min":%9d,`, t.Min()) + fmt.Fprintf(w, `"max":%9d,`, t.Max()) + fmt.Fprintf(w, `"mean":%12.2f,`, t.Mean()) + fmt.Fprintf(w, `"stddev":%12.2f,`, t.StdDev()) + fmt.Fprintf(w, `"percentiles":{`) + fmt.Fprintf(w, `"median":%12.2f,`, p[0]) + fmt.Fprintf(w, `"75%%":%12.2f,`, p[1]) + fmt.Fprintf(w, `"95%%":%12.2f,`, p[2]) + fmt.Fprintf(w, `"99%%":%12.2f,`, p[3]) + fmt.Fprintf(w, `"99.9%%":%12.2f},`, p[4]) + fmt.Fprintf(w, `"rates":{`) + fmt.Fprintf(w, `"1-min":%12.2f,`, t.Rate1()) + fmt.Fprintf(w, `"5-min":%12.2f,`, t.Rate5()) + fmt.Fprintf(w, `"15-min":%12.2f,`, t.Rate15()) + fmt.Fprintf(w, `"mean":%12.2f}}`, t.RateMean()) +} + +func WriteTimerCSVHeader(w io.Writer, prefix string) { + fmt.Fprintf(w, "%s-count,", prefix) + fmt.Fprintf(w, "%s-min,", prefix) + fmt.Fprintf(w, "%s-max,", prefix) + fmt.Fprintf(w, "%s-mean,", prefix) + fmt.Fprintf(w, "%s-stddev,", prefix) + fmt.Fprintf(w, "%s-percentile-50%%,", prefix) + fmt.Fprintf(w, "%s-percentile-75%%,", prefix) + fmt.Fprintf(w, "%s-percentile-95%%,", prefix) + fmt.Fprintf(w, "%s-percentile-99%%,", prefix) + fmt.Fprintf(w, "%s-percentile-99.9%%,", prefix) + fmt.Fprintf(w, "%s-rate-1-min,", prefix) + fmt.Fprintf(w, "%s-rate-5-min,", prefix) + fmt.Fprintf(w, "%s-rate-15-min,", prefix) + fmt.Fprintf(w, "%s-rate-mean", prefix) +} + +func WriteTimerCSV(w io.Writer, timer metrics.Timer) { + t := timer.Snapshot() + p := t.Percentiles(timerPercentiles) + + fmt.Fprintf(w, `%d,`, t.Count()) + fmt.Fprintf(w, `%d,`, t.Min()) + fmt.Fprintf(w, `%d,`, t.Max()) + fmt.Fprintf(w, `%f,`, t.Mean()) + fmt.Fprintf(w, `%f,`, t.StdDev()) + fmt.Fprintf(w, `%f,`, p[0]) + fmt.Fprintf(w, `%f,`, p[1]) + fmt.Fprintf(w, `%f,`, p[2]) + fmt.Fprintf(w, `%f,`, p[3]) + fmt.Fprintf(w, `%f,`, p[4]) + fmt.Fprintf(w, `%f,`, t.Rate1()) + fmt.Fprintf(w, `%f,`, t.Rate5()) + fmt.Fprintf(w, `%f,`, t.Rate15()) + fmt.Fprintf(w, `%f`, t.RateMean()) +} diff --git a/index/upsidedown/store/metrics/writer.go b/index/upsidedown/store/metrics/writer.go new file mode 100644 index 0000000..c278e38 --- /dev/null +++ b/index/upsidedown/store/metrics/writer.go @@ -0,0 +1,60 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "fmt" + + store "github.com/blevesearch/upsidedown_store_api" +) + +type Writer struct { + s *Store + o store.KVWriter +} + +func (w *Writer) Close() error { + err := w.o.Close() + if err != nil { + w.s.AddError("Writer.Close", err, nil) + } + return err +} + +func (w *Writer) NewBatch() store.KVBatch { + return &Batch{s: w.s, o: w.o.NewBatch()} +} + +func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) { + buf, b, err := w.o.NewBatchEx(options) + if err != nil { + return nil, nil, err + } + return buf, &Batch{s: w.s, o: b}, nil +} + +func (w *Writer) ExecuteBatch(b store.KVBatch) (err error) { + batch, ok := b.(*Batch) + if !ok { + return fmt.Errorf("wrong type of batch") + } + w.s.timerWriterExecuteBatch.Time(func() { + err = w.o.ExecuteBatch(batch.o) + if err != nil { + w.s.AddError("Writer.ExecuteBatch", err, nil) + } + }) + return +} diff --git a/index/upsidedown/store/moss/batch.go b/index/upsidedown/store/moss/batch.go new file mode 100644 index 0000000..0aa998a --- /dev/null +++ b/index/upsidedown/store/moss/batch.go @@ -0,0 +1,87 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package moss + +import ( + "github.com/couchbase/moss" + + store "github.com/blevesearch/upsidedown_store_api" +) + +type Batch struct { + store *Store + merge *store.EmulatedMerge + batch moss.Batch + buf []byte // Non-nil when using pre-alloc'ed / NewBatchEx(). + bufUsed int +} + +func (b *Batch) Set(key, val []byte) { + var err error + if b.buf != nil { + b.bufUsed += len(key) + len(val) + err = b.batch.AllocSet(key, val) + } else { + err = b.batch.Set(key, val) + } + + if err != nil { + b.store.Logf("bleve moss batch.Set err: %v", err) + } +} + +func (b *Batch) Delete(key []byte) { + var err error + if b.buf != nil { + b.bufUsed += len(key) + err = b.batch.AllocDel(key) + } else { + err = b.batch.Del(key) + } + + if err != nil { + b.store.Logf("bleve moss batch.Delete err: %v", err) + } +} + +func (b *Batch) Merge(key, val []byte) { + if b.buf != nil { + b.bufUsed += len(key) + len(val) + } + b.merge.Merge(key, val) +} + +func (b *Batch) Reset() { + err := b.Close() + if err != nil { + b.store.Logf("bleve moss batch.Close err: %v", err) + return + } + + batch, err := b.store.ms.NewBatch(0, 0) + if err == nil { + b.batch = batch + b.merge = store.NewEmulatedMerge(b.store.mo) + b.buf = nil + b.bufUsed = 0 + } +} + +func (b *Batch) Close() error { + b.merge = nil + err := b.batch.Close() + b.batch = nil + return err +} diff --git a/index/upsidedown/store/moss/iterator.go b/index/upsidedown/store/moss/iterator.go new file mode 100644 index 0000000..756c383 --- /dev/null +++ b/index/upsidedown/store/moss/iterator.go @@ -0,0 +1,87 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package moss + +import ( + "github.com/couchbase/moss" +) + +type Iterator struct { + store *Store + ss moss.Snapshot + iter moss.Iterator + start []byte + end []byte + k []byte + v []byte + err error +} + +func (x *Iterator) Seek(seekToKey []byte) { + _ = x.iter.SeekTo(seekToKey) + + x.k, x.v, x.err = x.iter.Current() +} + +func (x *Iterator) Next() { + _ = x.iter.Next() + + x.k, x.v, x.err = x.iter.Current() +} + +func (x *Iterator) Current() ([]byte, []byte, bool) { + return x.k, x.v, x.err == nil +} + +func (x *Iterator) Key() []byte { + if x.err != nil { + return nil + } + + return x.k +} + +func (x *Iterator) Value() []byte { + if x.err != nil { + return nil + } + + return x.v +} + +func (x *Iterator) Valid() bool { + return x.err == nil +} + +func (x *Iterator) Close() error { + var err error + + x.ss = nil + + if x.iter != nil { + err = x.iter.Close() + x.iter = nil + } + + x.k = nil + x.v = nil + x.err = moss.ErrIteratorDone + + return err +} + +func (x *Iterator) current() { + x.k, x.v, x.err = x.iter.Current() +} diff --git a/index/upsidedown/store/moss/lower.go b/index/upsidedown/store/moss/lower.go new file mode 100644 index 0000000..d52bc30 --- /dev/null +++ b/index/upsidedown/store/moss/lower.go @@ -0,0 +1,571 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package moss provides a KVStore implementation based on the +// github.com/couchbase/moss library. + +package moss + +import ( + "fmt" + "os" + "sync" + + "github.com/couchbase/moss" + + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/util" + store "github.com/blevesearch/upsidedown_store_api" +) + +func initLowerLevelStore( + config map[string]interface{}, + lowerLevelStoreName string, + lowerLevelStoreConfig map[string]interface{}, + lowerLevelMaxBatchSize uint64, + options moss.CollectionOptions, +) (moss.Snapshot, moss.LowerLevelUpdate, store.KVStore, statsFunc, error) { + if lowerLevelStoreConfig == nil { + lowerLevelStoreConfig = map[string]interface{}{} + } + + for k, v := range config { + _, exists := lowerLevelStoreConfig[k] + if !exists { + lowerLevelStoreConfig[k] = v + } + } + + if lowerLevelStoreName == "mossStore" { + return InitMossStore(lowerLevelStoreConfig, options) + } + + constructor := registry.KVStoreConstructorByName(lowerLevelStoreName) + if constructor == nil { + return nil, nil, nil, nil, fmt.Errorf("moss store, initLowerLevelStore,"+ + " could not find lower level store: %s", lowerLevelStoreName) + } + + kvStore, err := constructor(options.MergeOperator, lowerLevelStoreConfig) + if err != nil { + return nil, nil, nil, nil, err + } + + llStore := &llStore{ + refs: 0, + config: config, + llConfig: lowerLevelStoreConfig, + kvStore: kvStore, + logf: options.Log, + } + + llUpdate := func(ssHigher moss.Snapshot) (ssLower moss.Snapshot, err error) { + return llStore.update(ssHigher, lowerLevelMaxBatchSize) + } + + llSnapshot, err := llUpdate(nil) + if err != nil { + _ = kvStore.Close() + return nil, nil, nil, nil, err + } + + return llSnapshot, llUpdate, kvStore, nil, nil // llStore.refs is now 1. +} + +// ------------------------------------------------ + +// llStore is a lower level store and provides ref-counting around a +// bleve store.KVStore. +type llStore struct { + kvStore store.KVStore + + config map[string]interface{} + llConfig map[string]interface{} + + logf func(format string, a ...interface{}) + + m sync.Mutex // Protects fields that follow. + refs int +} + +// llSnapshot represents a lower-level snapshot, wrapping a bleve +// store.KVReader, and implements the moss.Snapshot interface. +type llSnapshot struct { + llStore *llStore // Holds 1 refs on the llStore. + kvReader store.KVReader + childSnapshots map[string]*llSnapshot + + m sync.Mutex // Protects fields that follow. + refs int +} + +// llIterator represents a lower-level iterator, wrapping a bleve +// store.KVIterator, and implements the moss.Iterator interface. +type llIterator struct { + llSnapshot *llSnapshot // Holds 1 refs on the llSnapshot. + + // Some lower-level KVReader implementations need a separate + // KVReader clone, due to KVReader single-threaded'ness. + kvReader store.KVReader + + kvIterator store.KVIterator +} + +type readerSource interface { + Reader() (store.KVReader, error) +} + +// ------------------------------------------------ + +func (s *llStore) addRef() *llStore { + s.m.Lock() + s.refs += 1 + s.m.Unlock() + + return s +} + +func (s *llStore) decRef() { + s.m.Lock() + s.refs -= 1 + if s.refs <= 0 { + err := s.kvStore.Close() + if err != nil { + s.logf("llStore kvStore.Close err: %v", err) + } + } + s.m.Unlock() +} + +// update() mutates this lower level store with latest data from the +// given higher level moss.Snapshot and returns a new moss.Snapshot +// that the higher level can use which represents this lower level +// store. +func (s *llStore) update(ssHigher moss.Snapshot, maxBatchSize uint64) ( + ssLower moss.Snapshot, err error, +) { + if ssHigher != nil { + iter, err := ssHigher.StartIterator(nil, nil, moss.IteratorOptions{ + IncludeDeletions: true, + SkipLowerLevel: true, + }) + if err != nil { + return nil, err + } + + defer func() { + err = iter.Close() + if err != nil { + s.logf("llStore iter.Close err: %v", err) + } + }() + + kvWriter, err := s.kvStore.Writer() + if err != nil { + return nil, err + } + + defer func() { + err = kvWriter.Close() + if err != nil { + s.logf("llStore kvWriter.Close err: %v", err) + } + }() + + batch := kvWriter.NewBatch() + + defer func() { + if batch != nil { + err = batch.Close() + if err != nil { + s.logf("llStore batch.Close err: %v", err) + } + } + }() + + var readOptions moss.ReadOptions + + i := uint64(0) + for { + if i%1000000 == 0 { + s.logf("llStore.update, i: %d", i) + } + + ex, key, val, err := iter.CurrentEx() + if err == moss.ErrIteratorDone { + break + } + if err != nil { + return nil, err + } + + switch ex.Operation { + case moss.OperationSet: + batch.Set(key, val) + + case moss.OperationDel: + batch.Delete(key) + + case moss.OperationMerge: + val, err = ssHigher.Get(key, readOptions) + if err != nil { + return nil, err + } + + if val != nil { + batch.Set(key, val) + } else { + batch.Delete(key) + } + + default: + return nil, fmt.Errorf("moss store, update,"+ + " unexpected operation, ex: %v", ex) + } + + i++ + + err = iter.Next() + if err == moss.ErrIteratorDone { + break + } + if err != nil { + return nil, err + } + + if maxBatchSize > 0 && i%maxBatchSize == 0 { + err = kvWriter.ExecuteBatch(batch) + if err != nil { + return nil, err + } + + err = batch.Close() + if err != nil { + return nil, err + } + + batch = kvWriter.NewBatch() + } + } + + if i > 0 { + s.logf("llStore.update, ExecuteBatch,"+ + " path: %s, total: %d, start", s.llConfig["path"], i) + + err = kvWriter.ExecuteBatch(batch) + if err != nil { + return nil, err + } + + s.logf("llStore.update, ExecuteBatch,"+ + " path: %s: total: %d, done", s.llConfig["path"], i) + } + } + + kvReader, err := s.kvStore.Reader() + if err != nil { + return nil, err + } + + s.logf("llStore.update, new reader") + + return &llSnapshot{ + llStore: s.addRef(), + kvReader: kvReader, + refs: 1, + }, nil +} + +// ------------------------------------------------ + +func (llss *llSnapshot) addRef() *llSnapshot { + llss.m.Lock() + llss.refs += 1 + llss.m.Unlock() + + return llss +} + +func (llss *llSnapshot) decRef() { + llss.m.Lock() + llss.refs -= 1 + if llss.refs <= 0 { + if llss.kvReader != nil { + err := llss.kvReader.Close() + if err != nil { + llss.llStore.logf("llSnapshot kvReader.Close err: %v", err) + } + + llss.kvReader = nil + } + + if llss.llStore != nil { + llss.llStore.decRef() + llss.llStore = nil + } + } + llss.m.Unlock() +} + +// ChildCollectionNames returns an array of child collection name strings. +func (llss *llSnapshot) ChildCollectionNames() ([]string, error) { + childCollections := make([]string, len(llss.childSnapshots)) + idx := 0 + for name := range llss.childSnapshots { + childCollections[idx] = name + idx++ + } + return childCollections, nil +} + +// ChildCollectionSnapshot returns a Snapshot on a given child +// collection by its name. +func (llss *llSnapshot) ChildCollectionSnapshot(childCollectionName string) ( + moss.Snapshot, error, +) { + childSnapshot, exists := llss.childSnapshots[childCollectionName] + if !exists { + return nil, nil + } + childSnapshot.addRef() + return childSnapshot, nil +} + +func (llss *llSnapshot) Close() error { + llss.decRef() + + return nil +} + +func (llss *llSnapshot) Get(key []byte, + readOptions moss.ReadOptions, +) ([]byte, error) { + rs, ok := llss.kvReader.(readerSource) + if ok { + r2, err := rs.Reader() + if err != nil { + return nil, err + } + + val, err := r2.Get(key) + + _ = r2.Close() + + return val, err + } + + return llss.kvReader.Get(key) +} + +func (llss *llSnapshot) StartIterator( + startKeyInclusive, endKeyExclusive []byte, + iteratorOptions moss.IteratorOptions, +) (moss.Iterator, error) { + rs, ok := llss.kvReader.(readerSource) + if ok { + r2, err := rs.Reader() + if err != nil { + return nil, err + } + + i2 := r2.RangeIterator(startKeyInclusive, endKeyExclusive) + + return &llIterator{llSnapshot: llss.addRef(), kvReader: r2, kvIterator: i2}, nil + } + + i := llss.kvReader.RangeIterator(startKeyInclusive, endKeyExclusive) + + return &llIterator{llSnapshot: llss.addRef(), kvReader: nil, kvIterator: i}, nil +} + +// ------------------------------------------------ + +func (lli *llIterator) Close() error { + var err0 error + if lli.kvIterator != nil { + err0 = lli.kvIterator.Close() + lli.kvIterator = nil + } + + var err1 error + if lli.kvReader != nil { + err1 = lli.kvReader.Close() + lli.kvReader = nil + } + + lli.llSnapshot.decRef() + lli.llSnapshot = nil + + if err0 != nil { + return err0 + } + + if err1 != nil { + return err1 + } + + return nil +} + +func (lli *llIterator) Next() error { + lli.kvIterator.Next() + + return nil +} + +func (lli *llIterator) SeekTo(k []byte) error { + lli.kvIterator.Seek(k) + + return nil +} + +func (lli *llIterator) Current() (key, val []byte, err error) { + key, val, ok := lli.kvIterator.Current() + if !ok { + return nil, nil, moss.ErrIteratorDone + } + + return key, val, nil +} + +func (lli *llIterator) CurrentEx() ( + entryEx moss.EntryEx, key, val []byte, err error, +) { + return moss.EntryEx{}, nil, nil, moss.ErrUnimplemented +} + +// ------------------------------------------------ + +func InitMossStore(config map[string]interface{}, options moss.CollectionOptions) ( + moss.Snapshot, moss.LowerLevelUpdate, store.KVStore, statsFunc, error, +) { + path, ok := config["path"].(string) + if !ok { + return nil, nil, nil, nil, fmt.Errorf("lower: missing path for InitMossStore config") + } + if path == "" { + return nil, nil, nil, nil, os.ErrInvalid + } + + err := os.MkdirAll(path, 0o700) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("lower: InitMossStore mkdir,"+ + " path: %s, err: %v", path, err) + } + + storeOptions := moss.StoreOptions{ + CollectionOptions: options, + } + v, ok := config["mossStoreOptions"] + if ok { + b, err := util.MarshalJSON(v) // Convert from map[string]interface{}. + if err != nil { + return nil, nil, nil, nil, err + } + + err = util.UnmarshalJSON(b, &storeOptions) + if err != nil { + return nil, nil, nil, nil, err + } + } + + s, err := moss.OpenStore(path, storeOptions) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("lower: moss.OpenStore,"+ + " path: %s, err: %v", path, err) + } + + sw := &mossStoreWrapper{s: s} + + llUpdate := func(ssHigher moss.Snapshot) (moss.Snapshot, error) { + ss, err := sw.s.Persist(ssHigher, moss.StorePersistOptions{ + CompactionConcern: moss.CompactionAllow, + }) + if err != nil { + return nil, err + } + + sw.AddRef() // Ref-count to be owned by snapshot wrapper. + + return moss.NewSnapshotWrapper(ss, sw), nil + } + + llSnapshot, err := llUpdate(nil) + if err != nil { + _ = s.Close() + return nil, nil, nil, nil, err + } + + llStats := func() map[string]interface{} { + stats, err := s.Stats() + if err != nil { + return nil + } + return stats + } + + return llSnapshot, llUpdate, sw, llStats, nil +} + +// mossStoreWrapper implements the bleve.index.store.KVStore +// interface, but only barely enough to allow it to be passed around +// as a lower-level store. Advanced apps will likely cast the +// mossStoreWrapper to access the Actual() method. +type mossStoreWrapper struct { + m sync.Mutex + refs int + s *moss.Store +} + +func (w *mossStoreWrapper) AddRef() { + w.m.Lock() + w.refs++ + w.m.Unlock() +} + +func (w *mossStoreWrapper) Close() (err error) { + w.m.Lock() + w.refs-- + if w.refs <= 0 { + err = w.s.Close() + w.s = nil + } + w.m.Unlock() + return err +} + +func (w *mossStoreWrapper) Reader() (store.KVReader, error) { + return nil, fmt.Errorf("unexpected") +} + +func (w *mossStoreWrapper) Writer() (store.KVWriter, error) { + return nil, fmt.Errorf("unexpected") +} + +func (w *mossStoreWrapper) Actual() *moss.Store { + w.m.Lock() + rv := w.s + w.m.Unlock() + return rv +} + +func (w *mossStoreWrapper) histograms() string { + var rv string + w.m.Lock() + if w.s != nil { + rv = w.s.Histograms().String() + } + w.m.Unlock() + return rv +} diff --git a/index/upsidedown/store/moss/lower_test.go b/index/upsidedown/store/moss/lower_test.go new file mode 100644 index 0000000..85afe49 --- /dev/null +++ b/index/upsidedown/store/moss/lower_test.go @@ -0,0 +1,103 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package moss + +import ( + "os" + "testing" + + store "github.com/blevesearch/upsidedown_store_api" + "github.com/blevesearch/upsidedown_store_api/test" +) + +func openWithLower(t *testing.T, mo store.MergeOperator) (string, store.KVStore) { + tmpDir, _ := os.MkdirTemp("", "mossStore") + + config := map[string]interface{}{ + "path": tmpDir, + "mossLowerLevelStoreName": "mossStore", + } + + rv, err := New(mo, config) + if err != nil { + t.Fatal(err) + } + return tmpDir, rv +} + +func cleanupWithLower(t *testing.T, s store.KVStore, tmpDir string) { + err := s.Close() + if err != nil { + t.Fatal(err) + } + err = os.RemoveAll(tmpDir) + if err != nil { + t.Fatal(err) + } +} + +func TestMossWithLowerKVCrud(t *testing.T) { + tmpDir, s := openWithLower(t, nil) + defer cleanupWithLower(t, s, tmpDir) + test.CommonTestKVCrud(t, s) +} + +func TestMossWithLowerReaderIsolation(t *testing.T) { + tmpDir, s := openWithLower(t, nil) + defer cleanupWithLower(t, s, tmpDir) + test.CommonTestReaderIsolation(t, s) +} + +func TestMossWithLowerReaderOwnsGetBytes(t *testing.T) { + tmpDir, s := openWithLower(t, nil) + defer cleanupWithLower(t, s, tmpDir) + test.CommonTestReaderOwnsGetBytes(t, s) +} + +func TestMossWithLowerWriterOwnsBytes(t *testing.T) { + tmpDir, s := openWithLower(t, nil) + defer cleanupWithLower(t, s, tmpDir) + test.CommonTestWriterOwnsBytes(t, s) +} + +func TestMossWithLowerPrefixIterator(t *testing.T) { + tmpDir, s := openWithLower(t, nil) + defer cleanupWithLower(t, s, tmpDir) + test.CommonTestPrefixIterator(t, s) +} + +func TestMossWithLowerPrefixIteratorSeek(t *testing.T) { + tmpDir, s := openWithLower(t, nil) + defer cleanupWithLower(t, s, tmpDir) + test.CommonTestPrefixIteratorSeek(t, s) +} + +func TestMossWithLowerRangeIterator(t *testing.T) { + tmpDir, s := openWithLower(t, nil) + defer cleanupWithLower(t, s, tmpDir) + test.CommonTestRangeIterator(t, s) +} + +func TestMossWithLowerRangeIteratorSeek(t *testing.T) { + tmpDir, s := openWithLower(t, nil) + defer cleanupWithLower(t, s, tmpDir) + test.CommonTestRangeIteratorSeek(t, s) +} + +func TestMossWithLowerMerge(t *testing.T) { + tmpDir, s := openWithLower(t, &test.TestMergeCounter{}) + defer cleanupWithLower(t, s, tmpDir) + test.CommonTestMerge(t, s) +} diff --git a/index/upsidedown/store/moss/reader.go b/index/upsidedown/store/moss/reader.go new file mode 100644 index 0000000..93063cd --- /dev/null +++ b/index/upsidedown/store/moss/reader.go @@ -0,0 +1,97 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package moss + +import ( + "github.com/couchbase/moss" + + store "github.com/blevesearch/upsidedown_store_api" +) + +type Reader struct { + store *Store + ss moss.Snapshot +} + +func (r *Reader) Get(k []byte) (v []byte, err error) { + v, err = r.ss.Get(k, moss.ReadOptions{}) + if err != nil { + return nil, err + } + if v != nil { + return append(make([]byte, 0, len(v)), v...), nil + } + return nil, nil +} + +func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) { + return store.MultiGet(r, keys) +} + +func (r *Reader) PrefixIterator(k []byte) store.KVIterator { + kEnd := incrementBytes(k) + + iter, err := r.ss.StartIterator(k, kEnd, moss.IteratorOptions{}) + if err != nil { + return nil + } + + rv := &Iterator{ + store: r.store, + ss: r.ss, + iter: iter, + start: k, + end: kEnd, + } + + rv.current() + + return rv +} + +func (r *Reader) RangeIterator(start, end []byte) store.KVIterator { + iter, err := r.ss.StartIterator(start, end, moss.IteratorOptions{}) + if err != nil { + return nil + } + + rv := &Iterator{ + store: r.store, + ss: r.ss, + iter: iter, + start: start, + end: end, + } + + rv.current() + + return rv +} + +func (r *Reader) Close() error { + return r.ss.Close() +} + +func incrementBytes(in []byte) []byte { + rv := make([]byte, len(in)) + copy(rv, in) + for i := len(rv) - 1; i >= 0; i-- { + rv[i] = rv[i] + 1 + if rv[i] != 0 { + return rv // didn't overflow, so stop + } + } + return nil // overflowed +} diff --git a/index/upsidedown/store/moss/stats.go b/index/upsidedown/store/moss/stats.go new file mode 100644 index 0000000..3956bbd --- /dev/null +++ b/index/upsidedown/store/moss/stats.go @@ -0,0 +1,58 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package moss + +import ( + "github.com/blevesearch/bleve/v2/util" + store "github.com/blevesearch/upsidedown_store_api" +) + +type stats struct { + s *Store +} + +func (s *stats) statsMap() map[string]interface{} { + ms := map[string]interface{}{} + + var err error + ms["moss"], err = s.s.ms.Stats() + if err != nil { + return ms + } + + if s.s.llstore != nil { + if o, ok := s.s.llstore.(store.KVStoreStats); ok { + ms["kv"] = o.StatsMap() + } + } + + _, exists := ms["kv"] + if !exists && s.s.llstats != nil { + ms["kv"] = s.s.llstats() + } + + if msw, ok := s.s.llstore.(*mossStoreWrapper); ok { + ms["store_histograms"] = msw.histograms() + } + + ms["coll_histograms"] = s.s.ms.Histograms().String() + + return ms +} + +func (s *stats) MarshalJSON() ([]byte, error) { + m := s.statsMap() + return util.MarshalJSON(m) +} diff --git a/index/upsidedown/store/moss/store.go b/index/upsidedown/store/moss/store.go new file mode 100644 index 0000000..bccbe1f --- /dev/null +++ b/index/upsidedown/store/moss/store.go @@ -0,0 +1,231 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package moss provides a KVStore implementation based on the +// github.com/couchbase/moss library. + +package moss + +import ( + "encoding/json" + "fmt" + "sync" + + "github.com/couchbase/moss" + + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/util" + store "github.com/blevesearch/upsidedown_store_api" +) + +// RegistryCollectionOptions should be treated as read-only after +// process init()'ialization. +var RegistryCollectionOptions = map[string]moss.CollectionOptions{} + +const Name = "moss" + +type Store struct { + m sync.Mutex + ms moss.Collection + mo store.MergeOperator + llstore store.KVStore // May be nil. + llstats statsFunc // May be nil. + + s *stats + config map[string]interface{} +} + +type statsFunc func() map[string]interface{} + +// New initializes a moss storage with values from the optional +// config["mossCollectionOptions"] (a JSON moss.CollectionOptions). +// Next, values from the RegistryCollectionOptions, named by the +// optional config["mossCollectionOptionsName"], take precedence. +// Finally, base case defaults are taken from +// moss.DefaultCollectionOptions. +func New(mo store.MergeOperator, config map[string]interface{}) ( + store.KVStore, error) { + options := moss.DefaultCollectionOptions // Copy. + + v, ok := config["mossCollectionOptionsName"] + if ok { + name, ok := v.(string) + if !ok { + return nil, fmt.Errorf("moss store,"+ + " could not parse config[mossCollectionOptionsName]: %v", v) + } + + options, ok = RegistryCollectionOptions[name] // Copy. + if !ok { + return nil, fmt.Errorf("moss store,"+ + " could not find RegistryCollectionOptions, name: %s", name) + } + } + + options.MergeOperator = mo + options.DeferredSort = true + + v, ok = config["mossCollectionOptions"] + if ok { + b, err := util.MarshalJSON(v) // Convert from map[string]interface{}. + if err != nil { + return nil, fmt.Errorf("moss store,"+ + " could not marshal config[mossCollectionOptions]: %v, err: %v", v, err) + } + + err = util.UnmarshalJSON(b, &options) + if err != nil { + return nil, fmt.Errorf("moss store,"+ + " could not unmarshal config[mossCollectionOptions]: %v, err: %v", v, err) + } + } + + // -------------------------------------------------- + + if options.Log == nil || options.Debug <= 0 { + options.Log = func(format string, a ...interface{}) {} + } + + // -------------------------------------------------- + + mossLowerLevelStoreName := "" + v, ok = config["mossLowerLevelStoreName"] + if ok { + mossLowerLevelStoreName, ok = v.(string) + if !ok { + return nil, fmt.Errorf("moss store,"+ + " could not parse config[mossLowerLevelStoreName]: %v", v) + } + } + + var llStore store.KVStore + var llStats statsFunc + + if options.LowerLevelInit == nil && + options.LowerLevelUpdate == nil && + mossLowerLevelStoreName != "" { + mossLowerLevelStoreConfig := map[string]interface{}{} + v, ok := config["mossLowerLevelStoreConfig"] + if ok { + mossLowerLevelStoreConfig, ok = v.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("moss store, initLowerLevelStore,"+ + " could parse mossLowerLevelStoreConfig: %v", v) + } + } + + mossLowerLevelMaxBatchSize := uint64(0) + v, ok = config["mossLowerLevelMaxBatchSize"] + if ok { + mossLowerLevelMaxBatchSizeF, ok := v.(float64) + if !ok { + return nil, fmt.Errorf("moss store,"+ + " could not parse config[mossLowerLevelMaxBatchSize]: %v", v) + } + + mossLowerLevelMaxBatchSize = uint64(mossLowerLevelMaxBatchSizeF) + } + + lowerLevelInit, lowerLevelUpdate, lowerLevelStore, lowerLevelStats, err := + initLowerLevelStore(config, + mossLowerLevelStoreName, + mossLowerLevelStoreConfig, + mossLowerLevelMaxBatchSize, + options) + if err != nil { + return nil, err + } + + options.LowerLevelInit = lowerLevelInit + options.LowerLevelUpdate = lowerLevelUpdate + + llStore = lowerLevelStore + llStats = lowerLevelStats + } + + // -------------------------------------------------- + + ms, err := moss.NewCollection(options) + if err != nil { + return nil, err + } + err = ms.Start() + if err != nil { + return nil, err + } + rv := Store{ + ms: ms, + mo: mo, + llstore: llStore, + llstats: llStats, + config: config, + } + rv.s = &stats{s: &rv} + return &rv, nil +} + +func (s *Store) Close() error { + if val, ok := s.config["mossAbortCloseEnabled"]; ok { + if v, ok := val.(bool); ok && v { + if msw, ok := s.llstore.(*mossStoreWrapper); ok { + if s := msw.Actual(); s != nil { + _ = s.CloseEx(moss.StoreCloseExOptions{Abort: true}) + } + } + } + } + return s.ms.Close() +} + +func (s *Store) Reader() (store.KVReader, error) { + ss, err := s.ms.Snapshot() + if err != nil { + return nil, err + } + return &Reader{ss: ss}, nil +} + +func (s *Store) Writer() (store.KVWriter, error) { + return &Writer{s: s}, nil +} + +func (s *Store) Logf(fmt string, args ...interface{}) { + options := s.ms.Options() + if options.Log != nil { + options.Log(fmt, args...) + } +} + +func (s *Store) Stats() json.Marshaler { + return s.s +} + +func (s *Store) StatsMap() map[string]interface{} { + return s.s.statsMap() +} + +func (s *Store) LowerLevelStore() store.KVStore { + return s.llstore +} + +func (s *Store) Collection() moss.Collection { + return s.ms +} + +func init() { + err := registry.RegisterKVStore(Name, New) + if err != nil { + panic(err) + } +} diff --git a/index/upsidedown/store/moss/store_test.go b/index/upsidedown/store/moss/store_test.go new file mode 100644 index 0000000..49013b6 --- /dev/null +++ b/index/upsidedown/store/moss/store_test.go @@ -0,0 +1,91 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package moss + +import ( + "testing" + + store "github.com/blevesearch/upsidedown_store_api" + "github.com/blevesearch/upsidedown_store_api/test" +) + +func open(t *testing.T, mo store.MergeOperator) store.KVStore { + rv, err := New(mo, nil) + if err != nil { + t.Fatal(err) + } + return rv +} + +func cleanup(t *testing.T, s store.KVStore) { + err := s.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestMossKVCrud(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestKVCrud(t, s) +} + +func TestMossReaderIsolation(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestReaderIsolation(t, s) +} + +func TestMossReaderOwnsGetBytes(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestReaderOwnsGetBytes(t, s) +} + +func TestMossWriterOwnsBytes(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestWriterOwnsBytes(t, s) +} + +func TestMossPrefixIterator(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestPrefixIterator(t, s) +} + +func TestMossPrefixIteratorSeek(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestPrefixIteratorSeek(t, s) +} + +func TestMossRangeIterator(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestRangeIterator(t, s) +} + +func TestMossRangeIteratorSeek(t *testing.T) { + s := open(t, nil) + defer cleanup(t, s) + test.CommonTestRangeIteratorSeek(t, s) +} + +func TestMossMerge(t *testing.T) { + s := open(t, &test.TestMergeCounter{}) + defer cleanup(t, s) + test.CommonTestMerge(t, s) +} diff --git a/index/upsidedown/store/moss/writer.go b/index/upsidedown/store/moss/writer.go new file mode 100644 index 0000000..cb44716 --- /dev/null +++ b/index/upsidedown/store/moss/writer.go @@ -0,0 +1,97 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package moss + +import ( + "fmt" + + store "github.com/blevesearch/upsidedown_store_api" + + "github.com/couchbase/moss" +) + +type Writer struct { + s *Store +} + +func (w *Writer) NewBatch() store.KVBatch { + b, err := w.s.ms.NewBatch(0, 0) + if err != nil { + return nil + } + + return &Batch{ + store: w.s, + merge: store.NewEmulatedMerge(w.s.mo), + batch: b, + } +} + +func (w *Writer) NewBatchEx(options store.KVBatchOptions) ( + []byte, store.KVBatch, error) { + numOps := options.NumSets + options.NumDeletes + options.NumMerges + + b, err := w.s.ms.NewBatch(numOps, options.TotalBytes) + if err != nil { + return nil, nil, err + } + + buf, err := b.Alloc(options.TotalBytes) + if err != nil { + return nil, nil, err + } + + return buf, &Batch{ + store: w.s, + merge: store.NewEmulatedMerge(w.s.mo), + batch: b, + buf: buf, + bufUsed: 0, + }, nil +} + +func (w *Writer) ExecuteBatch(b store.KVBatch) (err error) { + batch, ok := b.(*Batch) + if !ok { + return fmt.Errorf("wrong type of batch") + } + + for kStr, mergeOps := range batch.merge.Merges { + for _, v := range mergeOps { + if batch.buf != nil { + kLen := len(kStr) + vLen := len(v) + kBuf := batch.buf[batch.bufUsed : batch.bufUsed+kLen] + vBuf := batch.buf[batch.bufUsed+kLen : batch.bufUsed+kLen+vLen] + copy(kBuf, kStr) + copy(vBuf, v) + batch.bufUsed += kLen + vLen + err = batch.batch.AllocMerge(kBuf, vBuf) + } else { + err = batch.batch.Merge([]byte(kStr), v) + } + if err != nil { + return err + } + } + } + + return w.s.ms.ExecuteBatch(batch.batch, moss.WriteOptions{}) +} + +func (w *Writer) Close() error { + w.s = nil + return nil +} diff --git a/index/upsidedown/store/null/null.go b/index/upsidedown/store/null/null.go new file mode 100644 index 0000000..952e1eb --- /dev/null +++ b/index/upsidedown/store/null/null.go @@ -0,0 +1,121 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package null + +import ( + "github.com/blevesearch/bleve/v2/registry" + store "github.com/blevesearch/upsidedown_store_api" +) + +const Name = "null" + +type Store struct{} + +func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) { + return &Store{}, nil +} + +func (i *Store) Close() error { + return nil +} + +func (i *Store) Reader() (store.KVReader, error) { + return &reader{}, nil +} + +func (i *Store) Writer() (store.KVWriter, error) { + return &writer{}, nil +} + +type reader struct{} + +func (r *reader) Get(key []byte) ([]byte, error) { + return nil, nil +} + +func (r *reader) MultiGet(keys [][]byte) ([][]byte, error) { + return make([][]byte, len(keys)), nil +} + +func (r *reader) PrefixIterator(prefix []byte) store.KVIterator { + return &iterator{} +} + +func (r *reader) RangeIterator(start, end []byte) store.KVIterator { + return &iterator{} +} + +func (r *reader) Close() error { + return nil +} + +type iterator struct{} + +func (i *iterator) SeekFirst() {} +func (i *iterator) Seek(k []byte) {} +func (i *iterator) Next() {} + +func (i *iterator) Current() ([]byte, []byte, bool) { + return nil, nil, false +} + +func (i *iterator) Key() []byte { + return nil +} + +func (i *iterator) Value() []byte { + return nil +} + +func (i *iterator) Valid() bool { + return false +} + +func (i *iterator) Close() error { + return nil +} + +type batch struct{} + +func (i *batch) Set(key, val []byte) {} +func (i *batch) Delete(key []byte) {} +func (i *batch) Merge(key, val []byte) {} +func (i *batch) Reset() {} +func (i *batch) Close() error { return nil } + +type writer struct{} + +func (w *writer) NewBatch() store.KVBatch { + return &batch{} +} + +func (w *writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) { + return make([]byte, options.TotalBytes), w.NewBatch(), nil +} + +func (w *writer) ExecuteBatch(store.KVBatch) error { + return nil +} + +func (w *writer) Close() error { + return nil +} + +func init() { + err := registry.RegisterKVStore(Name, New) + if err != nil { + panic(err) + } +} diff --git a/index/upsidedown/store/null/null_test.go b/index/upsidedown/store/null/null_test.go new file mode 100644 index 0000000..2042f4d --- /dev/null +++ b/index/upsidedown/store/null/null_test.go @@ -0,0 +1,92 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package null + +import ( + "testing" + + store "github.com/blevesearch/upsidedown_store_api" +) + +func TestStore(t *testing.T) { + s, err := New(nil, nil) + if err != nil { + t.Fatal(err) + } + + NullTestKVStore(t, s) +} + +// NullTestKVStore has very different expectations +// compared to CommonTestKVStore +func NullTestKVStore(t *testing.T, s store.KVStore) { + + writer, err := s.Writer() + if err != nil { + t.Error(err) + } + + batch := writer.NewBatch() + batch.Set([]byte("b"), []byte("val-b")) + batch.Set([]byte("c"), []byte("val-c")) + batch.Set([]byte("d"), []byte("val-d")) + batch.Set([]byte("e"), []byte("val-e")) + batch.Set([]byte("f"), []byte("val-f")) + batch.Set([]byte("g"), []byte("val-g")) + batch.Set([]byte("h"), []byte("val-h")) + batch.Set([]byte("i"), []byte("val-i")) + batch.Set([]byte("j"), []byte("val-j")) + + err = writer.ExecuteBatch(batch) + if err != nil { + t.Fatal(err) + } + err = writer.Close() + if err != nil { + t.Fatal(err) + } + + reader, err := s.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := reader.Close() + if err != nil { + t.Fatal(err) + } + }() + it := reader.RangeIterator([]byte("b"), nil) + key, val, valid := it.Current() + if valid { + t.Fatalf("valid true, expected false") + } + if key != nil { + t.Fatalf("expected key nil, got %s", key) + } + if val != nil { + t.Fatalf("expected value nil, got %s", val) + } + + err = it.Close() + if err != nil { + t.Fatal(err) + } + + err = s.Close() + if err != nil { + t.Fatal(err) + } +} diff --git a/index/upsidedown/upsidedown.go b/index/upsidedown/upsidedown.go new file mode 100644 index 0000000..7ef6e05 --- /dev/null +++ b/index/upsidedown/upsidedown.go @@ -0,0 +1,1079 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate protoc --gofast_out=. upsidedown.proto + +package upsidedown + +import ( + "encoding/binary" + "encoding/json" + "fmt" + "math" + "sync" + "sync/atomic" + "time" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/registry" + index "github.com/blevesearch/bleve_index_api" + store "github.com/blevesearch/upsidedown_store_api" + + "google.golang.org/protobuf/proto" +) + +const Name = "upside_down" + +// RowBufferSize should ideally this is sized to be the smallest +// size that can contain an index row key and its corresponding +// value. It is not a limit, if need be a larger buffer is +// allocated, but performance will be more optimal if *most* +// rows fit this size. +const RowBufferSize = 4 * 1024 + +var VersionKey = []byte{'v'} + +const Version uint8 = 7 + +var IncompatibleVersion = fmt.Errorf("incompatible version, %d is supported", Version) + +var ErrorUnknownStorageType = fmt.Errorf("unknown storage type") + +type UpsideDownCouch struct { + version uint8 + path string + storeName string + storeConfig map[string]interface{} + store store.KVStore + fieldCache *FieldCache + analysisQueue *index.AnalysisQueue + stats *indexStat + + m sync.RWMutex + // fields protected by m + docCount uint64 + + writeMutex sync.Mutex +} + +type docBackIndexRow struct { + docID string + doc index.Document // If deletion, doc will be nil. + backIndexRow *BackIndexRow +} + +func NewUpsideDownCouch(storeName string, storeConfig map[string]interface{}, analysisQueue *index.AnalysisQueue) (index.Index, error) { + rv := &UpsideDownCouch{ + version: Version, + fieldCache: NewFieldCache(), + storeName: storeName, + storeConfig: storeConfig, + analysisQueue: analysisQueue, + } + rv.stats = &indexStat{i: rv} + return rv, nil +} + +func (udc *UpsideDownCouch) init(kvwriter store.KVWriter) (err error) { + // version marker + rowsAll := [][]UpsideDownCouchRow{ + {NewVersionRow(udc.version)}, + } + + err = udc.batchRows(kvwriter, nil, rowsAll, nil) + return +} + +func (udc *UpsideDownCouch) loadSchema(kvreader store.KVReader) (err error) { + + it := kvreader.PrefixIterator([]byte{'f'}) + defer func() { + if cerr := it.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + key, val, valid := it.Current() + for valid { + var fieldRow *FieldRow + fieldRow, err = NewFieldRowKV(key, val) + if err != nil { + return + } + udc.fieldCache.AddExisting(fieldRow.name, fieldRow.index) + + it.Next() + key, val, valid = it.Current() + } + + val, err = kvreader.Get([]byte{'v'}) + if err != nil { + return + } + var vr *VersionRow + vr, err = NewVersionRowKV([]byte{'v'}, val) + if err != nil { + return + } + if vr.version != Version { + err = IncompatibleVersion + return + } + + return +} + +type rowBuffer struct { + buf []byte +} + +var rowBufferPool sync.Pool + +func GetRowBuffer() *rowBuffer { + if rb, ok := rowBufferPool.Get().(*rowBuffer); ok { + return rb + } else { + buf := make([]byte, RowBufferSize) + return &rowBuffer{buf: buf} + } +} + +func PutRowBuffer(rb *rowBuffer) { + rowBufferPool.Put(rb) +} + +func (udc *UpsideDownCouch) batchRows(writer store.KVWriter, addRowsAll [][]UpsideDownCouchRow, updateRowsAll [][]UpsideDownCouchRow, deleteRowsAll [][]UpsideDownCouchRow) (err error) { + dictionaryDeltas := make(map[string]int64) + + // count up bytes needed for buffering. + addNum := 0 + addKeyBytes := 0 + addValBytes := 0 + + updateNum := 0 + updateKeyBytes := 0 + updateValBytes := 0 + + deleteNum := 0 + deleteKeyBytes := 0 + + rowBuf := GetRowBuffer() + + for _, addRows := range addRowsAll { + for _, row := range addRows { + tfr, ok := row.(*TermFrequencyRow) + if ok { + if tfr.DictionaryRowKeySize() > len(rowBuf.buf) { + rowBuf.buf = make([]byte, tfr.DictionaryRowKeySize()) + } + dictKeySize, err := tfr.DictionaryRowKeyTo(rowBuf.buf) + if err != nil { + return err + } + dictionaryDeltas[string(rowBuf.buf[:dictKeySize])] += 1 + } + addKeyBytes += row.KeySize() + addValBytes += row.ValueSize() + } + addNum += len(addRows) + } + + for _, updateRows := range updateRowsAll { + for _, row := range updateRows { + updateKeyBytes += row.KeySize() + updateValBytes += row.ValueSize() + } + updateNum += len(updateRows) + } + + for _, deleteRows := range deleteRowsAll { + for _, row := range deleteRows { + tfr, ok := row.(*TermFrequencyRow) + if ok { + // need to decrement counter + if tfr.DictionaryRowKeySize() > len(rowBuf.buf) { + rowBuf.buf = make([]byte, tfr.DictionaryRowKeySize()) + } + dictKeySize, err := tfr.DictionaryRowKeyTo(rowBuf.buf) + if err != nil { + return err + } + dictionaryDeltas[string(rowBuf.buf[:dictKeySize])] -= 1 + } + deleteKeyBytes += row.KeySize() + } + deleteNum += len(deleteRows) + } + + PutRowBuffer(rowBuf) + + mergeNum := len(dictionaryDeltas) + mergeKeyBytes := 0 + mergeValBytes := mergeNum * DictionaryRowMaxValueSize + + for dictRowKey := range dictionaryDeltas { + mergeKeyBytes += len(dictRowKey) + } + + // prepare batch + totBytes := addKeyBytes + addValBytes + + updateKeyBytes + updateValBytes + + deleteKeyBytes + + 2*(mergeKeyBytes+mergeValBytes) + + buf, wb, err := writer.NewBatchEx(store.KVBatchOptions{ + TotalBytes: totBytes, + NumSets: addNum + updateNum, + NumDeletes: deleteNum, + NumMerges: mergeNum, + }) + if err != nil { + return err + } + defer func() { + _ = wb.Close() + }() + + // fill the batch + for _, addRows := range addRowsAll { + for _, row := range addRows { + keySize, err := row.KeyTo(buf) + if err != nil { + return err + } + valSize, err := row.ValueTo(buf[keySize:]) + if err != nil { + return err + } + wb.Set(buf[:keySize], buf[keySize:keySize+valSize]) + buf = buf[keySize+valSize:] + } + } + + for _, updateRows := range updateRowsAll { + for _, row := range updateRows { + keySize, err := row.KeyTo(buf) + if err != nil { + return err + } + valSize, err := row.ValueTo(buf[keySize:]) + if err != nil { + return err + } + wb.Set(buf[:keySize], buf[keySize:keySize+valSize]) + buf = buf[keySize+valSize:] + } + } + + for _, deleteRows := range deleteRowsAll { + for _, row := range deleteRows { + keySize, err := row.KeyTo(buf) + if err != nil { + return err + } + wb.Delete(buf[:keySize]) + buf = buf[keySize:] + } + } + + for dictRowKey, delta := range dictionaryDeltas { + dictRowKeyLen := copy(buf, dictRowKey) + binary.LittleEndian.PutUint64(buf[dictRowKeyLen:], uint64(delta)) + wb.Merge(buf[:dictRowKeyLen], buf[dictRowKeyLen:dictRowKeyLen+DictionaryRowMaxValueSize]) + buf = buf[dictRowKeyLen+DictionaryRowMaxValueSize:] + } + + // write out the batch + return writer.ExecuteBatch(wb) +} + +func (udc *UpsideDownCouch) Open() (err error) { + // acquire the write mutex for the duration of Open() + udc.writeMutex.Lock() + defer udc.writeMutex.Unlock() + + // open the kv store + storeConstructor := registry.KVStoreConstructorByName(udc.storeName) + if storeConstructor == nil { + err = ErrorUnknownStorageType + return + } + + // now open the store + udc.store, err = storeConstructor(&mergeOperator, udc.storeConfig) + if err != nil { + return + } + + // start a reader to look at the index + var kvreader store.KVReader + kvreader, err = udc.store.Reader() + if err != nil { + return + } + + var value []byte + value, err = kvreader.Get(VersionKey) + if err != nil { + _ = kvreader.Close() + return + } + + if value != nil { + err = udc.loadSchema(kvreader) + if err != nil { + _ = kvreader.Close() + return + } + + // set doc count + udc.m.Lock() + udc.docCount, err = udc.countDocs(kvreader) + udc.m.Unlock() + + err = kvreader.Close() + } else { + // new index, close the reader and open writer to init + err = kvreader.Close() + if err != nil { + return + } + + var kvwriter store.KVWriter + kvwriter, err = udc.store.Writer() + if err != nil { + return + } + defer func() { + if cerr := kvwriter.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + // init the index + err = udc.init(kvwriter) + } + + return +} + +func (udc *UpsideDownCouch) countDocs(kvreader store.KVReader) (count uint64, err error) { + it := kvreader.PrefixIterator([]byte{'b'}) + defer func() { + if cerr := it.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + _, _, valid := it.Current() + for valid { + count++ + it.Next() + _, _, valid = it.Current() + } + + return +} + +func (udc *UpsideDownCouch) rowCount() (count uint64, err error) { + // start an isolated reader for use during the rowcount + kvreader, err := udc.store.Reader() + if err != nil { + return + } + defer func() { + if cerr := kvreader.Close(); err == nil && cerr != nil { + err = cerr + } + }() + it := kvreader.RangeIterator(nil, nil) + defer func() { + if cerr := it.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + _, _, valid := it.Current() + for valid { + count++ + it.Next() + _, _, valid = it.Current() + } + + return +} + +func (udc *UpsideDownCouch) Close() error { + return udc.store.Close() +} + +func (udc *UpsideDownCouch) Update(doc index.Document) (err error) { + // do analysis before acquiring write lock + analysisStart := time.Now() + resultChan := make(chan *AnalysisResult) + + // put the work on the queue + udc.analysisQueue.Queue(func() { + ar := udc.analyze(doc) + resultChan <- ar + }) + + // wait for the result + result := <-resultChan + close(resultChan) + atomic.AddUint64(&udc.stats.analysisTime, uint64(time.Since(analysisStart))) + + udc.writeMutex.Lock() + defer udc.writeMutex.Unlock() + + // open a reader for backindex lookup + var kvreader store.KVReader + kvreader, err = udc.store.Reader() + if err != nil { + return + } + + // first we lookup the backindex row for the doc id if it exists + // lookup the back index row + var backIndexRow *BackIndexRow + backIndexRow, err = backIndexRowForDoc(kvreader, index.IndexInternalID(doc.ID())) + if err != nil { + _ = kvreader.Close() + atomic.AddUint64(&udc.stats.errors, 1) + return + } + + err = kvreader.Close() + if err != nil { + return + } + + return udc.UpdateWithAnalysis(doc, result, backIndexRow) +} + +func (udc *UpsideDownCouch) UpdateWithAnalysis(doc index.Document, + result *AnalysisResult, backIndexRow *BackIndexRow) (err error) { + // start a writer for this update + indexStart := time.Now() + var kvwriter store.KVWriter + kvwriter, err = udc.store.Writer() + if err != nil { + return + } + defer func() { + if cerr := kvwriter.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + // prepare a list of rows + var addRowsAll [][]UpsideDownCouchRow + var updateRowsAll [][]UpsideDownCouchRow + var deleteRowsAll [][]UpsideDownCouchRow + + addRows, updateRows, deleteRows := udc.mergeOldAndNew(backIndexRow, result.Rows) + if len(addRows) > 0 { + addRowsAll = append(addRowsAll, addRows) + } + if len(updateRows) > 0 { + updateRowsAll = append(updateRowsAll, updateRows) + } + if len(deleteRows) > 0 { + deleteRowsAll = append(deleteRowsAll, deleteRows) + } + + err = udc.batchRows(kvwriter, addRowsAll, updateRowsAll, deleteRowsAll) + if err == nil && backIndexRow == nil { + udc.m.Lock() + udc.docCount++ + udc.m.Unlock() + } + atomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart))) + if err == nil { + atomic.AddUint64(&udc.stats.updates, 1) + atomic.AddUint64(&udc.stats.numPlainTextBytesIndexed, doc.NumPlainTextBytes()) + } else { + atomic.AddUint64(&udc.stats.errors, 1) + } + return +} + +func (udc *UpsideDownCouch) mergeOldAndNew(backIndexRow *BackIndexRow, rows []IndexRow) (addRows []UpsideDownCouchRow, updateRows []UpsideDownCouchRow, deleteRows []UpsideDownCouchRow) { + addRows = make([]UpsideDownCouchRow, 0, len(rows)) + + if backIndexRow == nil { + addRows = addRows[0:len(rows)] + for i, row := range rows { + addRows[i] = row + } + return addRows, nil, nil + } + + updateRows = make([]UpsideDownCouchRow, 0, len(rows)) + deleteRows = make([]UpsideDownCouchRow, 0, len(rows)) + + var existingTermKeys map[string]struct{} + backIndexTermKeys := backIndexRow.AllTermKeys() + if len(backIndexTermKeys) > 0 { + existingTermKeys = make(map[string]struct{}, len(backIndexTermKeys)) + for _, key := range backIndexTermKeys { + existingTermKeys[string(key)] = struct{}{} + } + } + + var existingStoredKeys map[string]struct{} + backIndexStoredKeys := backIndexRow.AllStoredKeys() + if len(backIndexStoredKeys) > 0 { + existingStoredKeys = make(map[string]struct{}, len(backIndexStoredKeys)) + for _, key := range backIndexStoredKeys { + existingStoredKeys[string(key)] = struct{}{} + } + } + + keyBuf := GetRowBuffer() + for _, row := range rows { + switch row := row.(type) { + case *TermFrequencyRow: + if existingTermKeys != nil { + if row.KeySize() > len(keyBuf.buf) { + keyBuf.buf = make([]byte, row.KeySize()) + } + keySize, _ := row.KeyTo(keyBuf.buf) + if _, ok := existingTermKeys[string(keyBuf.buf[:keySize])]; ok { + updateRows = append(updateRows, row) + delete(existingTermKeys, string(keyBuf.buf[:keySize])) + continue + } + } + addRows = append(addRows, row) + case *StoredRow: + if existingStoredKeys != nil { + if row.KeySize() > len(keyBuf.buf) { + keyBuf.buf = make([]byte, row.KeySize()) + } + keySize, _ := row.KeyTo(keyBuf.buf) + if _, ok := existingStoredKeys[string(keyBuf.buf[:keySize])]; ok { + updateRows = append(updateRows, row) + delete(existingStoredKeys, string(keyBuf.buf[:keySize])) + continue + } + } + addRows = append(addRows, row) + default: + updateRows = append(updateRows, row) + } + } + PutRowBuffer(keyBuf) + + // any of the existing rows that weren't updated need to be deleted + for existingTermKey := range existingTermKeys { + termFreqRow, err := NewTermFrequencyRowK([]byte(existingTermKey)) + if err == nil { + deleteRows = append(deleteRows, termFreqRow) + } + } + + // any of the existing stored fields that weren't updated need to be deleted + for existingStoredKey := range existingStoredKeys { + storedRow, err := NewStoredRowK([]byte(existingStoredKey)) + if err == nil { + deleteRows = append(deleteRows, storedRow) + } + } + + return addRows, updateRows, deleteRows +} + +func (udc *UpsideDownCouch) storeField(docID []byte, field index.Field, fieldIndex uint16, rows []IndexRow, backIndexStoredEntries []*BackIndexStoreEntry) ([]IndexRow, []*BackIndexStoreEntry) { + fieldType := field.EncodedFieldType() + storedRow := NewStoredRow(docID, fieldIndex, field.ArrayPositions(), fieldType, field.Value()) + + // record the back index entry + backIndexStoredEntry := BackIndexStoreEntry{Field: proto.Uint32(uint32(fieldIndex)), ArrayPositions: field.ArrayPositions()} + + return append(rows, storedRow), append(backIndexStoredEntries, &backIndexStoredEntry) +} + +func (udc *UpsideDownCouch) indexField(docID []byte, includeTermVectors bool, fieldIndex uint16, fieldLength int, tokenFreqs index.TokenFrequencies, rows []IndexRow, backIndexTermsEntries []*BackIndexTermsEntry) ([]IndexRow, []*BackIndexTermsEntry) { + fieldNorm := float32(1.0 / math.Sqrt(float64(fieldLength))) + + termFreqRows := make([]TermFrequencyRow, len(tokenFreqs)) + termFreqRowsUsed := 0 + + terms := make([]string, 0, len(tokenFreqs)) + for k, tf := range tokenFreqs { + termFreqRow := &termFreqRows[termFreqRowsUsed] + termFreqRowsUsed++ + + InitTermFrequencyRow(termFreqRow, tf.Term, fieldIndex, docID, + uint64(frequencyFromTokenFreq(tf)), fieldNorm) + + if includeTermVectors { + termFreqRow.vectors, rows = udc.termVectorsFromTokenFreq(fieldIndex, tf, rows) + } + + // record the back index entry + terms = append(terms, k) + + rows = append(rows, termFreqRow) + } + backIndexTermsEntry := BackIndexTermsEntry{Field: proto.Uint32(uint32(fieldIndex)), Terms: terms} + backIndexTermsEntries = append(backIndexTermsEntries, &backIndexTermsEntry) + + return rows, backIndexTermsEntries +} + +func (udc *UpsideDownCouch) Delete(id string) (err error) { + indexStart := time.Now() + + udc.writeMutex.Lock() + defer udc.writeMutex.Unlock() + + // open a reader for backindex lookup + var kvreader store.KVReader + kvreader, err = udc.store.Reader() + if err != nil { + return + } + + // first we lookup the backindex row for the doc id if it exists + // lookup the back index row + var backIndexRow *BackIndexRow + backIndexRow, err = backIndexRowForDoc(kvreader, index.IndexInternalID(id)) + if err != nil { + _ = kvreader.Close() + atomic.AddUint64(&udc.stats.errors, 1) + return + } + + err = kvreader.Close() + if err != nil { + return + } + + if backIndexRow == nil { + atomic.AddUint64(&udc.stats.deletes, 1) + return + } + + // start a writer for this delete + var kvwriter store.KVWriter + kvwriter, err = udc.store.Writer() + if err != nil { + return + } + defer func() { + if cerr := kvwriter.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + var deleteRowsAll [][]UpsideDownCouchRow + + deleteRows := udc.deleteSingle(id, backIndexRow, nil) + if len(deleteRows) > 0 { + deleteRowsAll = append(deleteRowsAll, deleteRows) + } + + err = udc.batchRows(kvwriter, nil, nil, deleteRowsAll) + if err == nil { + udc.m.Lock() + udc.docCount-- + udc.m.Unlock() + } + atomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart))) + if err == nil { + atomic.AddUint64(&udc.stats.deletes, 1) + } else { + atomic.AddUint64(&udc.stats.errors, 1) + } + return +} + +func (udc *UpsideDownCouch) deleteSingle(id string, backIndexRow *BackIndexRow, deleteRows []UpsideDownCouchRow) []UpsideDownCouchRow { + idBytes := []byte(id) + + for _, backIndexEntry := range backIndexRow.termsEntries { + for i := range backIndexEntry.Terms { + tfr := NewTermFrequencyRow([]byte(backIndexEntry.Terms[i]), uint16(*backIndexEntry.Field), idBytes, 0, 0) + deleteRows = append(deleteRows, tfr) + } + } + for _, se := range backIndexRow.storedEntries { + sf := NewStoredRow(idBytes, uint16(*se.Field), se.ArrayPositions, 'x', nil) + deleteRows = append(deleteRows, sf) + } + + // also delete the back entry itself + deleteRows = append(deleteRows, backIndexRow) + return deleteRows +} + +func decodeFieldType(typ byte, name string, pos []uint64, value []byte) document.Field { + switch typ { + case 't': + return document.NewTextField(name, pos, value) + case 'n': + return document.NewNumericFieldFromBytes(name, pos, value) + case 'd': + return document.NewDateTimeFieldFromBytes(name, pos, value) + case 'b': + return document.NewBooleanFieldFromBytes(name, pos, value) + case 'g': + return document.NewGeoPointFieldFromBytes(name, pos, value) + case 'i': + return document.NewIPFieldFromBytes(name, pos, value) + } + return nil +} + +func frequencyFromTokenFreq(tf *index.TokenFreq) int { + return tf.Frequency() +} + +func (udc *UpsideDownCouch) termVectorsFromTokenFreq(field uint16, tf *index.TokenFreq, rows []IndexRow) ([]*TermVector, []IndexRow) { + a := make([]TermVector, len(tf.Locations)) + rv := make([]*TermVector, len(tf.Locations)) + + for i, l := range tf.Locations { + var newFieldRow *FieldRow + fieldIndex := field + if l.Field != "" { + // lookup correct field + fieldIndex, newFieldRow = udc.fieldIndexOrNewRow(l.Field) + if newFieldRow != nil { + rows = append(rows, newFieldRow) + } + } + a[i] = TermVector{ + field: fieldIndex, + arrayPositions: l.ArrayPositions, + pos: uint64(l.Position), + start: uint64(l.Start), + end: uint64(l.End), + } + rv[i] = &a[i] + } + + return rv, rows +} + +func (udc *UpsideDownCouch) termFieldVectorsFromTermVectors(in []*TermVector) []*index.TermFieldVector { + if len(in) == 0 { + return nil + } + + a := make([]index.TermFieldVector, len(in)) + rv := make([]*index.TermFieldVector, len(in)) + + for i, tv := range in { + fieldName := udc.fieldCache.FieldIndexed(tv.field) + a[i] = index.TermFieldVector{ + Field: fieldName, + ArrayPositions: tv.arrayPositions, + Pos: tv.pos, + Start: tv.start, + End: tv.end, + } + rv[i] = &a[i] + } + return rv +} + +func (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) { + persistedCallback := batch.PersistedCallback() + if persistedCallback != nil { + defer persistedCallback(err) + } + analysisStart := time.Now() + + resultChan := make(chan *AnalysisResult, len(batch.IndexOps)) + + var numUpdates uint64 + var numPlainTextBytes uint64 + for _, doc := range batch.IndexOps { + if doc != nil { + numUpdates++ + numPlainTextBytes += doc.NumPlainTextBytes() + } + } + + if numUpdates > 0 { + go func() { + for k := range batch.IndexOps { + doc := batch.IndexOps[k] + if doc != nil { + // put the work on the queue + udc.analysisQueue.Queue(func() { + ar := udc.analyze(doc) + resultChan <- ar + }) + } + } + }() + } + + // retrieve back index rows concurrent with analysis + docBackIndexRowErr := error(nil) + docBackIndexRowCh := make(chan *docBackIndexRow, len(batch.IndexOps)) + + udc.writeMutex.Lock() + defer udc.writeMutex.Unlock() + + go func() { + defer close(docBackIndexRowCh) + + // open a reader for backindex lookup + var kvreader store.KVReader + kvreader, err = udc.store.Reader() + if err != nil { + docBackIndexRowErr = err + return + } + defer func() { + if cerr := kvreader.Close(); err == nil && cerr != nil { + docBackIndexRowErr = cerr + } + }() + + for docID, doc := range batch.IndexOps { + backIndexRow, err := backIndexRowForDoc(kvreader, index.IndexInternalID(docID)) + if err != nil { + docBackIndexRowErr = err + return + } + + docBackIndexRowCh <- &docBackIndexRow{docID, doc, backIndexRow} + } + }() + + // wait for analysis result + newRowsMap := make(map[string][]IndexRow) + var itemsDeQueued uint64 + for itemsDeQueued < numUpdates { + result := <-resultChan + newRowsMap[result.DocID] = result.Rows + itemsDeQueued++ + } + close(resultChan) + + atomic.AddUint64(&udc.stats.analysisTime, uint64(time.Since(analysisStart))) + + docsAdded := uint64(0) + docsDeleted := uint64(0) + + indexStart := time.Now() + + // prepare a list of rows + var addRowsAll [][]UpsideDownCouchRow + var updateRowsAll [][]UpsideDownCouchRow + var deleteRowsAll [][]UpsideDownCouchRow + + // add the internal ops + var updateRows []UpsideDownCouchRow + var deleteRows []UpsideDownCouchRow + + for internalKey, internalValue := range batch.InternalOps { + if internalValue == nil { + // delete + deleteInternalRow := NewInternalRow([]byte(internalKey), nil) + deleteRows = append(deleteRows, deleteInternalRow) + } else { + updateInternalRow := NewInternalRow([]byte(internalKey), internalValue) + updateRows = append(updateRows, updateInternalRow) + } + } + + if len(updateRows) > 0 { + updateRowsAll = append(updateRowsAll, updateRows) + } + if len(deleteRows) > 0 { + deleteRowsAll = append(deleteRowsAll, deleteRows) + } + + // process back index rows as they arrive + for dbir := range docBackIndexRowCh { + if dbir.doc == nil && dbir.backIndexRow != nil { + // delete + deleteRows := udc.deleteSingle(dbir.docID, dbir.backIndexRow, nil) + if len(deleteRows) > 0 { + deleteRowsAll = append(deleteRowsAll, deleteRows) + } + docsDeleted++ + } else if dbir.doc != nil { + addRows, updateRows, deleteRows := udc.mergeOldAndNew(dbir.backIndexRow, newRowsMap[dbir.docID]) + if len(addRows) > 0 { + addRowsAll = append(addRowsAll, addRows) + } + if len(updateRows) > 0 { + updateRowsAll = append(updateRowsAll, updateRows) + } + if len(deleteRows) > 0 { + deleteRowsAll = append(deleteRowsAll, deleteRows) + } + if dbir.backIndexRow == nil { + docsAdded++ + } + } + } + + if docBackIndexRowErr != nil { + return docBackIndexRowErr + } + + // start a writer for this batch + var kvwriter store.KVWriter + kvwriter, err = udc.store.Writer() + if err != nil { + return + } + + err = udc.batchRows(kvwriter, addRowsAll, updateRowsAll, deleteRowsAll) + if err != nil { + _ = kvwriter.Close() + atomic.AddUint64(&udc.stats.errors, 1) + return + } + + err = kvwriter.Close() + + atomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart))) + + if err == nil { + udc.m.Lock() + udc.docCount += docsAdded + udc.docCount -= docsDeleted + udc.m.Unlock() + atomic.AddUint64(&udc.stats.updates, numUpdates) + atomic.AddUint64(&udc.stats.deletes, docsDeleted) + atomic.AddUint64(&udc.stats.batches, 1) + atomic.AddUint64(&udc.stats.numPlainTextBytesIndexed, numPlainTextBytes) + } else { + atomic.AddUint64(&udc.stats.errors, 1) + } + + return +} + +func (udc *UpsideDownCouch) SetInternal(key, val []byte) (err error) { + internalRow := NewInternalRow(key, val) + udc.writeMutex.Lock() + defer udc.writeMutex.Unlock() + var writer store.KVWriter + writer, err = udc.store.Writer() + if err != nil { + return + } + defer func() { + if cerr := writer.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + batch := writer.NewBatch() + batch.Set(internalRow.Key(), internalRow.Value()) + + return writer.ExecuteBatch(batch) +} + +func (udc *UpsideDownCouch) DeleteInternal(key []byte) (err error) { + internalRow := NewInternalRow(key, nil) + udc.writeMutex.Lock() + defer udc.writeMutex.Unlock() + var writer store.KVWriter + writer, err = udc.store.Writer() + if err != nil { + return + } + defer func() { + if cerr := writer.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + batch := writer.NewBatch() + batch.Delete(internalRow.Key()) + return writer.ExecuteBatch(batch) +} + +func (udc *UpsideDownCouch) Reader() (index.IndexReader, error) { + kvr, err := udc.store.Reader() + if err != nil { + return nil, fmt.Errorf("error opening store reader: %v", err) + } + udc.m.RLock() + defer udc.m.RUnlock() + return &IndexReader{ + index: udc, + kvreader: kvr, + docCount: udc.docCount, + }, nil +} + +func (udc *UpsideDownCouch) Stats() json.Marshaler { + return udc.stats +} + +func (udc *UpsideDownCouch) StatsMap() map[string]interface{} { + return udc.stats.statsMap() +} + +func (udc *UpsideDownCouch) Advanced() (store.KVStore, error) { + return udc.store, nil +} + +func (udc *UpsideDownCouch) fieldIndexOrNewRow(name string) (uint16, *FieldRow) { + index, existed := udc.fieldCache.FieldNamed(name, true) + if !existed { + return index, NewFieldRow(index, name) + } + return index, nil +} + +func init() { + err := registry.RegisterIndexType(Name, NewUpsideDownCouch) + if err != nil { + panic(err) + } +} + +func backIndexRowForDoc(kvreader store.KVReader, docID index.IndexInternalID) (*BackIndexRow, error) { + // use a temporary row structure to build key + tempRow := BackIndexRow{ + doc: docID, + } + + keyBuf := GetRowBuffer() + if tempRow.KeySize() > len(keyBuf.buf) { + keyBuf.buf = make([]byte, 2*tempRow.KeySize()) + } + defer PutRowBuffer(keyBuf) + keySize, err := tempRow.KeyTo(keyBuf.buf) + if err != nil { + return nil, err + } + + value, err := kvreader.Get(keyBuf.buf[:keySize]) + if err != nil { + return nil, err + } + if value == nil { + return nil, nil + } + backIndexRow, err := NewBackIndexRowKV(keyBuf.buf[:keySize], value) + if err != nil { + return nil, err + } + return backIndexRow, nil +} diff --git a/index/upsidedown/upsidedown.pb.go b/index/upsidedown/upsidedown.pb.go new file mode 100644 index 0000000..5ce69ea --- /dev/null +++ b/index/upsidedown/upsidedown.pb.go @@ -0,0 +1,491 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.6 +// protoc v5.29.3 +// source: index/upsidedown/upsidedown.proto + +package upsidedown + +import ( + fmt "fmt" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + io "io" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +var ( + ErrInvalidLengthUpsidedown = fmt.Errorf("proto: negative length found during unmarshaling") +) + +type BackIndexTermsEntry struct { + state protoimpl.MessageState `protogen:"open.v1"` + Field *uint32 `protobuf:"varint,1,req,name=field" json:"field,omitempty"` + Terms []string `protobuf:"bytes,2,rep,name=terms" json:"terms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BackIndexTermsEntry) Reset() { + *x = BackIndexTermsEntry{} + mi := &file_index_upsidedown_upsidedown_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BackIndexTermsEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackIndexTermsEntry) ProtoMessage() {} + +func (x *BackIndexTermsEntry) ProtoReflect() protoreflect.Message { + mi := &file_index_upsidedown_upsidedown_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackIndexTermsEntry.ProtoReflect.Descriptor instead. +func (*BackIndexTermsEntry) Descriptor() ([]byte, []int) { + return file_index_upsidedown_upsidedown_proto_rawDescGZIP(), []int{0} +} + +func (x *BackIndexTermsEntry) GetField() uint32 { + if x != nil && x.Field != nil { + return *x.Field + } + return 0 +} + +func (x *BackIndexTermsEntry) GetTerms() []string { + if x != nil { + return x.Terms + } + return nil +} + +func (x *BackIndexTermsEntry) MarshalTo(data []byte) (n int, err error) { + var i int + _ = i + var l int + _ = l + if x.Field == nil { + return 0, fmt.Errorf("missing required `Field`") + } else { + data[i] = 0x8 + i++ + i = encodeVarintUpsidedown(data, i, uint64(*x.Field)) + } + if len(x.Terms) > 0 { + for _, s := range x.Terms { + data[i] = 0x12 + i++ + l = len(s) + for l >= 1<<7 { + data[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + data[i] = uint8(l) + i++ + i += copy(data[i:], s) + } + } + return i, nil +} + +func (x *BackIndexTermsEntry) Size() (n int) { + var l int + _ = l + if x.Field != nil { + n += 1 + sovUpsidedown(uint64(*x.Field)) + } + if len(x.Terms) > 0 { + for _, s := range x.Terms { + l = len(s) + n += 1 + l + sovUpsidedown(uint64(l)) + } + } + return n +} + +type BackIndexStoreEntry struct { + state protoimpl.MessageState `protogen:"open.v1"` + Field *uint32 `protobuf:"varint,1,req,name=field" json:"field,omitempty"` + ArrayPositions []uint64 `protobuf:"varint,2,rep,name=arrayPositions" json:"arrayPositions,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BackIndexStoreEntry) Reset() { + *x = BackIndexStoreEntry{} + mi := &file_index_upsidedown_upsidedown_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BackIndexStoreEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackIndexStoreEntry) ProtoMessage() {} + +func (x *BackIndexStoreEntry) ProtoReflect() protoreflect.Message { + mi := &file_index_upsidedown_upsidedown_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackIndexStoreEntry.ProtoReflect.Descriptor instead. +func (*BackIndexStoreEntry) Descriptor() ([]byte, []int) { + return file_index_upsidedown_upsidedown_proto_rawDescGZIP(), []int{1} +} + +func (x *BackIndexStoreEntry) GetField() uint32 { + if x != nil && x.Field != nil { + return *x.Field + } + return 0 +} + +func (x *BackIndexStoreEntry) GetArrayPositions() []uint64 { + if x != nil { + return x.ArrayPositions + } + return nil +} + +func (x *BackIndexStoreEntry) MarshalTo(data []byte) (n int, err error) { + var i int + _ = i + var l int + _ = l + if x.Field == nil { + return 0, fmt.Errorf("missing required `Field`") + } else { + data[i] = 0x8 + i++ + i = encodeVarintUpsidedown(data, i, uint64(*x.Field)) + } + if len(x.ArrayPositions) > 0 { + for _, num := range x.ArrayPositions { + data[i] = 0x10 + i++ + i = encodeVarintUpsidedown(data, i, uint64(num)) + } + } + return i, nil +} + +func (x *BackIndexStoreEntry) Size() (n int) { + var l int + _ = l + if x.Field != nil { + n += 1 + sovUpsidedown(uint64(*x.Field)) + } + if len(x.ArrayPositions) > 0 { + for _, e := range x.ArrayPositions { + n += 1 + sovUpsidedown(uint64(e)) + } + } + return n +} + +type BackIndexRowValue struct { + state protoimpl.MessageState `protogen:"open.v1"` + TermsEntries []*BackIndexTermsEntry `protobuf:"bytes,1,rep,name=termsEntries" json:"termsEntries,omitempty"` + StoredEntries []*BackIndexStoreEntry `protobuf:"bytes,2,rep,name=storedEntries" json:"storedEntries,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BackIndexRowValue) Reset() { + *x = BackIndexRowValue{} + mi := &file_index_upsidedown_upsidedown_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BackIndexRowValue) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackIndexRowValue) ProtoMessage() {} + +func (x *BackIndexRowValue) ProtoReflect() protoreflect.Message { + mi := &file_index_upsidedown_upsidedown_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackIndexRowValue.ProtoReflect.Descriptor instead. +func (*BackIndexRowValue) Descriptor() ([]byte, []int) { + return file_index_upsidedown_upsidedown_proto_rawDescGZIP(), []int{2} +} + +func (x *BackIndexRowValue) GetTermsEntries() []*BackIndexTermsEntry { + if x != nil { + return x.TermsEntries + } + return nil +} + +func (x *BackIndexRowValue) GetStoredEntries() []*BackIndexStoreEntry { + if x != nil { + return x.StoredEntries + } + return nil +} + +func (x *BackIndexRowValue) MarshalTo(data []byte) (n int, err error) { + var i int + _ = i + var l int + _ = l + if len(x.TermsEntries) > 0 { + for _, msg := range x.TermsEntries { + data[i] = 0xa + i++ + i = encodeVarintUpsidedown(data, i, uint64(msg.Size())) + n, err := msg.MarshalTo(data[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if len(x.StoredEntries) > 0 { + for _, msg := range x.StoredEntries { + data[i] = 0x12 + i++ + i = encodeVarintUpsidedown(data, i, uint64(msg.Size())) + n, err := msg.MarshalTo(data[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (x *BackIndexRowValue) Size() (n int) { + var l int + _ = l + if len(x.TermsEntries) > 0 { + for _, e := range x.TermsEntries { + l = e.Size() + n += 1 + l + sovUpsidedown(uint64(l)) + } + } + if len(x.StoredEntries) > 0 { + for _, e := range x.StoredEntries { + l = e.Size() + n += 1 + l + sovUpsidedown(uint64(l)) + } + } + return n +} + +func skipUpsidedown(data []byte) (n int, err error) { + l := len(data) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for { + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if data[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthUpsidedown + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipUpsidedown(data[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +func sovUpsidedown(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} + +func encodeVarintUpsidedown(data []byte, offset int, v uint64) int { + for v >= 1<<7 { + data[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + data[offset] = uint8(v) + return offset + 1 +} + +var File_index_upsidedown_upsidedown_proto protoreflect.FileDescriptor + +const file_index_upsidedown_upsidedown_proto_rawDesc = "" + + "\n" + + "!index/upsidedown/upsidedown.proto\"A\n" + + "\x13BackIndexTermsEntry\x12\x14\n" + + "\x05field\x18\x01 \x02(\rR\x05field\x12\x14\n" + + "\x05terms\x18\x02 \x03(\tR\x05terms\"S\n" + + "\x13BackIndexStoreEntry\x12\x14\n" + + "\x05field\x18\x01 \x02(\rR\x05field\x12&\n" + + "\x0earrayPositions\x18\x02 \x03(\x04R\x0earrayPositions\"\x89\x01\n" + + "\x11BackIndexRowValue\x128\n" + + "\ftermsEntries\x18\x01 \x03(\v2\x14.BackIndexTermsEntryR\ftermsEntries\x12:\n" + + "\rstoredEntries\x18\x02 \x03(\v2\x14.BackIndexStoreEntryR\rstoredEntries" + +var ( + file_index_upsidedown_upsidedown_proto_rawDescOnce sync.Once + file_index_upsidedown_upsidedown_proto_rawDescData []byte +) + +func file_index_upsidedown_upsidedown_proto_rawDescGZIP() []byte { + file_index_upsidedown_upsidedown_proto_rawDescOnce.Do(func() { + file_index_upsidedown_upsidedown_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_index_upsidedown_upsidedown_proto_rawDesc), len(file_index_upsidedown_upsidedown_proto_rawDesc))) + }) + return file_index_upsidedown_upsidedown_proto_rawDescData +} + +var file_index_upsidedown_upsidedown_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_index_upsidedown_upsidedown_proto_goTypes = []any{ + (*BackIndexTermsEntry)(nil), // 0: BackIndexTermsEntry + (*BackIndexStoreEntry)(nil), // 1: BackIndexStoreEntry + (*BackIndexRowValue)(nil), // 2: BackIndexRowValue +} +var file_index_upsidedown_upsidedown_proto_depIdxs = []int32{ + 0, // 0: BackIndexRowValue.termsEntries:type_name -> BackIndexTermsEntry + 1, // 1: BackIndexRowValue.storedEntries:type_name -> BackIndexStoreEntry + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_index_upsidedown_upsidedown_proto_init() } +func file_index_upsidedown_upsidedown_proto_init() { + if File_index_upsidedown_upsidedown_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_index_upsidedown_upsidedown_proto_rawDesc), len(file_index_upsidedown_upsidedown_proto_rawDesc)), + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_index_upsidedown_upsidedown_proto_goTypes, + DependencyIndexes: file_index_upsidedown_upsidedown_proto_depIdxs, + MessageInfos: file_index_upsidedown_upsidedown_proto_msgTypes, + }.Build() + File_index_upsidedown_upsidedown_proto = out.File + file_index_upsidedown_upsidedown_proto_goTypes = nil + file_index_upsidedown_upsidedown_proto_depIdxs = nil +} diff --git a/index/upsidedown/upsidedown.proto b/index/upsidedown/upsidedown.proto new file mode 100644 index 0000000..cf0492a --- /dev/null +++ b/index/upsidedown/upsidedown.proto @@ -0,0 +1,14 @@ +message BackIndexTermsEntry { + required uint32 field = 1; + repeated string terms = 2; +} + +message BackIndexStoreEntry { + required uint32 field = 1; + repeated uint64 arrayPositions = 2; +} + +message BackIndexRowValue { + repeated BackIndexTermsEntry termsEntries = 1; + repeated BackIndexStoreEntry storedEntries = 2; +} diff --git a/index/upsidedown/upsidedown_test.go b/index/upsidedown/upsidedown_test.go new file mode 100644 index 0000000..36ffed0 --- /dev/null +++ b/index/upsidedown/upsidedown_test.go @@ -0,0 +1,1529 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "context" + "log" + "reflect" + "regexp" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" + regexpTokenizer "github.com/blevesearch/bleve/v2/analysis/tokenizer/regexp" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/null" + "github.com/blevesearch/bleve/v2/registry" + index "github.com/blevesearch/bleve_index_api" +) + +var testAnalyzer = &analysis.DefaultAnalyzer{ + Tokenizer: regexpTokenizer.NewRegexpTokenizer(regexp.MustCompile(`\w+`)), +} + +func TestIndexOpenReopen(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + + var expectedCount uint64 + reader, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err := reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + // opening the database should have inserted a version + expectedLength := uint64(1) + rowCount, err := idx.(*UpsideDownCouch).rowCount() + if err != nil { + t.Error(err) + } + if rowCount != expectedLength { + t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) + } + + // now close it + err = idx.Close() + if err != nil { + t.Fatal(err) + } + + idx, err = NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + + // now close it + err = idx.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestIndexInsert(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + var expectedCount uint64 + reader, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err := reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err = reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + // should have 4 rows (1 for version, 1 for schema field, and 1 for single term, and 1 for the term count, and 1 for the back index entry) + expectedLength := uint64(1 + 1 + 1 + 1 + 1) + rowCount, err := idx.(*UpsideDownCouch).rowCount() + if err != nil { + t.Error(err) + } + if rowCount != expectedLength { + t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) + } +} + +func TestIndexInsertThenDelete(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + var expectedCount uint64 + reader, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err := reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + doc2 := document.NewDocument("2") + doc2.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc2) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err = reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + err = idx.Delete("1") + if err != nil { + t.Errorf("Error deleting entry from index: %v", err) + } + expectedCount-- + + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err = reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + err = idx.Delete("2") + if err != nil { + t.Errorf("Error deleting entry from index: %v", err) + } + expectedCount-- + + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err = reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + // should have 2 rows (1 for version, 1 for schema field, 1 for dictionary row garbage) + expectedLength := uint64(1 + 1 + 1) + rowCount, err := idx.(*UpsideDownCouch).rowCount() + if err != nil { + t.Error(err) + } + if rowCount != expectedLength { + t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) + } +} + +func TestIndexInsertThenUpdate(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + // this update should overwrite one term, and introduce one new one + doc = document.NewDocument("1") + doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test fail"), testAnalyzer)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error deleting entry from index: %v", err) + } + + // should have 2 rows (1 for version, 1 for schema field, and 2 for the two term, and 2 for the term counts, and 1 for the back index entry) + expectedLength := uint64(1 + 1 + 2 + 2 + 1) + rowCount, err := idx.(*UpsideDownCouch).rowCount() + if err != nil { + t.Error(err) + } + if rowCount != expectedLength { + t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) + } + + // now do another update that should remove one of the terms + doc = document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("fail"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error deleting entry from index: %v", err) + } + + // should have 2 rows (1 for version, 1 for schema field, and 1 for the remaining term, and 2 for the term diciontary, and 1 for the back index entry) + expectedLength = uint64(1 + 1 + 1 + 2 + 1) + rowCount, err = idx.(*UpsideDownCouch).rowCount() + if err != nil { + t.Error(err) + } + if rowCount != expectedLength { + t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) + } +} + +func TestIndexInsertMultiple(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + + var expectedCount uint64 + + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + doc = document.NewDocument("2") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + // should have 4 rows (1 for version, 1 for schema field, and 2 for single term, and 1 for the term count, and 2 for the back index entries) + expectedLength := uint64(1 + 1 + 2 + 1 + 2) + rowCount, err := idx.(*UpsideDownCouch).rowCount() + if err != nil { + t.Error(err) + } + if rowCount != expectedLength { + t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) + } + + // close, reopen and add one more to test that counting works correctly + err = idx.Close() + if err != nil { + t.Fatal(err) + } + + idx, err = NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + doc = document.NewDocument("3") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + reader, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err := reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestIndexInsertWithStore(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + var expectedCount uint64 + reader, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err := reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + doc := document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), index.IndexField|index.StoreField)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err = reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + // should have 6 rows (1 for version, 1 for schema field, and 1 for single term, and 1 for the stored field and 1 for the term count, and 1 for the back index entry) + expectedLength := uint64(1 + 1 + 1 + 1 + 1 + 1) + rowCount, err := idx.(*UpsideDownCouch).rowCount() + if err != nil { + t.Error(err) + } + if rowCount != expectedLength { + t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + storedDocInt, err := indexReader.Document("1") + if err != nil { + t.Error(err) + } + storedDoc := storedDocInt.(*document.Document) + + if len(storedDoc.Fields) != 1 { + t.Errorf("expected 1 stored field, got %d", len(storedDoc.Fields)) + } + textField, ok := storedDoc.Fields[0].(*document.TextField) + if !ok { + t.Errorf("expected text field") + } + if string(textField.Value()) != "test" { + t.Errorf("expected field content 'test', got '%s'", string(textField.Value())) + } +} + +func TestIndexInternalCRUD(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + + // get something that doesn't exist yet + val, err := indexReader.GetInternal([]byte("key")) + if err != nil { + t.Error(err) + } + if val != nil { + t.Errorf("expected nil, got %s", val) + } + + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + + // set + err = idx.SetInternal([]byte("key"), []byte("abc")) + if err != nil { + t.Error(err) + } + + indexReader2, err := idx.Reader() + if err != nil { + t.Error(err) + } + + // get + val, err = indexReader2.GetInternal([]byte("key")) + if err != nil { + t.Error(err) + } + if string(val) != "abc" { + t.Errorf("expected %s, got '%s'", "abc", val) + } + + err = indexReader2.Close() + if err != nil { + t.Fatal(err) + } + + // delete + err = idx.DeleteInternal([]byte("key")) + if err != nil { + t.Error(err) + } + + indexReader3, err := idx.Reader() + if err != nil { + t.Error(err) + } + + // get again + val, err = indexReader3.GetInternal([]byte("key")) + if err != nil { + t.Error(err) + } + if val != nil { + t.Errorf("expected nil, got %s", val) + } + + err = indexReader3.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestIndexBatch(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + var expectedCount uint64 + + // first create 2 docs the old fashioned way + doc := document.NewDocument("1") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + doc = document.NewDocument("2") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test2"))) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + // now create a batch which does 3 things + // insert new doc + // update existing doc + // delete existing doc + // net document count change 0 + + batch := index.NewBatch() + doc = document.NewDocument("3") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test3"))) + batch.Update(doc) + doc = document.NewDocument("2") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test2updated"))) + batch.Update(doc) + batch.Delete("1") + + err = idx.Batch(batch) + if err != nil { + t.Error(err) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + docCount, err := indexReader.DocCount() + if err != nil { + t.Fatal(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + + docIDReader, err := indexReader.DocIDReaderAll() + if err != nil { + t.Error(err) + } + var docIds []index.IndexInternalID + docID, err := docIDReader.Next() + for docID != nil && err == nil { + docIds = append(docIds, docID) + docID, err = docIDReader.Next() + } + if err != nil { + t.Error(err) + } + expectedDocIds := []index.IndexInternalID{index.IndexInternalID("2"), index.IndexInternalID("3")} + if !reflect.DeepEqual(docIds, expectedDocIds) { + t.Errorf("expected ids: %v, got ids: %v", expectedDocIds, docIds) + } +} + +func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + var expectedCount uint64 + reader, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err := reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + doc := document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), index.IndexField|index.StoreField)) + doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, index.IndexField|index.StoreField)) + df, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), time.RFC3339, index.IndexField|index.StoreField) + if err != nil { + t.Error(err) + } + doc.AddField(df) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + expectedCount++ + + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err = reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } + + // should have 72 rows + // 1 for version + // 3 for schema fields + // 1 for text term + // 16 for numeric terms + // 16 for date terms + // 3 for the stored field + // 1 for the text term count + // 16 for numeric term counts + // 16 for date term counts + // 1 for the back index entry + expectedLength := uint64(1 + 3 + 1 + (64 / document.DefaultPrecisionStep) + (64 / document.DefaultPrecisionStep) + 3 + 1 + (64 / document.DefaultPrecisionStep) + (64 / document.DefaultPrecisionStep) + 1) + rowCount, err := idx.(*UpsideDownCouch).rowCount() + if err != nil { + t.Error(err) + } + if rowCount != expectedLength { + t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + + storedDocInt, err := indexReader.Document("1") + if err != nil { + t.Error(err) + } + storedDoc := storedDocInt.(*document.Document) + + err = indexReader.Close() + if err != nil { + t.Error(err) + } + + if len(storedDoc.Fields) != 3 { + t.Errorf("expected 3 stored field, got %d", len(storedDoc.Fields)) + } + textField, ok := storedDoc.Fields[0].(*document.TextField) + if !ok { + t.Errorf("expected text field") + } + if string(textField.Value()) != "test" { + t.Errorf("expected field content 'test', got '%s'", string(textField.Value())) + } + numField, ok := storedDoc.Fields[1].(*document.NumericField) + if !ok { + t.Errorf("expected numeric field") + } + numFieldNumer, err := numField.Number() + if err != nil { + t.Error(err) + } else { + if numFieldNumer != 35.99 { + t.Errorf("expected numeric value 35.99, got %f", numFieldNumer) + } + } + dateField, ok := storedDoc.Fields[2].(*document.DateTimeField) + if !ok { + t.Errorf("expected date field") + } + dateFieldDate, _, err := dateField.DateTime() + if err != nil { + t.Error(err) + } else { + if dateFieldDate != time.Unix(0, 0).UTC() { + t.Errorf("expected date value unix epoch, got %v", dateFieldDate) + } + } + + // now update the document, but omit one of the fields + doc = document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testup"), index.IndexField|index.StoreField)) + doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 36.99, index.IndexField|index.StoreField)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + indexReader2, err := idx.Reader() + if err != nil { + t.Error(err) + } + + // expected doc count shouldn't have changed + docCount, err = indexReader2.DocCount() + if err != nil { + t.Fatal(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + + // should only get 2 fields back now though + storedDocInt, err = indexReader2.Document("1") + if err != nil { + t.Error(err) + } + storedDoc = storedDocInt.(*document.Document) + + err = indexReader2.Close() + if err != nil { + t.Error(err) + } + + if len(storedDoc.Fields) != 2 { + t.Errorf("expected 3 stored field, got %d", len(storedDoc.Fields)) + } + textField, ok = storedDoc.Fields[0].(*document.TextField) + if !ok { + t.Errorf("expected text field") + } + if string(textField.Value()) != "testup" { + t.Errorf("expected field content 'testup', got '%s'", string(textField.Value())) + } + numField, ok = storedDoc.Fields[1].(*document.NumericField) + if !ok { + t.Errorf("expected numeric field") + } + numFieldNumer, err = numField.Number() + if err != nil { + t.Error(err) + } else { + if numFieldNumer != 36.99 { + t.Errorf("expected numeric value 36.99, got %f", numFieldNumer) + } + } + + // now delete the document + err = idx.Delete("1") + if err != nil { + t.Errorf("Error deleting entry from index: %v", err) + } + + expectedCount-- + + // expected doc count shouldn't have changed + reader, err = idx.Reader() + if err != nil { + t.Fatal(err) + } + docCount, err = reader.DocCount() + if err != nil { + t.Error(err) + } + if docCount != expectedCount { + t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) + } + err = reader.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestIndexInsertFields(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + doc := document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), index.IndexField|index.StoreField)) + doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, index.IndexField|index.StoreField)) + dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), time.RFC3339, index.IndexField|index.StoreField) + if err != nil { + t.Error(err) + } + doc.AddField(dateField) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + fields, err := indexReader.Fields() + if err != nil { + t.Error(err) + } else { + expectedFields := []string{"name", "age", "unixEpoch"} + if !reflect.DeepEqual(fields, expectedFields) { + t.Errorf("expected fields: %v, got %v", expectedFields, fields) + } + } +} + +func TestIndexUpdateComposites(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + doc := document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), index.IndexField|index.StoreField)) + doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), index.IndexField|index.StoreField)) + doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, index.IndexField)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + // should have 72 rows + // 1 for version + // 3 for schema fields + // 4 for text term + // 2 for the stored field + // 4 for the text term count + // 1 for the back index entry + expectedLength := uint64(1 + 3 + 4 + 2 + 4 + 1) + rowCount, err := idx.(*UpsideDownCouch).rowCount() + if err != nil { + t.Error(err) + } + if rowCount != expectedLength { + t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) + } + + // now lets update it + doc = document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testupdated"), index.IndexField|index.StoreField)) + doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("misterupdated"), index.IndexField|index.StoreField)) + doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, index.IndexField)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + // make sure new values are in index + storedDocInt, err := indexReader.Document("1") + if err != nil { + t.Error(err) + } + storedDoc := storedDocInt.(*document.Document) + if len(storedDoc.Fields) != 2 { + t.Errorf("expected 2 stored field, got %d", len(storedDoc.Fields)) + } + textField, ok := storedDoc.Fields[0].(*document.TextField) + if !ok { + t.Errorf("expected text field") + } + if string(textField.Value()) != "testupdated" { + t.Errorf("expected field content 'test', got '%s'", string(textField.Value())) + } + + // should have the same row count as before, plus 4 term dictionary garbage rows + expectedLength += 4 + rowCount, err = idx.(*UpsideDownCouch).rowCount() + if err != nil { + t.Error(err) + } + if rowCount != expectedLength { + t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) + } +} + +func TestIndexFieldsMisc(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + doc := document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), index.IndexField|index.StoreField)) + doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), index.IndexField|index.StoreField)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + fieldName1 := idx.(*UpsideDownCouch).fieldCache.FieldIndexed(0) + if fieldName1 != "name" { + t.Errorf("expected field named 'name', got '%s'", fieldName1) + } + fieldName2 := idx.(*UpsideDownCouch).fieldCache.FieldIndexed(1) + if fieldName2 != "title" { + t.Errorf("expected field named 'title', got '%s'", fieldName2) + } + fieldName3 := idx.(*UpsideDownCouch).fieldCache.FieldIndexed(2) + if fieldName3 != "" { + t.Errorf("expected field named '', got '%s'", fieldName3) + } +} + +func TestIndexTermReaderCompositeFields(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + doc := document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), index.IndexField|index.StoreField|index.IncludeTermVectors)) + doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), index.IndexField|index.StoreField|index.IncludeTermVectors)) + doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, index.IndexField|index.IncludeTermVectors)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + termFieldReader, err := indexReader.TermFieldReader(context.TODO(), []byte("mister"), "_all", true, true, true) + if err != nil { + t.Error(err) + } + + tfd, err := termFieldReader.Next(nil) + for tfd != nil && err == nil { + if !tfd.ID.Equals(index.IndexInternalID("1")) { + t.Errorf("expected to find document id 1") + } + tfd, err = termFieldReader.Next(nil) + } + if err != nil { + t.Error(err) + } +} + +func TestIndexDocValueReader(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + doc := document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), index.IndexField|index.StoreField|index.IncludeTermVectors)) + doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), index.IndexField|index.StoreField|index.IncludeTermVectors)) + err = idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + + indexReader, err := idx.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + actualFieldTerms := make(fieldTerms) + + dvr, err := indexReader.DocValueReader([]string{"name", "title"}) + if err != nil { + t.Error(err) + } + + err = dvr.VisitDocValues(index.IndexInternalID("1"), func(field string, term []byte) { + actualFieldTerms[field] = append(actualFieldTerms[field], string(term)) + }) + if err != nil { + t.Error(err) + } + expectedFieldTerms := fieldTerms{ + "name": []string{"test"}, + "title": []string{"mister"}, + } + if !reflect.DeepEqual(actualFieldTerms, expectedFieldTerms) { + t.Errorf("expected field terms: %#v, got: %#v", expectedFieldTerms, actualFieldTerms) + } +} + +func BenchmarkBatch(b *testing.B) { + cache := registry.NewCache() + analyzer, err := cache.AnalyzerNamed(standard.Name) + if err != nil { + b.Fatal(err) + } + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(null.Name, nil, analysisQueue) + if err != nil { + b.Fatal(err) + } + err = idx.Open() + if err != nil { + b.Fatal(err) + } + + batch := index.NewBatch() + for i := 0; i < 100; i++ { + d := document.NewDocument(strconv.Itoa(i)) + f := document.NewTextFieldWithAnalyzer("desc", nil, bleveWikiArticle1K, analyzer) + d.AddField(f) + batch.Update(d) + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + err = idx.Batch(batch) + if err != nil { + b.Fatal(err) + } + } +} + +func TestConcurrentUpdate(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + // do some concurrent updates + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func(i int) { + doc := document.NewDocument("1") + doc.AddField(document.NewTextFieldWithIndexingOptions(strconv.Itoa(i), []uint64{}, []byte(strconv.Itoa(i)), index.StoreField)) + err := idx.Update(doc) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + wg.Done() + }(i) + } + wg.Wait() + + // now load the name field and see what we get + r, err := idx.Reader() + if err != nil { + log.Fatal(err) + } + + docInt, err := r.Document("1") + if err != nil { + log.Fatal(err) + } + doc := docInt.(*document.Document) + + if len(doc.Fields) > 1 { + t.Errorf("expected single field, found %d", len(doc.Fields)) + } + + err = r.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestLargeField(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + var largeFieldValue []byte + for len(largeFieldValue) < RowBufferSize { + largeFieldValue = append(largeFieldValue, bleveWikiArticle1K...) + } + t.Logf("large field size: %d", len(largeFieldValue)) + + d := document.NewDocument("large") + f := document.NewTextFieldWithIndexingOptions("desc", nil, largeFieldValue, index.IndexField|index.StoreField) + d.AddField(f) + + err = idx.Update(d) + if err != nil { + t.Fatal(err) + } +} + +func TestIndexBatchPersistedCallbackWithErrorUpsideDown(t *testing.T) { + defer func() { + err := DestroyTest() + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Errorf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + var callbackExecuted bool + batch := index.NewBatch() + batch.SetPersistedCallback(func(e error) { + callbackExecuted = true + }) + // By using a really large ID, we ensure that the batch will fail, + // because the key generated by upside down will be too large for BoltDB + reallyBigId := strings.Repeat("x", 32768+1) + doc := document.NewDocument(reallyBigId) + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test3"))) + batch.Update(doc) + + _ = idx.Batch(batch) + // don't fail on this error, that isn't what we're testing + + if !callbackExecuted { + t.Fatal("expected callback to fire, it did not") + } +} + +// fieldTerms contains the terms used by a document, keyed by field +type fieldTerms map[string][]string + +// FieldsNotYetCached returns a list of fields not yet cached out of a larger list of fields +func (f fieldTerms) FieldsNotYetCached(fields []string) []string { + rv := make([]string, 0, len(fields)) + for _, field := range fields { + if _, ok := f[field]; !ok { + rv = append(rv, field) + } + } + return rv +} + +// Merge will combine two fieldTerms +// it assumes that the terms lists are complete (thus do not need to be merged) +// field terms from the other list always replace the ones in the receiver +func (f fieldTerms) Merge(other fieldTerms) { + for field, terms := range other { + f[field] = terms + } +} diff --git a/index_alias.go b/index_alias.go new file mode 100644 index 0000000..7a85d72 --- /dev/null +++ b/index_alias.go @@ -0,0 +1,37 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +// An IndexAlias is a wrapper around one or more +// Index objects. It has two distinct modes of +// operation. +// 1. When it points to a single index, ALL index +// operations are valid and will be passed through +// to the underlying index. +// 2. When it points to more than one index, the only +// valid operation is Search. In this case the +// search will be performed across all the +// underlying indexes and the results merged. +// Calls to Add/Remove/Swap the underlying indexes +// are atomic, so you can safely change the +// underlying Index objects while other components +// are performing operations. +type IndexAlias interface { + Index + + Add(i ...Index) + Remove(i ...Index) + Swap(in, out []Index) +} diff --git a/index_alias_impl.go b/index_alias_impl.go new file mode 100644 index 0000000..c58891d --- /dev/null +++ b/index_alias_impl.go @@ -0,0 +1,1312 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "context" + "fmt" + "sort" + "sync" + "time" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/collector" + "github.com/blevesearch/bleve/v2/search/query" + index "github.com/blevesearch/bleve_index_api" +) + +type indexAliasImpl struct { + name string + indexes []Index + mutex sync.RWMutex + open bool + // if all the indexes in that alias have the same mapping + // then the user can set the mapping here to avoid + // checking the mapping of each index in the alias + mapping mapping.IndexMapping +} + +// NewIndexAlias creates a new IndexAlias over the provided +// Index objects. +func NewIndexAlias(indexes ...Index) *indexAliasImpl { + return &indexAliasImpl{ + name: "alias", + indexes: indexes, + open: true, + } +} + +// VisitIndexes invokes the visit callback on every +// indexes included in the index alias. +func (i *indexAliasImpl) VisitIndexes(visit func(Index)) { + i.mutex.RLock() + for _, idx := range i.indexes { + visit(idx) + } + i.mutex.RUnlock() +} + +func (i *indexAliasImpl) isAliasToSingleIndex() error { + if len(i.indexes) < 1 { + return ErrorAliasEmpty + } else if len(i.indexes) > 1 { + return ErrorAliasMulti + } + return nil +} + +func (i *indexAliasImpl) Index(id string, data interface{}) error { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + err := i.isAliasToSingleIndex() + if err != nil { + return err + } + + return i.indexes[0].Index(id, data) +} + +func (i *indexAliasImpl) IndexSynonym(id string, collection string, definition *SynonymDefinition) error { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + err := i.isAliasToSingleIndex() + if err != nil { + return err + } + + if si, ok := i.indexes[0].(SynonymIndex); ok { + return si.IndexSynonym(id, collection, definition) + } + return ErrorSynonymSearchNotSupported +} + +func (i *indexAliasImpl) Train(batch *Batch) error { + i.mutex.RLock() + defer i.mutex.RUnlock() + if !i.open { + return ErrorIndexClosed + } + + err := i.isAliasToSingleIndex() + if err != nil { + return err + } + + if vi, ok := i.indexes[0].(TrainableIndex); ok { + return vi.Train(batch) + } + return ErrorTrainingNotSupported +} + +func (i *indexAliasImpl) Delete(id string) error { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + err := i.isAliasToSingleIndex() + if err != nil { + return err + } + + return i.indexes[0].Delete(id) +} + +func (i *indexAliasImpl) Batch(b *Batch) error { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + err := i.isAliasToSingleIndex() + if err != nil { + return err + } + + return i.indexes[0].Batch(b) +} + +func (i *indexAliasImpl) Document(id string) (index.Document, error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return nil, ErrorIndexClosed + } + + err := i.isAliasToSingleIndex() + if err != nil { + return nil, err + } + + return i.indexes[0].Document(id) +} + +func (i *indexAliasImpl) DocCount() (uint64, error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + rv := uint64(0) + + if !i.open { + return 0, ErrorIndexClosed + } + + for _, index := range i.indexes { + otherCount, err := index.DocCount() + if err == nil { + rv += otherCount + } + // tolerate errors to produce partial counts + } + + return rv, nil +} + +func (i *indexAliasImpl) Search(req *SearchRequest) (*SearchResult, error) { + return i.SearchInContext(context.Background(), req) +} + +func (i *indexAliasImpl) SearchInContext(ctx context.Context, req *SearchRequest) (*SearchResult, error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return nil, ErrorIndexClosed + } + + if len(i.indexes) < 1 { + return nil, ErrorAliasEmpty + } + + if _, ok := ctx.Value(search.PreSearchKey).(bool); ok { + // since preSearchKey is set, it means that the request + // is being executed as part of a preSearch, which + // indicates that this index alias is set as an Index + // in another alias, so we need to do a preSearch search + // and NOT a real search + bm25PreSearch := isBM25Enabled(i.mapping) + flags := &preSearchFlags{ + knn: requestHasKNN(req), + synonyms: !isMatchNoneQuery(req.Query), + bm25: bm25PreSearch, + } + return preSearchDataSearch(ctx, req, flags, i.indexes...) + } + + // at this point we know we are doing a real search + // either after a preSearch is done, or directly + // on the alias + + // check if request has preSearchData which would indicate that the + // request has already been preSearched and we can skip the + // preSearch step now, we call an optional function to + // redistribute the preSearchData to the individual indexes + // if necessary + var preSearchData map[string]map[string]interface{} + if req.PreSearchData != nil { + var err error + preSearchData, err = redistributePreSearchData(req, i.indexes) + if err != nil { + return nil, err + } + } + + // short circuit the simple case + if len(i.indexes) == 1 { + if preSearchData != nil { + req.PreSearchData = preSearchData[i.indexes[0].Name()] + } + return i.indexes[0].SearchInContext(ctx, req) + } + + // rescorer will be set if score fusion is supposed to happen + // at this alias (root alias), else will be nil + var rescorer *rescorer + if _, ok := ctx.Value(search.ScoreFusionKey).(bool); !ok { + // new context will be used in internal functions to collect data + // as suitable for fusion. Rescorer is used for rescoring + // using fusion algorithms. + if IsScoreFusionRequested(req) { + ctx = context.WithValue(ctx, search.ScoreFusionKey, true) + rescorer = newRescorer(req) + rescorer.prepareSearchRequest() + defer rescorer.restoreSearchRequest() + } + } + + // at this stage we know we have multiple indexes + // check if preSearchData needs to be gathered from all indexes + // before executing the query + var err error + // only perform preSearch if + // - the request does not already have preSearchData + // - the request requires preSearch + var preSearchDuration time.Duration + var sr *SearchResult + + // fusionKnnHits stores the KnnHits at the root alias. + // This is used with score fusion in case there is no need to + // send the knn hits to the leaf indexes in search phase. + // Refer to constructPreSearchDataAndFusionKnnHits for more info. + // This variable is left nil if we have to send the knn hits to leaf + // indexes again, else contains the knn hits if not required. + var fusionKnnHits search.DocumentMatchCollection + flags, err := preSearchRequired(ctx, req, i.mapping) + if err != nil { + return nil, err + } + if req.PreSearchData == nil && flags != nil { + searchStart := time.Now() + preSearchResult, err := preSearch(ctx, req, flags, i.indexes...) + if err != nil { + return nil, err + } + + // check if the preSearch result has any errors and if so + // return the search result as is without executing the query + // so that the errors are not lost + if preSearchResult.Status.Failed > 0 || len(preSearchResult.Status.Errors) > 0 { + return preSearchResult, nil + } + // finalize the preSearch result now + finalizePreSearchResult(req, flags, preSearchResult) + + // if there are no errors, then merge the data in the preSearch result + // and construct the preSearchData to be used in the actual search + // if the request is satisfied by the preSearch result, then we can + // directly return the preSearch result as the final result + if requestSatisfiedByPreSearch(req, flags) { + sr = finalizeSearchResult(ctx, req, preSearchResult, rescorer) + // no need to run the 2nd phase MultiSearch(..) + } else { + preSearchData, fusionKnnHits, err = constructPreSearchDataAndFusionKnnHits(req, flags, preSearchResult, rescorer, i.indexes) + if err != nil { + return nil, err + } + } + preSearchDuration = time.Since(searchStart) + } + + // check if search result was generated as part of preSearch itself + if sr == nil { + multiSearchParams := &multiSearchParams{preSearchData, rescorer, fusionKnnHits} + sr, err = MultiSearch(ctx, req, multiSearchParams, i.indexes...) + if err != nil { + return nil, err + } + } + sr.Took += preSearchDuration + return sr, nil +} + +func (i *indexAliasImpl) Fields() ([]string, error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return nil, ErrorIndexClosed + } + + err := i.isAliasToSingleIndex() + if err != nil { + return nil, err + } + + return i.indexes[0].Fields() +} + +func (i *indexAliasImpl) FieldDict(field string) (index.FieldDict, error) { + i.mutex.RLock() + + if !i.open { + i.mutex.RUnlock() + return nil, ErrorIndexClosed + } + + err := i.isAliasToSingleIndex() + if err != nil { + i.mutex.RUnlock() + return nil, err + } + + fieldDict, err := i.indexes[0].FieldDict(field) + if err != nil { + i.mutex.RUnlock() + return nil, err + } + + return &indexAliasImplFieldDict{ + index: i, + fieldDict: fieldDict, + }, nil +} + +func (i *indexAliasImpl) FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) { + i.mutex.RLock() + + if !i.open { + i.mutex.RUnlock() + return nil, ErrorIndexClosed + } + + err := i.isAliasToSingleIndex() + if err != nil { + i.mutex.RUnlock() + return nil, err + } + + fieldDict, err := i.indexes[0].FieldDictRange(field, startTerm, endTerm) + if err != nil { + i.mutex.RUnlock() + return nil, err + } + + return &indexAliasImplFieldDict{ + index: i, + fieldDict: fieldDict, + }, nil +} + +func (i *indexAliasImpl) FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) { + i.mutex.RLock() + + if !i.open { + i.mutex.RUnlock() + return nil, ErrorIndexClosed + } + + err := i.isAliasToSingleIndex() + if err != nil { + i.mutex.RUnlock() + return nil, err + } + + fieldDict, err := i.indexes[0].FieldDictPrefix(field, termPrefix) + if err != nil { + i.mutex.RUnlock() + return nil, err + } + + return &indexAliasImplFieldDict{ + index: i, + fieldDict: fieldDict, + }, nil +} + +func (i *indexAliasImpl) Close() error { + i.mutex.Lock() + defer i.mutex.Unlock() + + i.open = false + return nil +} + +// SetIndexMapping sets the mapping for the alias and must be used +// ONLY when all the indexes in the alias have the same mapping. +// This is to avoid checking the mapping of each index in the alias +// when executing a search request. +func (i *indexAliasImpl) SetIndexMapping(m mapping.IndexMapping) error { + i.mutex.Lock() + defer i.mutex.Unlock() + if !i.open { + return ErrorIndexClosed + } + i.mapping = m + return nil +} + +func (i *indexAliasImpl) Mapping() mapping.IndexMapping { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return nil + } + + // if the mapping is already set, return it + if i.mapping != nil { + return i.mapping + } + + err := i.isAliasToSingleIndex() + if err != nil { + return nil + } + + return i.indexes[0].Mapping() +} + +func (i *indexAliasImpl) Stats() *IndexStat { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return nil + } + + err := i.isAliasToSingleIndex() + if err != nil { + return nil + } + + return i.indexes[0].Stats() +} + +func (i *indexAliasImpl) StatsMap() map[string]interface{} { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return nil + } + + err := i.isAliasToSingleIndex() + if err != nil { + return nil + } + + return i.indexes[0].StatsMap() +} + +func (i *indexAliasImpl) GetInternal(key []byte) ([]byte, error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return nil, ErrorIndexClosed + } + + err := i.isAliasToSingleIndex() + if err != nil { + return nil, err + } + + return i.indexes[0].GetInternal(key) +} + +func (i *indexAliasImpl) SetInternal(key, val []byte) error { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + err := i.isAliasToSingleIndex() + if err != nil { + return err + } + + return i.indexes[0].SetInternal(key, val) +} + +func (i *indexAliasImpl) DeleteInternal(key []byte) error { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + err := i.isAliasToSingleIndex() + if err != nil { + return err + } + + return i.indexes[0].DeleteInternal(key) +} + +func (i *indexAliasImpl) Advanced() (index.Index, error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return nil, ErrorIndexClosed + } + + err := i.isAliasToSingleIndex() + if err != nil { + return nil, err + } + + return i.indexes[0].Advanced() +} + +func (i *indexAliasImpl) Add(indexes ...Index) { + i.mutex.Lock() + defer i.mutex.Unlock() + + i.indexes = append(i.indexes, indexes...) +} + +func (i *indexAliasImpl) removeSingle(index Index) { + for pos, in := range i.indexes { + if in == index { + i.indexes = append(i.indexes[:pos], i.indexes[pos+1:]...) + break + } + } +} + +func (i *indexAliasImpl) Remove(indexes ...Index) { + i.mutex.Lock() + defer i.mutex.Unlock() + + for _, in := range indexes { + i.removeSingle(in) + } +} + +func (i *indexAliasImpl) Swap(in, out []Index) { + i.mutex.Lock() + defer i.mutex.Unlock() + + // add + i.indexes = append(i.indexes, in...) + + // delete + for _, ind := range out { + i.removeSingle(ind) + } +} + +// createChildSearchRequest creates a separate +// request from the original +// For now, avoid data race on req structure. +// TODO disable highlight/field load on child +// requests, and add code to do this only on +// the actual final results. +// Perhaps that part needs to be optional, +// could be slower in remote usages. +func createChildSearchRequest(req *SearchRequest, preSearchData map[string]interface{}) *SearchRequest { + return copySearchRequest(req, preSearchData) +} + +type asyncSearchResult struct { + Name string + Result *SearchResult + Err error +} + +// preSearchFlags is a struct to hold flags indicating why preSearch is required +type preSearchFlags struct { + knn bool + synonyms bool + bm25 bool // needs presearch for this too +} + +func isBM25Enabled(m mapping.IndexMapping) bool { + var rv bool + if m, ok := m.(*mapping.IndexMappingImpl); ok { + rv = m.ScoringModel == index.BM25Scoring + } + return rv +} + +// preSearchRequired checks if preSearch is required and returns the presearch flags struct +// indicating which preSearch is required +func preSearchRequired(ctx context.Context, req *SearchRequest, m mapping.IndexMapping) (*preSearchFlags, error) { + // Check for KNN query + knn := requestHasKNN(req) + var synonyms bool + if !isMatchNoneQuery(req.Query) { + // Check if synonyms are defined in the mapping + if sm, ok := m.(mapping.SynonymMapping); ok && sm.SynonymCount() > 0 { + // check if any of the fields queried have a synonym source + // in the index mapping, to prevent unnecessary preSearch + fs, err := query.ExtractFields(req.Query, m, nil) + if err != nil { + return nil, err + } + for field := range fs { + if sm.SynonymSourceForPath(field) != "" { + synonyms = true + break + } + } + } + } + var bm25 bool + if !isMatchNoneQuery(req.Query) { + if ctx != nil { + if searchType := ctx.Value(search.SearchTypeKey); searchType != nil { + if searchType.(string) == search.GlobalScoring { + bm25 = isBM25Enabled(m) + } + } + } + } + + if knn || synonyms || bm25 { + return &preSearchFlags{ + knn: knn, + synonyms: synonyms, + bm25: bm25, + }, nil + } + return nil, nil +} + +func preSearch(ctx context.Context, req *SearchRequest, flags *preSearchFlags, indexes ...Index) (*SearchResult, error) { + // create a dummy request with a match none query + // since we only care about the preSearchData in PreSearch + dummyQuery := req.Query + if !flags.bm25 && !flags.synonyms { + // create a dummy request with a match none query + // since we only care about the preSearchData in PreSearch + dummyQuery = query.NewMatchNoneQuery() + } + dummyRequest := &SearchRequest{ + Query: dummyQuery, + } + newCtx := context.WithValue(ctx, search.PreSearchKey, true) + if flags.knn { + addKnnToDummyRequest(dummyRequest, req) + } + return preSearchDataSearch(newCtx, dummyRequest, flags, indexes...) +} + +// if the request is satisfied by just the preSearch result, +// finalize the result and return it directly without +// performing multi search +func finalizeSearchResult(ctx context.Context, req *SearchRequest, preSearchResult *SearchResult, rescorer *rescorer) *SearchResult { + if preSearchResult == nil { + return nil + } + + // global values across all hits irrespective of pagination settings + preSearchResult.Total = uint64(preSearchResult.Hits.Len()) + maxScore := float64(0) + for i, hit := range preSearchResult.Hits { + // since we are now using the preSearch result as the final result + // we can discard the indexNames from the hits as they are no longer + // relevant. + hit.IndexNames = nil + if hit.Score > maxScore { + maxScore = hit.Score + } + hit.HitNumber = uint64(i) + } + preSearchResult.MaxScore = maxScore + // now apply pagination settings + var reverseQueryExecution bool + if req.SearchBefore != nil { + reverseQueryExecution = true + req.Sort.Reverse() + req.SearchAfter = req.SearchBefore + } + if req.SearchAfter != nil { + preSearchResult.Hits = collector.FilterHitsBySearchAfter(preSearchResult.Hits, req.Sort, req.SearchAfter) + } + + if rescorer != nil { + // rescore takes ftsHits and knnHits as first and second argument respectively + // since this is pure knn, set ftsHits to nil. preSearchResult.Hits contains knn results + preSearchResult.Hits, preSearchResult.Total, preSearchResult.MaxScore = rescorer.rescore(nil, preSearchResult.Hits) + rescorer.restoreSearchRequest() + } + + preSearchResult.Hits = hitsInCurrentPage(req, preSearchResult.Hits) + + if reverseQueryExecution { + // reverse the sort back to the original + req.Sort.Reverse() + // resort using the original order + mhs := newSearchHitSorter(req.Sort, preSearchResult.Hits) + req.SortFunc()(mhs) + req.SearchAfter = nil + } + + if req.Explain { + preSearchResult.Request = req + } + return preSearchResult +} + +func requestSatisfiedByPreSearch(req *SearchRequest, flags *preSearchFlags) bool { + if flags == nil { + return false + } + // if the synonyms presearch flag is set the request can never be satisfied by + // the preSearch result as synonyms are not part of the preSearch result + if flags.synonyms { + return false + } + if flags.knn && isKNNrequestSatisfiedByPreSearch(req) { + return true + } + return false +} + +func constructSynonymPreSearchData(rv map[string]map[string]interface{}, sr *SearchResult, indexes []Index) map[string]map[string]interface{} { + for _, index := range indexes { + rv[index.Name()][search.SynonymPreSearchDataKey] = sr.SynonymResult + } + return rv +} + +func constructBM25PreSearchData(rv map[string]map[string]interface{}, sr *SearchResult, indexes []Index) map[string]map[string]interface{} { + bmStats := sr.BM25Stats + if bmStats != nil { + for _, index := range indexes { + rv[index.Name()][search.BM25PreSearchDataKey] = &search.BM25Stats{ + DocCount: bmStats.DocCount, + FieldCardinality: bmStats.FieldCardinality, + } + } + } + return rv +} + +func constructPreSearchData(req *SearchRequest, flags *preSearchFlags, + preSearchResult *SearchResult, indexes []Index, +) (map[string]map[string]interface{}, error) { + if flags == nil || preSearchResult == nil { + return nil, fmt.Errorf("invalid input, flags: %v, preSearchResult: %v", flags, preSearchResult) + } + mergedOut := make(map[string]map[string]interface{}, len(indexes)) + for _, index := range indexes { + mergedOut[index.Name()] = make(map[string]interface{}) + } + var err error + if flags.knn { + mergedOut, err = constructKnnPreSearchData(mergedOut, preSearchResult, indexes) + if err != nil { + return nil, err + } + } + if flags.synonyms { + mergedOut = constructSynonymPreSearchData(mergedOut, preSearchResult, indexes) + } + if flags.bm25 { + mergedOut = constructBM25PreSearchData(mergedOut, preSearchResult, indexes) + } + return mergedOut, nil +} + +// Constructs the presearch data if required during the search phase. +// Also if we need to store knn hits at alias. +// If we need to store knn hits at alias: returns all the knn hits +// If we should send it to leaf indexes: includes in presearch data +func constructPreSearchDataAndFusionKnnHits(req *SearchRequest, flags *preSearchFlags, + preSearchResult *SearchResult, rescorer *rescorer, indexes []Index, +) (map[string]map[string]interface{}, search.DocumentMatchCollection, error) { + var fusionknnhits search.DocumentMatchCollection + + // Checks if we need to send the KNN hits to the indexes in the + // search phase. If there is score fusion enabled, we do not + // send the KNN hits to the indexes. + if rescorer != nil && flags.knn { + fusionknnhits = preSearchResult.Hits + preSearchResult.Hits = nil + } + + preSearchData, err := constructPreSearchData(req, flags, preSearchResult, indexes) + if err != nil { + return nil, nil, err + } + + return preSearchData, fusionknnhits, nil +} + +func preSearchDataSearch(ctx context.Context, req *SearchRequest, flags *preSearchFlags, indexes ...Index) (*SearchResult, error) { + asyncResults := make(chan *asyncSearchResult, len(indexes)) + // run search on each index in separate go routine + var waitGroup sync.WaitGroup + searchChildIndex := func(in Index, childReq *SearchRequest) { + rv := asyncSearchResult{Name: in.Name()} + rv.Result, rv.Err = in.SearchInContext(ctx, childReq) + asyncResults <- &rv + waitGroup.Done() + } + waitGroup.Add(len(indexes)) + for _, in := range indexes { + go searchChildIndex(in, createChildSearchRequest(req, nil)) + } + // on another go routine, close after finished + go func() { + waitGroup.Wait() + close(asyncResults) + }() + // the final search result to be returned after combining the preSearch results + var sr *SearchResult + // the preSearch result processor + var prp preSearchResultProcessor + // error map + indexErrors := make(map[string]error) + for asr := range asyncResults { + if asr.Err == nil { + // a valid preSearch result + if prp == nil { + // first valid preSearch result + // create a new preSearch result processor + prp = createPreSearchResultProcessor(req, flags) + } + prp.add(asr.Result, asr.Name) + if sr == nil { + // first result + sr = &SearchResult{ + Status: asr.Result.Status, + Cost: asr.Result.Cost, + } + } else { + // merge with previous + sr.Status.Merge(asr.Result.Status) + sr.Cost += asr.Result.Cost + } + } else { + indexErrors[asr.Name] = asr.Err + } + } + // handle case where no results were successful + if sr == nil { + sr = &SearchResult{ + Status: &SearchStatus{ + Errors: make(map[string]error), + }, + } + } + // in preSearch, partial results are not allowed as it can lead to + // the real search giving incorrect results, and hence the search + // result is not populated with any of the processed data from + // the preSearch result processor if there are any errors + // or the preSearch result status has any failures + if len(indexErrors) > 0 || sr.Status.Failed > 0 { + if sr.Status.Errors == nil { + sr.Status.Errors = make(map[string]error) + } + for indexName, indexErr := range indexErrors { + sr.Status.Errors[indexName] = indexErr + sr.Status.Total++ + } + // At this point, all errors have been recorded—either from the preSearch phase + // (via status.Merge) or from individual index search failures (indexErrors). + // Since partial results are not allowed, mark the entire request as failed. + sr.Status.Successful = 0 + sr.Status.Failed = sr.Status.Total + } else { + prp.finalize(sr) + } + return sr, nil +} + +// redistributePreSearchData redistributes the preSearchData sent in the search request to an index alias +// which would happen in the case of an alias tree and depending on the level of the tree, the preSearchData +// needs to be redistributed to the indexes at that level +func redistributePreSearchData(req *SearchRequest, indexes []Index) (map[string]map[string]interface{}, error) { + rv := make(map[string]map[string]interface{}, len(indexes)) + for _, index := range indexes { + rv[index.Name()] = make(map[string]interface{}) + } + if knnHits, ok := req.PreSearchData[search.KnnPreSearchDataKey].([]*search.DocumentMatch); ok { + // the preSearchData for KNN is a list of DocumentMatch objects + // that need to be redistributed to the right index. + // This is used only in the case of an alias tree, where the indexes + // are at the leaves of the tree, and the master alias is at the root. + // At each level of the tree, the preSearchData needs to be redistributed + // to the indexes/aliases at that level. Because the preSearchData is + // specific to each final index at the leaf. + segregatedKnnHits, err := validateAndDistributeKNNHits(knnHits, indexes) + if err != nil { + return nil, err + } + for _, index := range indexes { + rv[index.Name()][search.KnnPreSearchDataKey] = segregatedKnnHits[index.Name()] + } + } + if fts, ok := req.PreSearchData[search.SynonymPreSearchDataKey].(search.FieldTermSynonymMap); ok { + for _, index := range indexes { + rv[index.Name()][search.SynonymPreSearchDataKey] = fts + } + } + + if bm25Data, ok := req.PreSearchData[search.BM25PreSearchDataKey].(*search.BM25Stats); ok { + for _, index := range indexes { + rv[index.Name()][search.BM25PreSearchDataKey] = bm25Data + } + } + return rv, nil +} + +// finalizePreSearchResult finalizes the preSearch result by applying the finalization steps +// specific to the preSearch flags +func finalizePreSearchResult(req *SearchRequest, flags *preSearchFlags, preSearchResult *SearchResult) { + // if flags is nil then return + if flags == nil { + return + } + if flags.knn { + preSearchResult.Hits = finalizeKNNResults(req, preSearchResult.Hits) + } +} + +// hitsInCurrentPage returns the hits in the current page +// using the From and Size parameters in the request +func hitsInCurrentPage(req *SearchRequest, hits []*search.DocumentMatch) []*search.DocumentMatch { + sortFunc := req.SortFunc() + // sort all hits with the requested order + if len(req.Sort) > 0 { + sorter := newSearchHitSorter(req.Sort, hits) + sortFunc(sorter) + } + // now skip over the correct From + if req.From > 0 && len(hits) > req.From { + hits = hits[req.From:] + } else if req.From > 0 { + hits = search.DocumentMatchCollection{} + } + // now trim to the correct size + if req.Size > 0 && len(hits) > req.Size { + hits = hits[0:req.Size] + } + return hits +} + +// Extra parameters for MultiSearch +type multiSearchParams struct { + preSearchData map[string]map[string]interface{} + rescorer *rescorer + fusionKnnHits search.DocumentMatchCollection +} + +// MultiSearch executes a SearchRequest across multiple Index objects, +// then merges the results. The indexes must honor any ctx deadline. +func MultiSearch(ctx context.Context, req *SearchRequest, params *multiSearchParams, indexes ...Index) (*SearchResult, error) { + searchStart := time.Now() + asyncResults := make(chan *asyncSearchResult, len(indexes)) + + var preSearchData map[string]map[string]interface{} + var rescorer *rescorer + var fusionKnnHits search.DocumentMatchCollection + if params != nil { + preSearchData = params.preSearchData + rescorer = params.rescorer + fusionKnnHits = params.fusionKnnHits + } + + var reverseQueryExecution bool + if req.SearchBefore != nil { + reverseQueryExecution = true + req.Sort.Reverse() + req.SearchAfter = req.SearchBefore + req.SearchBefore = nil + } + + // run search on each index in separate go routine + var waitGroup sync.WaitGroup + + searchChildIndex := func(in Index, childReq *SearchRequest) { + rv := asyncSearchResult{Name: in.Name()} + rv.Result, rv.Err = in.SearchInContext(ctx, childReq) + asyncResults <- &rv + waitGroup.Done() + } + + waitGroup.Add(len(indexes)) + for _, in := range indexes { + var payload map[string]interface{} + if preSearchData != nil { + payload = preSearchData[in.Name()] + } + go searchChildIndex(in, createChildSearchRequest(req, payload)) + } + + // on another go routine, close after finished + go func() { + waitGroup.Wait() + close(asyncResults) + }() + + var sr *SearchResult + indexErrors := make(map[string]error) + + for asr := range asyncResults { + if asr.Err == nil { + if sr == nil { + // first result + sr = asr.Result + } else { + // merge with previous + sr.Merge(asr.Result) + } + } else { + indexErrors[asr.Name] = asr.Err + } + } + + // merge just concatenated all the hits + // now lets clean it up + + // handle case where no results were successful + if sr == nil { + sr = &SearchResult{ + Status: &SearchStatus{ + Errors: make(map[string]error), + }, + } + } + + if rescorer != nil { + sr.Hits, sr.Total, sr.MaxScore = rescorer.rescore(sr.Hits, fusionKnnHits) + rescorer.restoreSearchRequest() + } + + sr.Hits = hitsInCurrentPage(req, sr.Hits) + + // fix up facets + for name, fr := range req.Facets { + sr.Facets.Fixup(name, fr.Size) + } + + if reverseQueryExecution { + // reverse the sort back to the original + req.Sort.Reverse() + // resort using the original order + mhs := newSearchHitSorter(req.Sort, sr.Hits) + req.SortFunc()(mhs) + // reset request + req.SearchBefore = req.SearchAfter + req.SearchAfter = nil + } + + // fix up original request + if req.Explain { + sr.Request = req + } + searchDuration := time.Since(searchStart) + sr.Took = searchDuration + + // fix up errors + if len(indexErrors) > 0 { + if sr.Status.Errors == nil { + sr.Status.Errors = make(map[string]error) + } + for indexName, indexErr := range indexErrors { + sr.Status.Errors[indexName] = indexErr + sr.Status.Total++ + sr.Status.Failed++ + } + } + + return sr, nil +} + +func (i *indexAliasImpl) NewBatch() *Batch { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return nil + } + + err := i.isAliasToSingleIndex() + if err != nil { + return nil + } + + return i.indexes[0].NewBatch() +} + +func (i *indexAliasImpl) Name() string { + return i.name +} + +func (i *indexAliasImpl) SetName(name string) { + i.name = name +} + +type indexAliasImplFieldDict struct { + index *indexAliasImpl + fieldDict index.FieldDict +} + +func (f *indexAliasImplFieldDict) BytesRead() uint64 { + return f.fieldDict.BytesRead() +} + +func (f *indexAliasImplFieldDict) Next() (*index.DictEntry, error) { + return f.fieldDict.Next() +} + +func (f *indexAliasImplFieldDict) Close() error { + defer f.index.mutex.RUnlock() + return f.fieldDict.Close() +} + +func (f *indexAliasImplFieldDict) Cardinality() int { + return f.fieldDict.Cardinality() +} + +// ----------------------------------------------------------------------------- + +func (i *indexAliasImpl) TermFrequencies(field string, limit int, descending bool) ( + []index.TermFreq, error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return nil, ErrorIndexClosed + } + + if len(i.indexes) < 1 { + return nil, ErrorAliasEmpty + } + + // short circuit the simple case + if len(i.indexes) == 1 { + if idx, ok := i.indexes[0].(InsightsIndex); ok { + return idx.TermFrequencies(field, limit, descending) + } + return nil, nil + } + + // run search on each index in separate go routine + var waitGroup sync.WaitGroup + asyncResults := make(chan []index.TermFreq, len(i.indexes)) + + searchChildIndex := func(in Index, field string, limit int, descending bool) { + var rv []index.TermFreq + if idx, ok := in.(InsightsIndex); ok { + // over sample for higher accuracy + rv, _ = idx.TermFrequencies(field, limit*5, descending) + } + asyncResults <- rv + waitGroup.Done() + } + + waitGroup.Add(len(i.indexes)) + for _, in := range i.indexes { + go searchChildIndex(in, field, limit, descending) + } + + // on another go routine, close after finished + go func() { + waitGroup.Wait() + close(asyncResults) + }() + + rvTermFreqsMap := make(map[string]uint64) + for asr := range asyncResults { + for _, entry := range asr { + rvTermFreqsMap[entry.Term] += entry.Frequency + } + } + + rvTermFreqs := make([]index.TermFreq, 0, len(rvTermFreqsMap)) + for term, freq := range rvTermFreqsMap { + rvTermFreqs = append(rvTermFreqs, index.TermFreq{ + Term: term, + Frequency: freq, + }) + } + + sort.Slice(rvTermFreqs, func(i, j int) bool { + if rvTermFreqs[i].Frequency == rvTermFreqs[j].Frequency { + // If frequencies are equal, sort by term lexicographically + return rvTermFreqs[i].Term < rvTermFreqs[j].Term + } + if descending { + return rvTermFreqs[i].Frequency > rvTermFreqs[j].Frequency + } + return rvTermFreqs[i].Frequency < rvTermFreqs[j].Frequency + }) + + if limit > len(rvTermFreqs) { + limit = len(rvTermFreqs) + } + + return rvTermFreqs[:limit], nil +} + +func (i *indexAliasImpl) CentroidCardinalities(field string, limit int, descending bool) ( + []index.CentroidCardinality, error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return nil, ErrorIndexClosed + } + + if len(i.indexes) < 1 { + return nil, ErrorAliasEmpty + } + + // short circuit the simple case + if len(i.indexes) == 1 { + if idx, ok := i.indexes[0].(InsightsIndex); ok { + return idx.CentroidCardinalities(field, limit, descending) + } + return nil, nil + } + + // run search on each index in separate go routine + var waitGroup sync.WaitGroup + asyncResults := make(chan []index.CentroidCardinality, len(i.indexes)) + + searchChildIndex := func(in Index, field string, limit int, descending bool) { + var rv []index.CentroidCardinality + if idx, ok := in.(InsightsIndex); ok { + rv, _ = idx.CentroidCardinalities(field, limit, descending) + } + asyncResults <- rv + waitGroup.Done() + } + + waitGroup.Add(len(i.indexes)) + for _, in := range i.indexes { + go searchChildIndex(in, field, limit, descending) + } + + // on another go routine, close after finished + go func() { + waitGroup.Wait() + close(asyncResults) + }() + + rvCentroidCardinalities := make([]index.CentroidCardinality, 0, limit*len(i.indexes)) + for asr := range asyncResults { + rvCentroidCardinalities = append(rvCentroidCardinalities, asr...) + } + + sort.Slice(rvCentroidCardinalities, func(i, j int) bool { + if descending { + return rvCentroidCardinalities[i].Cardinality > rvCentroidCardinalities[j].Cardinality + } else { + return rvCentroidCardinalities[i].Cardinality < rvCentroidCardinalities[j].Cardinality + } + }) + + if limit > len(rvCentroidCardinalities) { + limit = len(rvCentroidCardinalities) + } + + return rvCentroidCardinalities[:limit], nil +} diff --git a/index_alias_impl_test.go b/index_alias_impl_test.go new file mode 100644 index 0000000..0480139 --- /dev/null +++ b/index_alias_impl_test.go @@ -0,0 +1,1355 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "context" + "fmt" + "reflect" + "testing" + "time" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func TestIndexAliasSingle(t *testing.T) { + expectedError := fmt.Errorf("expected") + ei1 := &stubIndex{ + err: expectedError, + } + + alias := NewIndexAlias(ei1) + + err := alias.Index("a", "a") + if err != expectedError { + t.Errorf("expected %v, got %v", expectedError, err) + } + + err = alias.Delete("a") + if err != expectedError { + t.Errorf("expected %v, got %v", expectedError, err) + } + + batch := alias.NewBatch() + err = alias.Batch(batch) + if err != expectedError { + t.Errorf("expected %v, got %v", expectedError, err) + } + + _, err = alias.Document("a") + if err != expectedError { + t.Errorf("expected %v, got %v", expectedError, err) + } + + _, err = alias.Fields() + if err != expectedError { + t.Errorf("expected %v, got %v", expectedError, err) + } + + _, err = alias.GetInternal([]byte("a")) + if err != expectedError { + t.Errorf("expected %v, got %v", expectedError, err) + } + + err = alias.SetInternal([]byte("a"), []byte("a")) + if err != expectedError { + t.Errorf("expected %v, got %v", expectedError, err) + } + + err = alias.DeleteInternal([]byte("a")) + if err != expectedError { + t.Errorf("expected %v, got %v", expectedError, err) + } + + mapping := alias.Mapping() + if mapping != nil { + t.Errorf("expected nil, got %v", mapping) + } + + indexStat := alias.Stats() + if indexStat != nil { + t.Errorf("expected nil, got %v", indexStat) + } + + // now a few things that should work + sr := NewSearchRequest(NewTermQuery("test")) + _, err = alias.Search(sr) + if err != expectedError { + t.Errorf("expected %v, got %v", expectedError, err) + } + + count, err := alias.DocCount() + if err != nil { + t.Errorf("expected no error, got %v", err) + } + if count != 0 { + t.Errorf("expected count 0, got %d", count) + } + + // now change the def using add/remove + expectedError2 := fmt.Errorf("expected2") + ei2 := &stubIndex{ + err: expectedError2, + } + + alias.Add(ei2) + alias.Remove(ei1) + + err = alias.Index("a", "a") + if err != expectedError2 { + t.Errorf("expected %v, got %v", expectedError2, err) + } + + err = alias.Delete("a") + if err != expectedError2 { + t.Errorf("expected %v, got %v", expectedError2, err) + } + + err = alias.Batch(batch) + if err != expectedError2 { + t.Errorf("expected %v, got %v", expectedError2, err) + } + + _, err = alias.Document("a") + if err != expectedError2 { + t.Errorf("expected %v, got %v", expectedError2, err) + } + + _, err = alias.Fields() + if err != expectedError2 { + t.Errorf("expected %v, got %v", expectedError2, err) + } + + _, err = alias.GetInternal([]byte("a")) + if err != expectedError2 { + t.Errorf("expected %v, got %v", expectedError2, err) + } + + err = alias.SetInternal([]byte("a"), []byte("a")) + if err != expectedError2 { + t.Errorf("expected %v, got %v", expectedError2, err) + } + + err = alias.DeleteInternal([]byte("a")) + if err != expectedError2 { + t.Errorf("expected %v, got %v", expectedError2, err) + } + + mapping = alias.Mapping() + if mapping != nil { + t.Errorf("expected nil, got %v", mapping) + } + + indexStat = alias.Stats() + if indexStat != nil { + t.Errorf("expected nil, got %v", indexStat) + } + + // now a few things that should work + _, err = alias.Search(sr) + if err != expectedError2 { + t.Errorf("expected %v, got %v", expectedError2, err) + } + + count, err = alias.DocCount() + if err != nil { + t.Errorf("expected no error, got %v", err) + } + if count != 0 { + t.Errorf("expected count 0, got %d", count) + } + + // now change the def using swap + expectedError3 := fmt.Errorf("expected3") + ei3 := &stubIndex{ + err: expectedError3, + } + + alias.Swap([]Index{ei3}, []Index{ei2}) + + err = alias.Index("a", "a") + if err != expectedError3 { + t.Errorf("expected %v, got %v", expectedError3, err) + } + + err = alias.Delete("a") + if err != expectedError3 { + t.Errorf("expected %v, got %v", expectedError3, err) + } + + err = alias.Batch(batch) + if err != expectedError3 { + t.Errorf("expected %v, got %v", expectedError3, err) + } + + _, err = alias.Document("a") + if err != expectedError3 { + t.Errorf("expected %v, got %v", expectedError3, err) + } + + _, err = alias.Fields() + if err != expectedError3 { + t.Errorf("expected %v, got %v", expectedError3, err) + } + + _, err = alias.GetInternal([]byte("a")) + if err != expectedError3 { + t.Errorf("expected %v, got %v", expectedError3, err) + } + + err = alias.SetInternal([]byte("a"), []byte("a")) + if err != expectedError3 { + t.Errorf("expected %v, got %v", expectedError3, err) + } + + err = alias.DeleteInternal([]byte("a")) + if err != expectedError3 { + t.Errorf("expected %v, got %v", expectedError3, err) + } + + mapping = alias.Mapping() + if mapping != nil { + t.Errorf("expected nil, got %v", mapping) + } + + indexStat = alias.Stats() + if indexStat != nil { + t.Errorf("expected nil, got %v", indexStat) + } + + // now a few things that should work + _, err = alias.Search(sr) + if err != expectedError3 { + t.Errorf("expected %v, got %v", expectedError3, err) + } + + count, err = alias.DocCount() + if err != nil { + t.Errorf("expected no error, got %v", err) + } + if count != 0 { + t.Errorf("expected count 0, got %d", count) + } +} + +func TestIndexAliasClosed(t *testing.T) { + alias := NewIndexAlias() + err := alias.Close() + if err != nil { + t.Fatal(err) + } + + err = alias.Index("a", "a") + if err != ErrorIndexClosed { + t.Errorf("expected %v, got %v", ErrorIndexClosed, err) + } + + err = alias.Delete("a") + if err != ErrorIndexClosed { + t.Errorf("expected %v, got %v", ErrorIndexClosed, err) + } + + batch := alias.NewBatch() + err = alias.Batch(batch) + if err != ErrorIndexClosed { + t.Errorf("expected %v, got %v", ErrorIndexClosed, err) + } + + _, err = alias.Document("a") + if err != ErrorIndexClosed { + t.Errorf("expected %v, got %v", ErrorIndexClosed, err) + } + + _, err = alias.Fields() + if err != ErrorIndexClosed { + t.Errorf("expected %v, got %v", ErrorIndexClosed, err) + } + + _, err = alias.GetInternal([]byte("a")) + if err != ErrorIndexClosed { + t.Errorf("expected %v, got %v", ErrorIndexClosed, err) + } + + err = alias.SetInternal([]byte("a"), []byte("a")) + if err != ErrorIndexClosed { + t.Errorf("expected %v, got %v", ErrorIndexClosed, err) + } + + err = alias.DeleteInternal([]byte("a")) + if err != ErrorIndexClosed { + t.Errorf("expected %v, got %v", ErrorIndexClosed, err) + } + + mapping := alias.Mapping() + if mapping != nil { + t.Errorf("expected nil, got %v", mapping) + } + + indexStat := alias.Stats() + if indexStat != nil { + t.Errorf("expected nil, got %v", indexStat) + } + + // now a few things that should work + sr := NewSearchRequest(NewTermQuery("test")) + _, err = alias.Search(sr) + if err != ErrorIndexClosed { + t.Errorf("expected %v, got %v", ErrorIndexClosed, err) + } + + _, err = alias.DocCount() + if err != ErrorIndexClosed { + t.Errorf("expected %v, got %v", ErrorIndexClosed, err) + } +} + +func TestIndexAliasEmpty(t *testing.T) { + alias := NewIndexAlias() + + err := alias.Index("a", "a") + if err != ErrorAliasEmpty { + t.Errorf("expected %v, got %v", ErrorAliasEmpty, err) + } + + err = alias.Delete("a") + if err != ErrorAliasEmpty { + t.Errorf("expected %v, got %v", ErrorAliasEmpty, err) + } + + batch := alias.NewBatch() + err = alias.Batch(batch) + if err != ErrorAliasEmpty { + t.Errorf("expected %v, got %v", ErrorAliasEmpty, err) + } + + _, err = alias.Document("a") + if err != ErrorAliasEmpty { + t.Errorf("expected %v, got %v", ErrorAliasEmpty, err) + } + + _, err = alias.Fields() + if err != ErrorAliasEmpty { + t.Errorf("expected %v, got %v", ErrorAliasEmpty, err) + } + + _, err = alias.GetInternal([]byte("a")) + if err != ErrorAliasEmpty { + t.Errorf("expected %v, got %v", ErrorAliasEmpty, err) + } + + err = alias.SetInternal([]byte("a"), []byte("a")) + if err != ErrorAliasEmpty { + t.Errorf("expected %v, got %v", ErrorAliasEmpty, err) + } + + err = alias.DeleteInternal([]byte("a")) + if err != ErrorAliasEmpty { + t.Errorf("expected %v, got %v", ErrorAliasEmpty, err) + } + + mapping := alias.Mapping() + if mapping != nil { + t.Errorf("expected nil, got %v", mapping) + } + + indexStat := alias.Stats() + if indexStat != nil { + t.Errorf("expected nil, got %v", indexStat) + } + + // now a few things that should work + sr := NewSearchRequest(NewTermQuery("test")) + _, err = alias.Search(sr) + if err != ErrorAliasEmpty { + t.Errorf("expected %v, got %v", ErrorAliasEmpty, err) + } + + count, err := alias.DocCount() + if err != nil { + t.Errorf("error getting alias doc count: %v", err) + } + if count != 0 { + t.Errorf("expected %d, got %d", 0, count) + } +} + +func TestIndexAliasMulti(t *testing.T) { + score1, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(1.0), 0) + score2, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(2.0), 0) + ei1Count := uint64(7) + ei1 := &stubIndex{ + err: nil, + docCountResult: &ei1Count, + searchResult: &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + Errors: make(map[string]error), + }, + Total: 1, + Hits: search.DocumentMatchCollection{ + { + ID: "a", + Score: 1.0, + Sort: []string{string(score1)}, + }, + }, + MaxScore: 1.0, + }, + } + ei2Count := uint64(8) + ei2 := &stubIndex{ + err: nil, + docCountResult: &ei2Count, + searchResult: &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + Errors: make(map[string]error), + }, + Total: 1, + Hits: search.DocumentMatchCollection{ + { + ID: "b", + Score: 2.0, + Sort: []string{string(score2)}, + }, + }, + MaxScore: 2.0, + }, + } + + alias := NewIndexAlias(ei1, ei2) + + err := alias.Index("a", "a") + if err != ErrorAliasMulti { + t.Errorf("expected %v, got %v", ErrorAliasMulti, err) + } + + err = alias.Delete("a") + if err != ErrorAliasMulti { + t.Errorf("expected %v, got %v", ErrorAliasMulti, err) + } + + batch := alias.NewBatch() + err = alias.Batch(batch) + if err != ErrorAliasMulti { + t.Errorf("expected %v, got %v", ErrorAliasMulti, err) + } + + _, err = alias.Document("a") + if err != ErrorAliasMulti { + t.Errorf("expected %v, got %v", ErrorAliasMulti, err) + } + + _, err = alias.Fields() + if err != ErrorAliasMulti { + t.Errorf("expected %v, got %v", ErrorAliasMulti, err) + } + + _, err = alias.GetInternal([]byte("a")) + if err != ErrorAliasMulti { + t.Errorf("expected %v, got %v", ErrorAliasMulti, err) + } + + err = alias.SetInternal([]byte("a"), []byte("a")) + if err != ErrorAliasMulti { + t.Errorf("expected %v, got %v", ErrorAliasMulti, err) + } + + err = alias.DeleteInternal([]byte("a")) + if err != ErrorAliasMulti { + t.Errorf("expected %v, got %v", ErrorAliasMulti, err) + } + + mapping := alias.Mapping() + if mapping != nil { + t.Errorf("expected nil, got %v", mapping) + } + + indexStat := alias.Stats() + if indexStat != nil { + t.Errorf("expected nil, got %v", indexStat) + } + + // now a few things that should work + sr := NewSearchRequest(NewTermQuery("test")) + expected := &SearchResult{ + Status: &SearchStatus{ + Total: 2, + Successful: 2, + Errors: make(map[string]error), + }, + Total: 2, + Hits: search.DocumentMatchCollection{ + { + ID: "b", + Score: 2.0, + Sort: []string{string(score2)}, + }, + { + ID: "a", + Score: 1.0, + Sort: []string{string(score1)}, + }, + }, + MaxScore: 2.0, + } + results, err := alias.Search(sr) + if err != nil { + t.Error(err) + } + // cheat and ensure that Took field matches since it involves time + expected.Took = results.Took + if !reflect.DeepEqual(results, expected) { + t.Errorf("expected %#v, got %#v", expected, results) + } + + count, err := alias.DocCount() + if err != nil { + t.Errorf("error getting alias doc count: %v", err) + } + if count != (*ei1.docCountResult + *ei2.docCountResult) { + t.Errorf("expected %d, got %d", (*ei1.docCountResult + *ei2.docCountResult), count) + } +} + +// TestMultiSearchNoError +func TestMultiSearchNoError(t *testing.T) { + score1, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(1.0), 0) + score2, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(2.0), 0) + ei1 := &stubIndex{err: nil, searchResult: &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + Errors: make(map[string]error), + }, + Total: 1, + Hits: search.DocumentMatchCollection{ + { + Index: "1", + ID: "a", + Score: 1.0, + Sort: []string{string(score1)}, + }, + }, + MaxScore: 1.0, + }} + ei2 := &stubIndex{err: nil, searchResult: &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + Errors: make(map[string]error), + }, + Total: 1, + Hits: search.DocumentMatchCollection{ + { + Index: "2", + ID: "b", + Score: 2.0, + Sort: []string{string(score2)}, + }, + }, + MaxScore: 2.0, + }} + + sr := NewSearchRequest(NewTermQuery("test")) + expected := &SearchResult{ + Status: &SearchStatus{ + Total: 2, + Successful: 2, + Errors: make(map[string]error), + }, + Total: 2, + Hits: search.DocumentMatchCollection{ + { + Index: "2", + ID: "b", + Score: 2.0, + Sort: []string{string(score2)}, + }, + { + Index: "1", + ID: "a", + Score: 1.0, + Sort: []string{string(score1)}, + }, + }, + MaxScore: 2.0, + } + + results, err := MultiSearch(context.Background(), sr, nil, ei1, ei2) + if err != nil { + t.Error(err) + } + // cheat and ensure that Took field matches since it involves time + expected.Took = results.Took + if !reflect.DeepEqual(results, expected) { + t.Errorf("expected %#v, got %#v", expected, results) + } +} + +// TestMultiSearchSomeError +func TestMultiSearchSomeError(t *testing.T) { + ei1 := &stubIndex{name: "ei1", err: nil, searchResult: &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + Errors: make(map[string]error), + }, + Total: 1, + Hits: search.DocumentMatchCollection{ + { + ID: "a", + Score: 1.0, + }, + }, + Took: 1 * time.Second, + MaxScore: 1.0, + }} + ei2 := &stubIndex{name: "ei2", err: fmt.Errorf("deliberate error")} + sr := NewSearchRequest(NewTermQuery("test")) + res, err := MultiSearch(context.Background(), sr, nil, ei1, ei2) + if err != nil { + t.Errorf("expected no error, got %v", err) + } + if res.Status.Total != 2 { + t.Errorf("expected 2 indexes to be queried, got %d", res.Status.Total) + } + if res.Status.Failed != 1 { + t.Errorf("expected 1 index to fail, got %d", res.Status.Failed) + } + if res.Status.Successful != 1 { + t.Errorf("expected 1 index to be successful, got %d", res.Status.Successful) + } + if len(res.Status.Errors) != 1 { + t.Fatalf("expected 1 status error message, got %d", len(res.Status.Errors)) + } + if res.Status.Errors["ei2"].Error() != "deliberate error" { + t.Errorf("expected ei2 index error message 'deliberate error', got '%s'", res.Status.Errors["ei2"]) + } +} + +// TestMultiSearchAllError +// reproduces https://github.com/blevesearch/bleve/issues/126 +func TestMultiSearchAllError(t *testing.T) { + ei1 := &stubIndex{name: "ei1", err: fmt.Errorf("deliberate error")} + ei2 := &stubIndex{name: "ei2", err: fmt.Errorf("deliberate error")} + sr := NewSearchRequest(NewTermQuery("test")) + res, err := MultiSearch(context.Background(), sr, nil, ei1, ei2) + if err != nil { + t.Errorf("expected no error, got %v", err) + } + if res.Status.Total != 2 { + t.Errorf("expected 2 indexes to be queried, got %d", res.Status.Total) + } + if res.Status.Failed != 2 { + t.Errorf("expected 2 indexes to fail, got %d", res.Status.Failed) + } + if res.Status.Successful != 0 { + t.Errorf("expected 0 indexes to be successful, got %d", res.Status.Successful) + } + if len(res.Status.Errors) != 2 { + t.Fatalf("expected 2 status error messages, got %d", len(res.Status.Errors)) + } + if res.Status.Errors["ei1"].Error() != "deliberate error" { + t.Errorf("expected ei1 index error message 'deliberate error', got '%s'", res.Status.Errors["ei1"]) + } + if res.Status.Errors["ei2"].Error() != "deliberate error" { + t.Errorf("expected ei2 index error message 'deliberate error', got '%s'", res.Status.Errors["ei2"]) + } +} + +func TestMultiSearchSecondPage(t *testing.T) { + checkRequest := func(sr *SearchRequest) error { + if sr.From != 0 { + return fmt.Errorf("child request from should be 0") + } + if sr.Size != 20 { + return fmt.Errorf("child request size should be 20") + } + return nil + } + + ei1 := &stubIndex{ + searchResult: &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + Errors: make(map[string]error), + }, + }, + checkRequest: checkRequest, + } + ei2 := &stubIndex{ + searchResult: &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + Errors: make(map[string]error), + }, + }, + checkRequest: checkRequest, + } + sr := NewSearchRequestOptions(NewTermQuery("test"), 10, 10, false) + _, err := MultiSearch(context.Background(), sr, nil, ei1, ei2) + if err != nil { + t.Errorf("unexpected error %v", err) + } +} + +// TestMultiSearchTimeout tests simple timeout cases +// 1. all searches finish successfully before timeout +// 2. no searchers finish before the timeout +// 3. no searches finish before cancellation +func TestMultiSearchTimeout(t *testing.T) { + score1, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(1.0), 0) + score2, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(2.0), 0) + var ctx context.Context + ei1 := &stubIndex{ + name: "ei1", + checkRequest: func(req *SearchRequest) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(50 * time.Millisecond): + return nil + } + }, + err: nil, + searchResult: &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + Errors: make(map[string]error), + }, + Total: 1, + Hits: []*search.DocumentMatch{ + { + Index: "1", + ID: "a", + Score: 1.0, + Sort: []string{string(score1)}, + }, + }, + MaxScore: 1.0, + }, + } + ei2 := &stubIndex{ + name: "ei2", + checkRequest: func(req *SearchRequest) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(50 * time.Millisecond): + return nil + } + }, + err: nil, + searchResult: &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + Errors: make(map[string]error), + }, + Total: 1, + Hits: []*search.DocumentMatch{ + { + Index: "2", + ID: "b", + Score: 2.0, + Sort: []string{string(score2)}, + }, + }, + MaxScore: 2.0, + }, + } + + // first run with absurdly long time out, should succeed + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + query := NewTermQuery("test") + sr := NewSearchRequest(query) + res, err := MultiSearch(ctx, sr, nil, ei1, ei2) + if err != nil { + t.Errorf("expected no error, got %v", err) + } + if res.Status.Total != 2 { + t.Errorf("expected 2 total, got %d", res.Status.Failed) + } + if res.Status.Successful != 2 { + t.Errorf("expected 0 success, got %d", res.Status.Successful) + } + if res.Status.Failed != 0 { + t.Errorf("expected 2 failed, got %d", res.Status.Failed) + } + if len(res.Status.Errors) != 0 { + t.Errorf("expected 0 errors, got %v", res.Status.Errors) + } + + // now run a search again with an absurdly low timeout (should timeout) + ctx, cancel = context.WithTimeout(context.Background(), 1*time.Microsecond) + defer cancel() + res, err = MultiSearch(ctx, sr, nil, ei1, ei2) + if err != nil { + t.Errorf("expected no error, got %v", err) + } + if res.Status.Total != 2 { + t.Errorf("expected 2 failed, got %d", res.Status.Failed) + } + if res.Status.Successful != 0 { + t.Errorf("expected 0 success, got %d", res.Status.Successful) + } + if res.Status.Failed != 2 { + t.Errorf("expected 2 failed, got %d", res.Status.Failed) + } + if len(res.Status.Errors) != 2 { + t.Errorf("expected 2 errors, got %v", res.Status.Errors) + } else { + if res.Status.Errors["ei1"].Error() != context.DeadlineExceeded.Error() { + t.Errorf("expected err for 'ei1' to be '%s' got '%s'", context.DeadlineExceeded.Error(), res.Status.Errors["ei1"]) + } + if res.Status.Errors["ei2"].Error() != context.DeadlineExceeded.Error() { + t.Errorf("expected err for 'ei2' to be '%s' got '%s'", context.DeadlineExceeded.Error(), res.Status.Errors["ei2"]) + } + } + + // now run a search again with a normal timeout, but cancel it first + ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second) + cancel() + res, err = MultiSearch(ctx, sr, nil, ei1, ei2) + if err != nil { + t.Errorf("expected no error, got %v", err) + } + if res.Status.Total != 2 { + t.Errorf("expected 2 failed, got %d", res.Status.Failed) + } + if res.Status.Successful != 0 { + t.Errorf("expected 0 success, got %d", res.Status.Successful) + } + if res.Status.Failed != 2 { + t.Errorf("expected 2 failed, got %d", res.Status.Failed) + } + if len(res.Status.Errors) != 2 { + t.Errorf("expected 2 errors, got %v", res.Status.Errors) + } else { + if res.Status.Errors["ei1"].Error() != context.Canceled.Error() { + t.Errorf("expected err for 'ei1' to be '%s' got '%s'", context.Canceled.Error(), res.Status.Errors["ei1"]) + } + if res.Status.Errors["ei2"].Error() != context.Canceled.Error() { + t.Errorf("expected err for 'ei2' to be '%s' got '%s'", context.Canceled.Error(), res.Status.Errors["ei2"]) + } + } +} + +// TestMultiSearchTimeoutPartial tests the case where some indexes exceed +// the timeout, while others complete successfully +func TestMultiSearchTimeoutPartial(t *testing.T) { + score1, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(1.0), 0) + score2, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(2.0), 0) + score3, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(3.0), 0) + var ctx context.Context + ei1 := &stubIndex{ + name: "ei1", + err: nil, + searchResult: &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + Errors: make(map[string]error), + }, + Total: 1, + Hits: []*search.DocumentMatch{ + { + Index: "1", + ID: "a", + Score: 1.0, + Sort: []string{string(score1)}, + }, + }, + MaxScore: 1.0, + }, + } + ei2 := &stubIndex{ + name: "ei2", + err: nil, + searchResult: &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + Errors: make(map[string]error), + }, + Total: 1, + Hits: []*search.DocumentMatch{ + { + Index: "2", + ID: "b", + Score: 2.0, + Sort: []string{string(score2)}, + }, + }, + MaxScore: 2.0, + }, + } + + ei3 := &stubIndex{ + name: "ei3", + checkRequest: func(req *SearchRequest) error { + <-ctx.Done() + return ctx.Err() + }, + err: nil, + searchResult: &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + Errors: make(map[string]error), + }, + Total: 1, + Hits: []*search.DocumentMatch{ + { + Index: "3", + ID: "c", + Score: 3.0, + Sort: []string{string(score3)}, + }, + }, + MaxScore: 3.0, + }, + } + + // ei3 is set to take >50ms, so run search with timeout less than + // this, this should return partial results + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(context.Background(), 25*time.Millisecond) + defer cancel() + query := NewTermQuery("test") + sr := NewSearchRequest(query) + expected := &SearchResult{ + Status: &SearchStatus{ + Total: 3, + Successful: 2, + Failed: 1, + Errors: map[string]error{ + "ei3": context.DeadlineExceeded, + }, + }, + Total: 2, + Hits: search.DocumentMatchCollection{ + { + Index: "2", + ID: "b", + Score: 2.0, + Sort: []string{string(score2)}, + }, + { + Index: "1", + ID: "a", + Score: 1.0, + Sort: []string{string(score1)}, + }, + }, + MaxScore: 2.0, + } + + res, err := MultiSearch(ctx, sr, nil, ei1, ei2, ei3) + if err != nil { + t.Fatalf("expected no err, got %v", err) + } + expected.Took = res.Took + if !reflect.DeepEqual(res, expected) { + t.Errorf("expected %#v, got %#v", expected, res) + } +} + +func TestIndexAliasMultipleLayer(t *testing.T) { + score1, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(1.0), 0) + score2, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(2.0), 0) + score3, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(3.0), 0) + score4, _ := numeric.NewPrefixCodedInt64(numeric.Float64ToInt64(4.0), 0) + var ctx context.Context + ei1 := &stubIndex{ + name: "ei1", + err: nil, + searchResult: &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + Errors: make(map[string]error), + }, + Total: 1, + Hits: []*search.DocumentMatch{ + { + Index: "1", + ID: "a", + Score: 1.0, + Sort: []string{string(score1)}, + }, + }, + MaxScore: 1.0, + }, + } + ei2 := &stubIndex{ + name: "ei2", + checkRequest: func(req *SearchRequest) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(250 * time.Millisecond): + return nil + } + }, + err: nil, + searchResult: &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + Errors: make(map[string]error), + }, + Total: 1, + Hits: []*search.DocumentMatch{ + { + Index: "2", + ID: "b", + Score: 2.0, + Sort: []string{string(score2)}, + }, + }, + MaxScore: 2.0, + }, + } + + ei3 := &stubIndex{ + name: "ei3", + checkRequest: func(req *SearchRequest) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(250 * time.Millisecond): + return nil + } + }, + err: nil, + searchResult: &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + Errors: make(map[string]error), + }, + Total: 1, + Hits: []*search.DocumentMatch{ + { + Index: "3", + ID: "c", + Score: 3.0, + Sort: []string{string(score3)}, + }, + }, + MaxScore: 3.0, + }, + } + + ei4 := &stubIndex{ + name: "ei4", + err: nil, + searchResult: &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + Errors: make(map[string]error), + }, + Total: 1, + Hits: []*search.DocumentMatch{ + { + Index: "4", + ID: "d", + Score: 4.0, + Sort: []string{string(score4)}, + }, + }, + MaxScore: 4.0, + }, + } + + alias1 := NewIndexAlias(ei1, ei2) + alias2 := NewIndexAlias(ei3, ei4) + aliasTop := NewIndexAlias(alias1, alias2) + + // ei2 and ei3 have 50ms delay + // search across aliasTop should still get results from ei1 and ei4 + // total should still be 4 + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(context.Background(), 25*time.Millisecond) + defer cancel() + query := NewTermQuery("test") + sr := NewSearchRequest(query) + expected := &SearchResult{ + Status: &SearchStatus{ + Total: 4, + Successful: 2, + Failed: 2, + Errors: map[string]error{ + "ei2": context.DeadlineExceeded, + "ei3": context.DeadlineExceeded, + }, + }, + Total: 2, + Hits: search.DocumentMatchCollection{ + { + Index: "4", + ID: "d", + Score: 4.0, + Sort: []string{string(score4)}, + }, + { + Index: "1", + ID: "a", + Score: 1.0, + Sort: []string{string(score1)}, + }, + }, + MaxScore: 4.0, + } + + res, err := aliasTop.SearchInContext(ctx, sr) + if err != nil { + t.Fatalf("expected no err, got %v", err) + } + expected.Took = res.Took + if !reflect.DeepEqual(res, expected) { + t.Errorf("expected %#v, got %#v", expected, res) + } +} + +// TestMultiSearchCustomSort +func TestMultiSearchCustomSort(t *testing.T) { + ei1 := &stubIndex{err: nil, searchResult: &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + Errors: make(map[string]error), + }, + Total: 2, + Hits: search.DocumentMatchCollection{ + { + Index: "1", + ID: "a", + Score: 1.0, + Sort: []string{"albert"}, + }, + { + Index: "1", + ID: "b", + Score: 2.0, + Sort: []string{"crown"}, + }, + }, + MaxScore: 2.0, + }} + ei2 := &stubIndex{err: nil, searchResult: &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + Errors: make(map[string]error), + }, + Total: 2, + Hits: search.DocumentMatchCollection{ + { + Index: "2", + ID: "c", + Score: 2.5, + Sort: []string{"frank"}, + }, + { + Index: "2", + ID: "d", + Score: 3.0, + Sort: []string{"zombie"}, + }, + }, + MaxScore: 3.0, + }} + + sr := NewSearchRequest(NewTermQuery("test")) + sr.Explain = true + sr.SortBy([]string{"name"}) + expected := &SearchResult{ + Status: &SearchStatus{ + Total: 2, + Successful: 2, + Errors: make(map[string]error), + }, + Request: sr, + Total: 4, + Hits: search.DocumentMatchCollection{ + { + Index: "1", + ID: "a", + Score: 1.0, + Sort: []string{"albert"}, + }, + { + Index: "1", + ID: "b", + Score: 2.0, + Sort: []string{"crown"}, + }, + { + Index: "2", + ID: "c", + Score: 2.5, + Sort: []string{"frank"}, + }, + { + Index: "2", + ID: "d", + Score: 3.0, + Sort: []string{"zombie"}, + }, + }, + MaxScore: 3.0, + } + + results, err := MultiSearch(context.Background(), sr, nil, ei1, ei2) + if err != nil { + t.Error(err) + } + // cheat and ensure that Took field matches since it involves time + expected.Took = results.Took + if !reflect.DeepEqual(results, expected) { + t.Errorf("expected %v, got %v", expected, results) + } +} + +// stubIndex is an Index impl for which all operations +// return the configured error value, unless the +// corresponding operation result value has been +// set, in which case that is returned instead +type stubIndex struct { + name string + err error + searchResult *SearchResult + documentResult *document.Document + docCountResult *uint64 + checkRequest func(*SearchRequest) error +} + +func (i *stubIndex) Index(id string, data interface{}) error { + return i.err +} + +func (i *stubIndex) Delete(id string) error { + return i.err +} + +func (i *stubIndex) Batch(b *Batch) error { + return i.err +} + +func (i *stubIndex) Document(id string) (index.Document, error) { + if i.documentResult != nil { + return i.documentResult, nil + } + return nil, i.err +} + +func (i *stubIndex) DocCount() (uint64, error) { + if i.docCountResult != nil { + return *i.docCountResult, nil + } + return 0, i.err +} + +func (i *stubIndex) Search(req *SearchRequest) (*SearchResult, error) { + return i.SearchInContext(context.Background(), req) +} + +func (i *stubIndex) SearchInContext(ctx context.Context, req *SearchRequest) (*SearchResult, error) { + if i.checkRequest != nil { + err := i.checkRequest(req) + if err != nil { + return nil, err + } + } + if i.searchResult != nil { + return i.searchResult, nil + } + return nil, i.err +} + +func (i *stubIndex) Fields() ([]string, error) { + return nil, i.err +} + +func (i *stubIndex) FieldDict(field string) (index.FieldDict, error) { + return nil, i.err +} + +func (i *stubIndex) FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) { + return nil, i.err +} + +func (i *stubIndex) FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) { + return nil, i.err +} + +func (i *stubIndex) Close() error { + return i.err +} + +func (i *stubIndex) Mapping() mapping.IndexMapping { + return nil +} + +func (i *stubIndex) Stats() *IndexStat { + return nil +} + +func (i *stubIndex) StatsMap() map[string]interface{} { + return nil +} + +func (i *stubIndex) GetInternal(key []byte) ([]byte, error) { + return nil, i.err +} + +func (i *stubIndex) SetInternal(key, val []byte) error { + return i.err +} + +func (i *stubIndex) DeleteInternal(key []byte) error { + return i.err +} + +func (i *stubIndex) Advanced() (index.Index, error) { + return nil, nil +} + +func (i *stubIndex) NewBatch() *Batch { + return &Batch{} +} + +func (i *stubIndex) Name() string { + return i.name +} + +func (i *stubIndex) SetName(name string) { + i.name = name +} diff --git a/index_impl.go b/index_impl.go new file mode 100644 index 0000000..a775f0f --- /dev/null +++ b/index_impl.go @@ -0,0 +1,1691 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/microseconds" + "github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/milliseconds" + "github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/nanoseconds" + "github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/seconds" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/index/upsidedown" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/collector" + "github.com/blevesearch/bleve/v2/search/facet" + "github.com/blevesearch/bleve/v2/search/highlight" + "github.com/blevesearch/bleve/v2/search/query" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/geo/s2" +) + +type indexImpl struct { + path string + name string + meta *indexMeta + i index.Index + m mapping.IndexMapping + mutex sync.RWMutex + open bool + stats *IndexStat +} + +const storePath = "store" + +const ( + SearchQueryStartCallbackKey search.ContextKey = "_search_query_start_callback_key" + SearchQueryEndCallbackKey search.ContextKey = "_search_query_end_callback_key" +) + +type ( + SearchQueryStartCallbackFn func(size uint64) error + SearchQueryEndCallbackFn func(size uint64) error +) + +func indexStorePath(path string) string { + return path + string(os.PathSeparator) + storePath +} + +func newIndexUsing(path string, mapping mapping.IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (*indexImpl, error) { + // first validate the mapping + err := mapping.Validate() + if err != nil { + return nil, err + } + + if kvconfig == nil { + kvconfig = map[string]interface{}{} + } + + if kvstore == "" { + return nil, fmt.Errorf("bleve not configured for file based indexing") + } + + rv := indexImpl{ + path: path, + name: path, + m: mapping, + } + rv.meta, err = newIndexMeta(indexType, kvstore, kvconfig, path) + if err != nil { + return nil, err + } + rv.stats = &IndexStat{i: &rv} + // at this point there is hope that we can be successful, so save index meta + if path != "" { + err = rv.meta.Save(path) + if err != nil { + return nil, err + } + kvconfig["create_if_missing"] = true + kvconfig["error_if_exists"] = true + kvconfig["path"] = indexStorePath(path) + } else { + kvconfig["path"] = "" + } + + // open the index + indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType) + if indexTypeConstructor == nil { + return nil, ErrorUnknownIndexType + } + + rv.i, err = indexTypeConstructor(rv.meta.Storage, kvconfig, Config.analysisQueue) + if err != nil { + return nil, err + } + err = rv.i.Open() + if err != nil { + return nil, err + } + defer func(rv *indexImpl) { + if !rv.open { + rv.i.Close() + } + }(&rv) + + // now persist the mapping + mappingBytes, err := util.MarshalJSON(mapping) + if err != nil { + return nil, err + } + err = rv.i.SetInternal(util.MappingInternalKey, mappingBytes) + if err != nil { + return nil, err + } + + // mark the index as open + rv.mutex.Lock() + defer rv.mutex.Unlock() + rv.open = true + indexStats.Register(&rv) + return &rv, nil +} + +func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *indexImpl, err error) { + rv = &indexImpl{ + path: path, + name: path, + } + rv.stats = &IndexStat{i: rv} + + rv.meta, err = openIndexMeta(path) + if err != nil { + return nil, err + } + + // backwards compatibility if index type is missing + if rv.meta.IndexType == "" { + rv.meta.IndexType = upsidedown.Name + } + + var um *mapping.IndexMappingImpl + var umBytes []byte + + storeConfig := rv.meta.Config + if storeConfig == nil { + storeConfig = map[string]interface{}{} + } + + storeConfig["path"] = indexStorePath(path) + storeConfig["create_if_missing"] = false + storeConfig["error_if_exists"] = false + for rck, rcv := range runtimeConfig { + storeConfig[rck] = rcv + if rck == "updated_mapping" { + if val, ok := rcv.(string); ok { + if len(val) == 0 { + return nil, fmt.Errorf("updated_mapping is empty") + } + umBytes = []byte(val) + + err = util.UnmarshalJSON(umBytes, &um) + if err != nil { + return nil, fmt.Errorf("error parsing updated_mapping into JSON: %v\nmapping contents:\n%v", err, rck) + } + } else { + return nil, fmt.Errorf("updated_mapping not of type string") + } + } + } + + // open the index + indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType) + if indexTypeConstructor == nil { + return nil, ErrorUnknownIndexType + } + + rv.i, err = indexTypeConstructor(rv.meta.Storage, storeConfig, Config.analysisQueue) + if err != nil { + return nil, err + } + + var ui index.UpdateIndex + if um != nil { + var ok bool + ui, ok = rv.i.(index.UpdateIndex) + if !ok { + return nil, fmt.Errorf("updated mapping present for unupdatable index") + } + err = ui.OpenMeta() + if err != nil { + return nil, err + } + } else { + err = rv.i.Open() + if err != nil { + return nil, err + } + } + defer func(rv *indexImpl) { + if !rv.open { + rv.i.Close() + } + }(rv) + + // now load the mapping + indexReader, err := rv.i.Reader() + if err != nil { + return nil, err + } + defer func() { + if cerr := indexReader.Close(); cerr != nil && err == nil { + err = cerr + } + }() + + mappingBytes, err := indexReader.GetInternal(util.MappingInternalKey) + if err != nil { + return nil, err + } + + var im *mapping.IndexMappingImpl + err = util.UnmarshalJSON(mappingBytes, &im) + if err != nil { + return nil, fmt.Errorf("error parsing mapping JSON: %v\nmapping contents:\n%s", err, string(mappingBytes)) + } + + // validate the mapping + err = im.Validate() + if err != nil { + // no longer return usable index on error because there + // is a chance the index is not open at this stage + return nil, err + } + + // Validate and update the index with the new mapping + if um != nil && ui != nil { + err = um.Validate() + if err != nil { + return nil, err + } + + var fieldInfo map[string]*index.UpdateFieldInfo + fieldInfo, err = DeletedFields(im, um) + if err != nil { + return nil, err + } + + err = ui.UpdateFields(fieldInfo, umBytes) + if err != nil { + return nil, err + } + im = um + + err = rv.i.Open() + if err != nil { + return nil, err + } + } + + // mark the index as open + rv.mutex.Lock() + defer rv.mutex.Unlock() + rv.open = true + + rv.m = im + indexStats.Register(rv) + return rv, err +} + +// Advanced returns internal index implementation +func (i *indexImpl) Advanced() (index.Index, error) { + return i.i, nil +} + +// Mapping returns the IndexMapping in use by this +// Index. +func (i *indexImpl) Mapping() mapping.IndexMapping { + return i.m +} + +// Index the object with the specified identifier. +// The IndexMapping for this index will determine +// how the object is indexed. +func (i *indexImpl) Index(id string, data interface{}) (err error) { + if id == "" { + return ErrorEmptyID + } + + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + i.FireIndexEvent() + + doc := document.NewDocument(id) + err = i.m.MapDocument(doc, data) + if err != nil { + return + } + err = i.i.Update(doc) + return +} + +// IndexSynonym indexes a synonym definition, with the specified id and belonging to the specified collection. +// Synonym definition defines term relationships for query expansion in searches. +func (i *indexImpl) IndexSynonym(id string, collection string, definition *SynonymDefinition) error { + if id == "" { + return ErrorEmptyID + } + + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + i.FireIndexEvent() + + synMap, ok := i.m.(mapping.SynonymMapping) + if !ok { + return ErrorSynonymSearchNotSupported + } + + if err := definition.Validate(); err != nil { + return err + } + + doc := document.NewSynonymDocument(id) + err := synMap.MapSynonymDocument(doc, collection, definition.Input, definition.Synonyms) + if err != nil { + return err + } + err = i.i.Update(doc) + return err +} + +func (i *indexImpl) Train(batch *Batch) error { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + if vi, ok := i.i.(index.TrainableIndex); ok { + return vi.Train(batch.internal) + } + return ErrorTrainingNotSupported +} + +// IndexAdvanced takes a document.Document object +// skips the mapping and indexes it. +func (i *indexImpl) IndexAdvanced(doc *document.Document) (err error) { + if doc.ID() == "" { + return ErrorEmptyID + } + + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + err = i.i.Update(doc) + return +} + +// Delete entries for the specified identifier from +// the index. +func (i *indexImpl) Delete(id string) (err error) { + if id == "" { + return ErrorEmptyID + } + + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + err = i.i.Delete(id) + return +} + +// Batch executes multiple Index and Delete +// operations at the same time. There are often +// significant performance benefits when performing +// operations in a batch. +func (i *indexImpl) Batch(b *Batch) error { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + return i.i.Batch(b.internal) +} + +// Document is used to find the values of all the +// stored fields for a document in the index. These +// stored fields are put back into a Document object +// and returned. +func (i *indexImpl) Document(id string) (doc index.Document, err error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return nil, ErrorIndexClosed + } + indexReader, err := i.i.Reader() + if err != nil { + return nil, err + } + defer func() { + if cerr := indexReader.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + doc, err = indexReader.Document(id) + if err != nil { + return nil, err + } + return doc, nil +} + +// DocCount returns the number of documents in the +// index. +func (i *indexImpl) DocCount() (count uint64, err error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return 0, ErrorIndexClosed + } + + // open a reader for this search + indexReader, err := i.i.Reader() + if err != nil { + return 0, fmt.Errorf("error opening index reader %v", err) + } + defer func() { + if cerr := indexReader.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + count, err = indexReader.DocCount() + return +} + +// Search executes a search request operation. +// Returns a SearchResult object or an error. +func (i *indexImpl) Search(req *SearchRequest) (sr *SearchResult, err error) { + return i.SearchInContext(context.Background(), req) +} + +// returns the set of file callback writer ids in use by the index +func (i *indexImpl) FileWriterIDsInUse() (map[string]struct{}, error) { + ids := map[string]struct{}{i.meta.fileReader.Id(): {}} + + if cidx, ok := i.i.(IndexWithCallbacks); ok { + cIds, err := cidx.FileWriterIDsInUse() + if err != nil { + return nil, err + } + for k := range cIds { + ids[k] = struct{}{} + } + } else { + // if the underlying index does not support callbacks, we + // assume that the data being written is with the default + // writer id which is the empty string + ids[util.DefaultFileCallbackId] = struct{}{} + } + + return ids, nil +} + +// drops the file callback writer ids from the index and +// re-processes data with the latest file callback writer id +func (i *indexImpl) DropFileWriterIDs(ids map[string]struct{}) error { + i.mutex.Lock() + if _, ok := ids[i.meta.fileReader.Id()]; ok { + var err error + err = i.meta.UpdateWriter(i.path) + if err != nil { + return err + } + } + i.mutex.Unlock() + + if cidx, ok := i.i.(IndexWithCallbacks); ok { + return cidx.DropFileWriterIDs(ids) + } else { + // if the underlying index does not support callbacks and the request is + // to drop the empty id, which is the default id, we return an error + // because it is not possible to drop it + if _, ok := ids[util.DefaultFileCallbackId]; ok { + return fmt.Errorf("underlying index does not support DropFileWriterIDs") + } + } + + return nil +} + +var ( + documentMatchEmptySize int + searchContextEmptySize int + facetResultEmptySize int + documentEmptySize int +) + +func init() { + var dm search.DocumentMatch + documentMatchEmptySize = dm.Size() + + var sc search.SearchContext + searchContextEmptySize = sc.Size() + + var fr search.FacetResult + facetResultEmptySize = fr.Size() + + var d document.Document + documentEmptySize = d.Size() +} + +// memNeededForSearch is a helper function that returns an estimate of RAM +// needed to execute a search request. +func memNeededForSearch(req *SearchRequest, + searcher search.Searcher, + topnCollector *collector.TopNCollector, +) uint64 { + backingSize := req.Size + req.From + 1 + if req.Size+req.From > collector.PreAllocSizeSkipCap { + backingSize = collector.PreAllocSizeSkipCap + 1 + } + numDocMatches := backingSize + searcher.DocumentMatchPoolSize() + + estimate := 0 + + // overhead, size in bytes from collector + estimate += topnCollector.Size() + + // pre-allocing DocumentMatchPool + estimate += searchContextEmptySize + numDocMatches*documentMatchEmptySize + + // searcher overhead + estimate += searcher.Size() + + // overhead from results, lowestMatchOutsideResults + estimate += (numDocMatches + 1) * documentMatchEmptySize + + // additional overhead from SearchResult + estimate += reflectStaticSizeSearchResult + reflectStaticSizeSearchStatus + + // overhead from facet results + if req.Facets != nil { + estimate += len(req.Facets) * facetResultEmptySize + } + + // highlighting, store + if len(req.Fields) > 0 || req.Highlight != nil { + // Size + From => number of hits + estimate += (req.Size + req.From) * documentEmptySize + } + + return uint64(estimate) +} + +func (i *indexImpl) preSearch(ctx context.Context, req *SearchRequest, reader index.IndexReader) (*SearchResult, error) { + var knnHits []*search.DocumentMatch + var err error + if requestHasKNN(req) { + knnHits, err = i.runKnnCollector(ctx, req, reader, true) + if err != nil { + return nil, err + } + } + + var fts search.FieldTermSynonymMap + var count uint64 + var fieldCardinality map[string]int + if !isMatchNoneQuery(req.Query) { + if synMap, ok := i.m.(mapping.SynonymMapping); ok { + if synReader, ok := reader.(index.ThesaurusReader); ok { + fts, err = query.ExtractSynonyms(ctx, synMap, synReader, req.Query, fts) + if err != nil { + return nil, err + } + } + } + if ok := isBM25Enabled(i.m); ok { + fieldCardinality = make(map[string]int) + count, err = reader.DocCount() + if err != nil { + return nil, err + } + + fs, err := query.ExtractFields(req.Query, i.m, search.NewFieldSet()) + if err != nil { + return nil, err + } + for field := range fs { + if bm25Reader, ok := reader.(index.BM25Reader); ok { + fieldCardinality[field], err = bm25Reader.FieldCardinality(field) + if err != nil { + return nil, err + } + } + } + } + } + + return &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + }, + Hits: knnHits, + SynonymResult: fts, + BM25Stats: &search.BM25Stats{ + DocCount: float64(count), + FieldCardinality: fieldCardinality, + }, + }, nil +} + +// SearchInContext executes a search request operation within the provided +// Context. Returns a SearchResult object or an error. +func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr *SearchResult, err error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + searchStart := time.Now() + + if !i.open { + return nil, ErrorIndexClosed + } + + // open a reader for this search + indexReader, err := i.i.Reader() + if err != nil { + return nil, fmt.Errorf("error opening index reader %v", err) + } + defer func() { + if cerr := indexReader.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + // rescorer will be set if score fusion is supposed to happen + // at this alias (root alias), else will be nil + var rescorer *rescorer + if _, ok := ctx.Value(search.ScoreFusionKey).(bool); !ok { + // new context will be used in internal functions to collect data + // as suitable for hybrid search. Rescorer is used for rescoring + // using fusion algorithms. + if IsScoreFusionRequested(req) { + ctx = context.WithValue(ctx, search.ScoreFusionKey, true) + rescorer = newRescorer(req) + rescorer.prepareSearchRequest() + defer rescorer.restoreSearchRequest() + } + } + + // ------------------------------------------------------------------------------------------ + // set up additional contexts for any search operation that will proceed from + // here, such as presearch, knn collector, topn collector etc. + + // Scoring model callback to be used to get scoring model + scoringModelCallback := func() string { + if isBM25Enabled(i.m) { + return index.BM25Scoring + } + return index.DefaultScoringModel + } + ctx = context.WithValue(ctx, search.GetScoringModelCallbackKey, + search.GetScoringModelCallbackFn(scoringModelCallback)) + + // This callback and variable handles the tracking of bytes read + // 1. as part of creation of tfr and its Next() calls which is + // accounted by invoking this callback when the TFR is closed. + // 2. the docvalues portion (accounted in collector) and the retrieval + // of stored fields bytes (by LoadAndHighlightFields) + var totalSearchCost uint64 + sendBytesRead := func(bytesRead uint64) { + totalSearchCost += bytesRead + } + // Ensure IO cost accounting and result cost assignment happen on all return paths + defer func() { + if sr != nil { + sr.Cost = totalSearchCost + } + if is, ok := indexReader.(*scorch.IndexSnapshot); ok { + is.UpdateIOStats(totalSearchCost) + } + search.RecordSearchCost(ctx, search.DoneM, 0) + }() + + ctx = context.WithValue(ctx, search.SearchIOStatsCallbackKey, search.SearchIOStatsCallbackFunc(sendBytesRead)) + + // Geo buffer pool callback to be used for getting geo buffer pool + var bufPool *s2.GeoBufferPool + getBufferPool := func() *s2.GeoBufferPool { + if bufPool == nil { + bufPool = s2.NewGeoBufferPool(search.MaxGeoBufPoolSize, search.MinGeoBufPoolSize) + } + + return bufPool + } + + ctx = context.WithValue(ctx, search.GeoBufferPoolCallbackKey, search.GeoBufferPoolCallbackFunc(getBufferPool)) + // check if the index mapping has any nested fields, which should force + // all collectors and searchers to be run in nested mode + if nm, ok := i.m.(mapping.NestedMapping); ok { + if nm.CountNested() > 0 { + ctx = context.WithValue(ctx, search.NestedSearchKey, true) + } + } + // Install here, before presearch and the knn collector, so the counter also + // covers presearch and KNN filter searchers, not just the main query's. + ctx = search.ContextWithTermSearchersCounter(ctx) + // ------------------------------------------------------------------------------------------ + + if _, ok := ctx.Value(search.PreSearchKey).(bool); ok { + sr, err = i.preSearch(ctx, req, indexReader) + if err != nil { + return nil, err + } + // increment the search count here itself, + // since the presearch may already satisfy + // the search request + atomic.AddUint64(&i.stats.searches, 1) + // increment the search time stat here as well, + // since presearch is part of the overall search + // operation and should be included in the search + // time stat + searchDuration := time.Since(searchStart) + atomic.AddUint64(&i.stats.searchTime, uint64(searchDuration)) + + return sr, nil + } + + var reverseQueryExecution bool + if req.SearchBefore != nil { + reverseQueryExecution = true + req.Sort.Reverse() + req.SearchAfter = req.SearchBefore + req.SearchBefore = nil + } + + coll, err := i.buildTopNCollector(ctx, req, indexReader) + if err != nil { + return nil, err + } + + // score="none" + Size means "return any Size+From matching docs", so the + // collector may stop scanning early — provided nothing below depends on + // unseen matches (facets, KNN, pagination cursor, nested rollup, field sort). + if req.Score == ScoreNone && req.Size > 0 && + len(req.Facets) == 0 && + !requestHasKNN(req) && + req.SearchAfter == nil && !reverseQueryExecution && + len(req.Sort) == 1 && req.Sort[0].RequiresScoring() { + if nestedMode, ok := ctx.Value(search.NestedSearchKey).(bool); !ok || !nestedMode { + coll.SetEarlyStop(req.Size + req.From) + } + } + + var knnHits []*search.DocumentMatch + var skipKNNCollector bool + + var fts search.FieldTermSynonymMap + var skipSynonymCollector bool + + var bm25Stats *search.BM25Stats + var ok bool + if req.PreSearchData != nil { + for k, v := range req.PreSearchData { + switch k { + case search.KnnPreSearchDataKey: + if v != nil { + knnHits, ok = v.([]*search.DocumentMatch) + if !ok { + return nil, fmt.Errorf("knn preSearchData must be of type []*search.DocumentMatch") + } + skipKNNCollector = true + } + case search.SynonymPreSearchDataKey: + if v != nil { + fts, ok = v.(search.FieldTermSynonymMap) + if !ok { + return nil, fmt.Errorf("synonym preSearchData must be of type search.FieldTermSynonymMap") + } + skipSynonymCollector = true + } + case search.BM25PreSearchDataKey: + if v != nil { + bm25Stats, ok = v.(*search.BM25Stats) + if !ok { + return nil, fmt.Errorf("bm25 preSearchData must be of type *search.BM25Stats") + } + } + } + } + } + + _, contextScoreFusionKeyExists := ctx.Value(search.ScoreFusionKey).(bool) + + if !contextScoreFusionKeyExists { + // if no score fusion, default behaviour + if !skipKNNCollector && requestHasKNN(req) { + knnHits, err = i.runKnnCollector(ctx, req, indexReader, false) + if err != nil { + return nil, err + } + } + } else { + // if score fusion, run collect if rescorer is defined + if rescorer != nil && requestHasKNN(req) { + knnHits, err = i.runKnnCollector(ctx, req, indexReader, false) + if err != nil { + return nil, err + } + } + } + + if !skipSynonymCollector { + if synMap, ok := i.m.(mapping.SynonymMapping); ok && synMap.SynonymCount() > 0 { + if synReader, ok := indexReader.(index.ThesaurusReader); ok { + fts, err = query.ExtractSynonyms(ctx, synMap, synReader, req.Query, fts) + if err != nil { + return nil, err + } + } + } + } + + // if score fusion, no faceting for knn hits is done + // hence we can skip setting the knn hits in the collector + if !contextScoreFusionKeyExists { + setKnnHitsInCollector(knnHits, coll) + } + + if fts != nil { + if is, ok := indexReader.(*scorch.IndexSnapshot); ok { + is.UpdateSynonymSearchCount(1) + } + ctx = context.WithValue(ctx, search.FieldTermSynonymMapKey, fts) + } + + // set the bm25Stats (stats important for consistent scoring) in + // the context object + if bm25Stats != nil { + ctx = context.WithValue(ctx, search.BM25StatsKey, bm25Stats) + } + + searcher, err := req.Query.Searcher(ctx, indexReader, i.m, search.SearcherOptions{ + Explain: req.Explain, + IncludeTermVectors: req.IncludeLocations || req.Highlight != nil, + Score: req.Score, + }) + if err != nil { + return nil, err + } + defer func() { + if serr := searcher.Close(); err == nil && serr != nil { + err = serr + } + }() + + if req.Facets != nil { + facetsBuilder := search.NewFacetsBuilder(indexReader) + for facetName, facetRequest := range req.Facets { + if facetRequest.NumericRanges != nil { + // build numeric range facet + facetBuilder := facet.NewNumericFacetBuilder(facetRequest.Field, facetRequest.Size) + for _, nr := range facetRequest.NumericRanges { + facetBuilder.AddRange(nr.Name, nr.Min, nr.Max) + } + facetsBuilder.Add(facetName, facetBuilder) + } else if facetRequest.DateTimeRanges != nil { + // build date range facet + facetBuilder := facet.NewDateTimeFacetBuilder(facetRequest.Field, facetRequest.Size) + for _, dr := range facetRequest.DateTimeRanges { + dateTimeParserName := defaultDateTimeParser + if dr.DateTimeParser != "" { + dateTimeParserName = dr.DateTimeParser + } + dateTimeParser := i.m.DateTimeParserNamed(dateTimeParserName) + if dateTimeParser == nil { + return nil, fmt.Errorf("no date time parser named `%s` registered", dateTimeParserName) + } + start, end, err := dr.ParseDates(dateTimeParser) + if err != nil { + return nil, fmt.Errorf("ParseDates err: %v, using date time parser named %s", err, dateTimeParserName) + } + if start.IsZero() && end.IsZero() { + return nil, fmt.Errorf("date range query must specify either start, end or both for date range name '%s'", dr.Name) + } + facetBuilder.AddRange(dr.Name, start, end) + } + facetsBuilder.Add(facetName, facetBuilder) + } else { + // build terms facet + facetBuilder := facet.NewTermsFacetBuilder(facetRequest.Field, facetRequest.Size) + + // Set prefix filter if provided + if facetRequest.TermPrefix != "" { + facetBuilder.SetPrefixFilter(facetRequest.TermPrefix) + } + + // Set regex filter if provided + if facetRequest.TermPattern != "" { + // Use cached compiled pattern if available, otherwise compile it now + if facetRequest.compiledPattern != nil { + facetBuilder.SetRegexFilter(facetRequest.compiledPattern) + } else { + regex, err := regexp.Compile(facetRequest.TermPattern) + if err != nil { + return nil, fmt.Errorf("error compiling regex pattern for facet '%s': %v", facetName, err) + } + facetBuilder.SetRegexFilter(regex) + } + } + + facetsBuilder.Add(facetName, facetBuilder) + } + } + coll.SetFacetsBuilder(facetsBuilder) + } + + memNeeded := memNeededForSearch(req, searcher, coll) + if cb := ctx.Value(SearchQueryStartCallbackKey); cb != nil { + if cbF, ok := cb.(SearchQueryStartCallbackFn); ok { + err = cbF(memNeeded) + } + } + if err != nil { + return nil, err + } + + if cb := ctx.Value(SearchQueryEndCallbackKey); cb != nil { + if cbF, ok := cb.(SearchQueryEndCallbackFn); ok { + defer func() { + _ = cbF(memNeeded) + }() + } + } + + err = coll.Collect(ctx, searcher, indexReader) + if err != nil { + return nil, err + } + + hits := coll.Results() + + var highlighter highlight.Highlighter + + if req.Highlight != nil { + // get the right highlighter + highlighter, err = Config.Cache.HighlighterNamed(Config.DefaultHighlighter) + if err != nil { + return nil, err + } + if req.Highlight.Style != nil { + highlighter, err = Config.Cache.HighlighterNamed(*req.Highlight.Style) + if err != nil { + return nil, err + } + } + if highlighter == nil { + return nil, fmt.Errorf("no highlighter named `%s` registered", *req.Highlight.Style) + } + } + + var storedFieldsCost uint64 + for _, hit := range hits { + // KNN documents will already have their Index value set as part of the knn collector output + // so check if the index is empty and set it to the current index name + if i.name != "" && hit.Index == "" { + hit.Index = i.name + } + err, storedFieldsBytes := LoadAndHighlightAllFields(hit, req, i.name, indexReader, highlighter) + if err != nil { + return nil, err + } + storedFieldsCost += storedFieldsBytes + } + + totalSearchCost += storedFieldsCost + search.RecordSearchCost(ctx, search.AddM, storedFieldsCost) + + if req.PreSearchData == nil { + // increment the search count only if this is not a second-phase search + // (e.g., for Hybrid Search), since the first-phase search already increments it + atomic.AddUint64(&i.stats.searches, 1) + } + // increment the search time stat, as the first-phase search is part of + // the overall operation; adding second-phase time later keeps it accurate + searchDuration := time.Since(searchStart) + atomic.AddUint64(&i.stats.searchTime, uint64(searchDuration)) + + if Config.SlowSearchLogThreshold > 0 && + searchDuration > Config.SlowSearchLogThreshold { + logger.Printf("slow search took %s - %v", searchDuration, req) + } + + if reverseQueryExecution { + // reverse the sort back to the original + req.Sort.Reverse() + // resort using the original order + mhs := newSearchHitSorter(req.Sort, hits) + req.SortFunc()(mhs) + // reset request + req.SearchBefore = req.SearchAfter + req.SearchAfter = nil + } + + totalRelation := TotalRelationEq + if coll.EarlyStopped() { + totalRelation = TotalRelationGte + } + rv := &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Successful: 1, + }, + Hits: hits, + Total: coll.Total(), + TotalRelation: totalRelation, + MaxScore: coll.MaxScore(), + Took: searchDuration, + Facets: coll.FacetResults(), + } + + // rescore if fusion flag is set + if rescorer != nil { + rv.Hits, rv.Total, rv.MaxScore = rescorer.rescore(rv.Hits, knnHits) + rescorer.restoreSearchRequest() + rv.Hits = hitsInCurrentPage(req, rv.Hits) + } + + if req.Explain { + rv.Request = req + } + + return rv, nil +} + +func LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest, + indexName string, r index.IndexReader, + highlighter highlight.Highlighter, +) (error, uint64) { + var totalStoredFieldsBytes uint64 + if len(req.Fields) > 0 || highlighter != nil { + doc, err := r.Document(hit.ID) + if err == nil && doc != nil { + if len(req.Fields) > 0 && hit.Fields == nil { + totalStoredFieldsBytes = doc.StoredFieldsBytes() + fieldsToLoad := deDuplicate(req.Fields) + for _, f := range fieldsToLoad { + doc.VisitFields(func(docF index.Field) { + if f == "*" || docF.Name() == f { + var value interface{} + switch docF := docF.(type) { + case index.TextField: + value = docF.Text() + case index.NumericField: + num, err := docF.Number() + if err == nil { + value = num + } + case index.DateTimeField: + datetime, layout, err := docF.DateTime() + if err == nil { + if layout == "" { + // missing layout means we fallback to + // the default layout which is RFC3339 + value = datetime.Format(time.RFC3339) + } else { + // the layout here can now either be representative + // of an actual datetime layout or a timestamp + switch layout { + case seconds.Name: + value = strconv.FormatInt(datetime.Unix(), 10) + case milliseconds.Name: + value = strconv.FormatInt(datetime.UnixMilli(), 10) + case microseconds.Name: + value = strconv.FormatInt(datetime.UnixMicro(), 10) + case nanoseconds.Name: + value = strconv.FormatInt(datetime.UnixNano(), 10) + default: + // the layout for formatting the date to a string + // is provided by a datetime parser which is not + // handling the timestamp case, hence the layout + // can be directly used to format the date + value = datetime.Format(layout) + } + } + } + case index.BooleanField: + boolean, err := docF.Boolean() + if err == nil { + value = boolean + } + case index.GeoPointField: + lon, err := docF.Lon() + if err == nil { + lat, err := docF.Lat() + if err == nil { + value = []float64{lon, lat} + } + } + case index.GeoShapeField: + v, err := docF.GeoShape() + if err == nil { + value = v + } + case index.IPField: + ip, err := docF.IP() + if err == nil { + value = ip.String() + } + } + + if value != nil { + hit.AddFieldValue(docF.Name(), value) + } + } + }) + } + } + if highlighter != nil { + highlightFields := req.Highlight.Fields + if highlightFields == nil { + // add all fields with matches + highlightFields = make([]string, 0, len(hit.Locations)) + for k := range hit.Locations { + highlightFields = append(highlightFields, k) + } + } + for _, hf := range highlightFields { + highlighter.BestFragmentsInField(hit, doc, hf, 1) + } + } + } else if doc == nil { + // unexpected case, a doc ID that was found as a search hit + // was unable to be found during document lookup + return ErrorIndexReadInconsistency, 0 + } + } + + return nil, totalStoredFieldsBytes +} + +const NestedDocumentKey = "_$nested" + +// LoadAndHighlightAllFields loads stored fields + highlights for root and its descendants. +// All descendant documents are collected into a _$nested array in the root DocumentMatch. +func LoadAndHighlightAllFields( + root *search.DocumentMatch, + req *SearchRequest, + indexName string, + r index.IndexReader, + highlighter highlight.Highlighter, +) (error, uint64) { + var totalStoredFieldsBytes uint64 + // load root fields/highlights + err, bytes := LoadAndHighlightFields(root, req, indexName, r, highlighter) + totalStoredFieldsBytes += bytes + if err != nil { + return err, totalStoredFieldsBytes + } + // collect all descendant documents + nestedDocs := make([]*search.NestedDocumentMatch, 0, len(root.Descendants)) + // create a dummy desc DocumentMatch to reuse LoadAndHighlightFields + desc := &search.DocumentMatch{} + for _, descID := range root.Descendants { + extID, err := r.ExternalID(descID) + if err != nil { + return err, totalStoredFieldsBytes + } + // reset desc for reuse + desc.ID = extID + desc.IndexInternalID = descID + desc.Locations = root.Locations + err, bytes := LoadAndHighlightFields(desc, req, indexName, r, highlighter) + totalStoredFieldsBytes += bytes + if err != nil { + return err, totalStoredFieldsBytes + } + // copy fields to nested doc and append + if len(desc.Fields) != 0 || len(desc.Fragments) != 0 { + nestedDocs = append(nestedDocs, search.NewNestedDocumentMatch(desc.Fields, desc.Fragments)) + } + desc.Fields = nil + desc.Fragments = nil + } + // add nested documents to root under _$nested key + if len(nestedDocs) > 0 { + root.AddFieldValue(NestedDocumentKey, nestedDocs) + } + return nil, totalStoredFieldsBytes +} + +// Fields returns the name of all the fields this +// Index has operated on. +func (i *indexImpl) Fields() (fields []string, err error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return nil, ErrorIndexClosed + } + + indexReader, err := i.i.Reader() + if err != nil { + return nil, err + } + defer func() { + if cerr := indexReader.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + fields, err = indexReader.Fields() + if err != nil { + return nil, err + } + return fields, nil +} + +func (i *indexImpl) FieldDict(field string) (index.FieldDict, error) { + i.mutex.RLock() + + if !i.open { + i.mutex.RUnlock() + return nil, ErrorIndexClosed + } + + indexReader, err := i.i.Reader() + if err != nil { + i.mutex.RUnlock() + return nil, err + } + + fieldDict, err := indexReader.FieldDict(field) + if err != nil { + i.mutex.RUnlock() + return nil, err + } + + return &indexImplFieldDict{ + index: i, + indexReader: indexReader, + fieldDict: fieldDict, + }, nil +} + +func (i *indexImpl) FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) { + i.mutex.RLock() + + if !i.open { + i.mutex.RUnlock() + return nil, ErrorIndexClosed + } + + indexReader, err := i.i.Reader() + if err != nil { + i.mutex.RUnlock() + return nil, err + } + + fieldDict, err := indexReader.FieldDictRange(field, startTerm, endTerm) + if err != nil { + i.mutex.RUnlock() + return nil, err + } + + return &indexImplFieldDict{ + index: i, + indexReader: indexReader, + fieldDict: fieldDict, + }, nil +} + +func (i *indexImpl) FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) { + i.mutex.RLock() + + if !i.open { + i.mutex.RUnlock() + return nil, ErrorIndexClosed + } + + indexReader, err := i.i.Reader() + if err != nil { + i.mutex.RUnlock() + return nil, err + } + + fieldDict, err := indexReader.FieldDictPrefix(field, termPrefix) + if err != nil { + i.mutex.RUnlock() + return nil, err + } + + return &indexImplFieldDict{ + index: i, + indexReader: indexReader, + fieldDict: fieldDict, + }, nil +} + +func (i *indexImpl) Close() error { + i.mutex.Lock() + defer i.mutex.Unlock() + + indexStats.UnRegister(i) + + i.open = false + return i.i.Close() +} + +func (i *indexImpl) Stats() *IndexStat { + return i.stats +} + +func (i *indexImpl) StatsMap() map[string]interface{} { + return i.stats.statsMap() +} + +func (i *indexImpl) GetInternal(key []byte) (val []byte, err error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return nil, ErrorIndexClosed + } + + reader, err := i.i.Reader() + if err != nil { + return nil, err + } + defer func() { + if cerr := reader.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + val, err = reader.GetInternal(key) + if err != nil { + return nil, err + } + return val, nil +} + +func (i *indexImpl) SetInternal(key, val []byte) error { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + return i.i.SetInternal(key, val) +} + +func (i *indexImpl) DeleteInternal(key []byte) error { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + return i.i.DeleteInternal(key) +} + +// NewBatch creates a new empty batch. +func (i *indexImpl) NewBatch() *Batch { + return &Batch{ + index: i, + internal: index.NewBatch(), + } +} + +func (i *indexImpl) Name() string { + return i.name +} + +func (i *indexImpl) SetName(name string) { + indexStats.UnRegister(i) + i.name = name + indexStats.Register(i) +} + +type indexImplFieldDict struct { + index *indexImpl + indexReader index.IndexReader + fieldDict index.FieldDict +} + +func (f *indexImplFieldDict) BytesRead() uint64 { + return f.fieldDict.BytesRead() +} + +func (f *indexImplFieldDict) Next() (*index.DictEntry, error) { + return f.fieldDict.Next() +} + +func (f *indexImplFieldDict) Close() error { + defer f.index.mutex.RUnlock() + err := f.fieldDict.Close() + if err != nil { + return err + } + return f.indexReader.Close() +} + +func (f *indexImplFieldDict) Cardinality() int { + return f.fieldDict.Cardinality() +} + +// helper function to remove duplicate entries from slice of strings +func deDuplicate(fields []string) []string { + if len(fields) == 0 { + return fields + } + entries := make(map[string]struct{}) + ret := []string{} + for _, entry := range fields { + if _, exists := entries[entry]; !exists { + entries[entry] = struct{}{} + ret = append(ret, entry) + } + } + return ret +} + +type searchHitSorter struct { + hits search.DocumentMatchCollection + sort search.SortOrder + cachedScoring []bool + cachedDesc []bool +} + +func newSearchHitSorter(sort search.SortOrder, hits search.DocumentMatchCollection) *searchHitSorter { + return &searchHitSorter{ + sort: sort, + hits: hits, + cachedScoring: sort.CacheIsScore(), + cachedDesc: sort.CacheDescending(), + } +} + +func (m *searchHitSorter) Len() int { return len(m.hits) } +func (m *searchHitSorter) Swap(i, j int) { m.hits[i], m.hits[j] = m.hits[j], m.hits[i] } +func (m *searchHitSorter) Less(i, j int) bool { + c := m.sort.Compare(m.cachedScoring, m.cachedDesc, m.hits[i], m.hits[j]) + return c < 0 +} + +func (i *indexImpl) CopyTo(d index.Directory) (err error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + copyIndex, ok := i.i.(index.CopyIndex) + if !ok { + return fmt.Errorf("index implementation does not support copy reader") + } + + copyReader := copyIndex.CopyReader() + if copyReader == nil { + return fmt.Errorf("index's copyReader is nil") + } + + defer func() { + if cerr := copyReader.CloseCopyReader(); err == nil && cerr != nil { + err = cerr + } + }() + + err = copyReader.CopyTo(d) + if err != nil { + return fmt.Errorf("error copying index data: %v", err) + } + + // copy the metadata + return i.meta.CopyTo(i.path, d) +} + +func (i *indexImpl) CopyFile(file string, d index.IndexDirectory) (err error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + fileCopyIndex, ok := i.i.(IndexFileCopyable) + if !ok { + return fmt.Errorf("index implementation does not support file copy reader") + } + + return fileCopyIndex.CopyFile(file, d) +} + +func (i *indexImpl) SetPathInBolt(key []byte, value []byte) error { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + fileCopyIndex, ok := i.i.(IndexFileCopyable) + if !ok { + return fmt.Errorf("index implementation does not support file copy") + } + + return fileCopyIndex.SetPathInBolt(key, value) +} + +func (f FileSystemDirectory) GetWriter(filePath string) (io.WriteCloser, + error, +) { + dir, file := filepath.Split(filePath) + if dir != "" { + err := os.MkdirAll(filepath.Join(string(f), dir), os.ModePerm) + if err != nil { + return nil, err + } + } + + return os.OpenFile(filepath.Join(string(f), dir, file), + os.O_RDWR|os.O_CREATE, 0o600) +} + +func (i *indexImpl) FireIndexEvent() { + // get the internal index implementation + internalIndex, err := i.Advanced() + if err != nil { + return + } + // check if the internal index implementation supports events + if internalEventIndex, ok := internalIndex.(index.EventIndex); ok { + // fire the Index() event + internalEventIndex.FireIndexEvent() + } +} + +// ----------------------------------------------------------------------------- + +func (i *indexImpl) TermFrequencies(field string, limit int, descending bool) ( + []index.TermFreq, error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return nil, ErrorIndexClosed + } + + reader, err := i.i.Reader() + if err != nil { + return nil, err + } + defer func() { + if cerr := reader.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + insightsReader, ok := reader.(index.IndexInsightsReader) + if !ok { + return nil, fmt.Errorf("index reader does not support TermFrequencies") + } + + return insightsReader.TermFrequencies(field, limit, descending) +} + +func (i *indexImpl) CentroidCardinalities(field string, limit int, descending bool) ( + []index.CentroidCardinality, error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return nil, ErrorIndexClosed + } + + reader, err := i.i.Reader() + if err != nil { + return nil, err + } + defer func() { + if cerr := reader.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + insightsReader, ok := reader.(index.IndexInsightsReader) + if !ok { + return nil, fmt.Errorf("index reader does not support CentroidCardinalities") + } + + centroidCardinalities, err := insightsReader.CentroidCardinalities(field, limit, descending) + if err != nil { + return nil, err + } + + for j := 0; j < len(centroidCardinalities); j++ { + centroidCardinalities[j].Index = i.name + } + + return centroidCardinalities, nil +} + +func (i *indexImpl) buildTopNCollector(ctx context.Context, req *SearchRequest, reader index.IndexReader) (*collector.TopNCollector, error) { + newCollector := func() *collector.TopNCollector { + if req.SearchAfter != nil { + return collector.NewTopNCollectorAfter(req.Size, req.Sort, req.SearchAfter) + } + return collector.NewTopNCollector(req.Size, req.From, req.Sort) + } + + newNestedCollector := func(nr index.NestedReader) *collector.TopNCollector { + if req.SearchAfter != nil { + return collector.NewNestedTopNCollectorAfter(req.Size, req.Sort, req.SearchAfter, nr) + } + return collector.NewNestedTopNCollector(req.Size, req.From, req.Sort, nr) + } + + // check if we are in nested mode + if nestedMode, ok := ctx.Value(search.NestedSearchKey).(bool); ok && nestedMode { + // get the nested reader from the index reader + if nr, ok := reader.(index.NestedReader); ok { + // check if the mapping has any nested fields that intersect + if nm, ok := i.m.(mapping.NestedMapping); ok { + var fs search.FieldSet + var err error + fs, err = query.ExtractFields(req.Query, i.m, fs) + if err != nil { + return nil, err + } + if fs.HasID() || nm.IntersectsPrefix(fs) { + return newNestedCollector(nr), nil + } + } + } + } + return newCollector(), nil +} diff --git a/index_meta.go b/index_meta.go new file mode 100644 index 0000000..68f63cd --- /dev/null +++ b/index_meta.go @@ -0,0 +1,260 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "encoding/binary" + "fmt" + "os" + "path/filepath" + + "github.com/blevesearch/bleve/v2/index/upsidedown" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +const metaFilename = "index_meta.json" + +type indexMeta struct { + Storage string `json:"storage"` + IndexType string `json:"index_type"` + Config map[string]interface{} `json:"config,omitempty"` + fileWriter util.FileWriter + fileReader util.FileReader +} + +func newIndexMeta(indexType string, storage string, config map[string]interface{}, path string) (*indexMeta, error) { + indexMetaPath := indexMetaPath(path) + fileWriter, err := util.NewFileWriter([]byte(indexMetaPath)) + if err != nil { + return nil, fmt.Errorf("failed to create file writer for index meta: %w", err) + } + fileReader, err := util.NewFileReader(fileWriter.Id(), []byte(indexMetaPath)) + if err != nil { + return nil, fmt.Errorf("failed to create file reader for index meta: %w", err) + } + return &indexMeta{ + IndexType: indexType, + Storage: storage, + Config: config, + fileWriter: fileWriter, + fileReader: fileReader, + }, nil +} + +func openIndexMeta(path string) (*indexMeta, error) { + if _, err := os.Stat(path); os.IsNotExist(err) { + return nil, ErrorIndexPathDoesNotExist + } + indexMetaPath := indexMetaPath(path) + metaBytes, err := os.ReadFile(indexMetaPath) + if err != nil { + return nil, ErrorIndexMetaMissing + } + + // check if indexMetaPath+_temp exists, if so, this means a writer update was in progress + // and we should attempt to recover using the temp file + if _, err := os.Stat(indexMetaPath + "_temp"); err == nil { + tempBytes, err := os.ReadFile(indexMetaPath + "_temp") + if err == nil { + err = os.Rename(indexMetaPath+"_temp", indexMetaPath) + if err != nil { + return nil, err + } + metaBytes = tempBytes + } + } + + var im indexMeta + var fileReader util.FileReader + // attempt to unmarshal metabytes directly. If this succeeds, + // then we know there was no file callback writer used and we can + // proceed as normal. + err = util.UnmarshalJSON(metaBytes, &im) + if err != nil { + // on failure, we expect the last 4 bytes to be the length of the file + // callback id and the preceding bytes to be the file callback id, which + // we can use to obtain the file reader to read the actual meta data bytes + if len(metaBytes) < 4 { + return nil, ErrorIndexMetaCorrupt + } + + // read the length of the file callback id from the last 4 bytes + pos := len(metaBytes) - 4 + fileWriterIDLen := int(binary.BigEndian.Uint32(metaBytes[pos:])) + pos -= fileWriterIDLen + if pos < 0 { + return nil, ErrorIndexMetaCorrupt + } + + // read and initialize the file reader using the file callback id + fileWriterID := metaBytes[pos : pos+fileWriterIDLen] + fileReader, err = util.NewFileReader(string(fileWriterID), []byte(indexMetaPath)) + if err != nil { + return nil, err + } + + buf, err := fileReader.Process(metaBytes[0:pos]) + if err != nil { + return nil, err + } + err = util.UnmarshalJSON(buf, &im) + if err != nil { + return nil, ErrorIndexMetaCorrupt + } + } else { + fileReader, err = util.NewFileReader("", []byte(indexMetaPath)) + if err != nil { + return nil, err + } + } + im.fileReader = fileReader + + if im.IndexType == "" { + im.IndexType = upsidedown.Name + } + return &im, nil +} + +func (i *indexMeta) Save(path string) (err error) { + indexMetaPath := indexMetaPath(path) + // ensure any necessary parent directories exist + err = os.MkdirAll(path, 0700) + if err != nil { + if os.IsExist(err) { + return ErrorIndexPathExists + } + return err + } + metaBytes, err := util.MarshalJSON(i) + if err != nil { + return err + } + indexMetaFile, err := os.OpenFile(indexMetaPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + if err != nil { + if os.IsExist(err) { + return ErrorIndexPathExists + } + return err + } + defer func() { + if ierr := indexMetaFile.Close(); err == nil && ierr != nil { + err = ierr + } + }() + + metaBytes = i.fileWriter.Process(metaBytes) + + _, err = indexMetaFile.Write(metaBytes) + if err != nil { + return err + } + + _, err = indexMetaFile.Write([]byte(i.fileWriter.Id())) + if err != nil { + return err + } + + err = binary.Write(indexMetaFile, binary.BigEndian, uint32(len(i.fileWriter.Id()))) + if err != nil { + return err + } + + return nil +} + +func (i *indexMeta) CopyTo(path string, d index.Directory) (err error) { + metaBytes, err := os.ReadFile(indexMetaPath(path)) + if err != nil { + return err + } + + w, err := d.GetWriter(metaFilename) + if w == nil || err != nil { + return fmt.Errorf("invalid writer for file: %s, err: %v", + metaFilename, err) + } + defer w.Close() + + _, err = w.Write(metaBytes) + return err +} + +// updates the file callback writer id in the index meta, +// and re-processes data with the latest file callback writer +// returns the new file callback writer and reader to be used for +// future processing of index meta data +func (i *indexMeta) UpdateWriter(path string) error { + indexMetaPath := indexMetaPath(path) + metaBytes, err := util.MarshalJSON(i) + if err != nil { + return err + } + + i.fileWriter, err = util.NewFileWriter([]byte(indexMetaPath)) + if err != nil { + return err + } + metaBytes = i.fileWriter.Process(metaBytes) + + // write out new meta with new writer id, using temp file and rename to ensure atomicity + // if we crash in the middle of this, on next open we will see the temp file and recover using it + tempMetaPath := indexMetaPath + "_temp" + tempMetaFile, err := os.OpenFile(tempMetaPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + if err != nil { + if os.IsExist(err) { + return ErrorIndexPathExists + } + return err + } + + // write the meta bytes + _, err = tempMetaFile.Write(metaBytes) + if err != nil { + return err + } + // write the file callback id + _, err = tempMetaFile.Write([]byte(i.fileWriter.Id())) + if err != nil { + return err + } + // write the length of the file callback id + err = binary.Write(tempMetaFile, binary.BigEndian, uint32(len(i.fileWriter.Id()))) + if err != nil { + return err + } + // close file before renaming + err = tempMetaFile.Close() + if err != nil { + return err + } + // atomically rename temp file to index meta file + err = os.Rename(tempMetaPath, indexMetaPath) + if err != nil { + return err + } + + // initialize the new file reader for index meta + i.fileReader, err = util.NewFileReader(string(i.fileWriter.Id()), []byte(indexMetaPath)) + if err != nil { + return err + } + + return nil +} + +func indexMetaPath(path string) string { + return filepath.Join(path, metaFilename) +} diff --git a/index_meta_test.go b/index_meta_test.go new file mode 100644 index 0000000..57a5b69 --- /dev/null +++ b/index_meta_test.go @@ -0,0 +1,72 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "os" + "testing" + + "github.com/blevesearch/bleve/v2/util" +) + +func TestIndexMeta(t *testing.T) { + var testIndexPath = "doesnotexit.bleve" + defer func() { + err := os.RemoveAll(testIndexPath) + if err != nil { + t.Fatal(err) + } + }() + + // open non-existent meta should give an error + _, err := openIndexMeta(testIndexPath) + if err == nil { + t.Errorf("expected error, got nil") + } + + // create file reader and writer for meta + metaPath := indexMetaPath(testIndexPath) + fileWriter, err := util.NewFileWriter([]byte(metaPath)) + if err != nil { + t.Error(err) + } + fileReader, err := util.NewFileReader(fileWriter.Id(), []byte(metaPath)) + if err != nil { + t.Error(err) + } + + // create meta + im := &indexMeta{Storage: "boltdb", fileWriter: fileWriter, fileReader: fileReader} + err = im.Save(testIndexPath) + if err != nil { + t.Error(err) + } + im = nil + + // open a meta that exists + im, err = openIndexMeta(testIndexPath) + if err != nil { + t.Error(err) + } + if im.Storage != "boltdb" { + t.Errorf("expected storage 'boltdb', got '%s'", im.Storage) + } + + // save a meta that already exists + err = im.Save(testIndexPath) + if err == nil { + t.Errorf("expected error, got nil") + } +} diff --git a/index_stats.go b/index_stats.go new file mode 100644 index 0000000..2d303f6 --- /dev/null +++ b/index_stats.go @@ -0,0 +1,75 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "encoding/json" + "sync" + "sync/atomic" +) + +type IndexStat struct { + searches uint64 + searchTime uint64 + i *indexImpl +} + +func (is *IndexStat) statsMap() map[string]interface{} { + m := map[string]interface{}{} + m["index"] = is.i.i.StatsMap() + m["searches"] = atomic.LoadUint64(&is.searches) + m["search_time"] = atomic.LoadUint64(&is.searchTime) + return m +} + +func (is *IndexStat) MarshalJSON() ([]byte, error) { + m := is.statsMap() + return json.Marshal(m) +} + +type IndexStats struct { + indexes map[string]*IndexStat + mutex sync.RWMutex +} + +func NewIndexStats() *IndexStats { + return &IndexStats{ + indexes: make(map[string]*IndexStat), + } +} + +func (i *IndexStats) Register(index Index) { + i.mutex.Lock() + defer i.mutex.Unlock() + i.indexes[index.Name()] = index.Stats() +} + +func (i *IndexStats) UnRegister(index Index) { + i.mutex.Lock() + defer i.mutex.Unlock() + delete(i.indexes, index.Name()) +} + +func (i *IndexStats) String() string { + i.mutex.RLock() + defer i.mutex.RUnlock() + bytes, err := json.Marshal(i.indexes) + if err != nil { + return "error marshaling stats" + } + return string(bytes) +} + +var indexStats *IndexStats diff --git a/index_test.go b/index_test.go new file mode 100644 index 0000000..203f3f4 --- /dev/null +++ b/index_test.go @@ -0,0 +1,3247 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log" + "math" + "os" + "path/filepath" + "reflect" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/null" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/query" + index "github.com/blevesearch/bleve_index_api" + + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/index/upsidedown" +) + +type Fatalfable interface { + Fatalf(format string, args ...interface{}) +} + +func createTmpIndexPath(f Fatalfable) string { + tmpIndexPath, err := os.MkdirTemp("", "bleve-testidx") + if err != nil { + f.Fatalf("error creating temp dir: %v", err) + } + return tmpIndexPath +} + +func cleanupTmpIndexPath(f Fatalfable, path string) { + err := os.RemoveAll(path) + if err != nil { + f.Fatalf("error removing temp dir: %v", err) + } +} + +func TestCrud(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + idx, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + + doca := map[string]interface{}{ + "name": "marty", + "desc": "gophercon india", + } + err = idx.Index("a", doca) + if err != nil { + t.Error(err) + } + + docy := map[string]interface{}{ + "name": "jasper", + "desc": "clojure", + } + err = idx.Index("y", docy) + if err != nil { + t.Error(err) + } + + err = idx.Delete("y") + if err != nil { + t.Error(err) + } + + docx := map[string]interface{}{ + "name": "rose", + "desc": "googler", + } + err = idx.Index("x", docx) + if err != nil { + t.Error(err) + } + + err = idx.SetInternal([]byte("status"), []byte("pending")) + if err != nil { + t.Error(err) + } + + docb := map[string]interface{}{ + "name": "steve", + "desc": "cbft master", + } + batch := idx.NewBatch() + err = batch.Index("b", docb) + if err != nil { + t.Error(err) + } + batch.Delete("x") + batch.SetInternal([]byte("batchi"), []byte("batchv")) + batch.DeleteInternal([]byte("status")) + err = idx.Batch(batch) + if err != nil { + t.Error(err) + } + val, err := idx.GetInternal([]byte("batchi")) + if err != nil { + t.Error(err) + } + if string(val) != "batchv" { + t.Errorf("expected 'batchv', got '%s'", val) + } + val, err = idx.GetInternal([]byte("status")) + if err != nil { + t.Error(err) + } + if val != nil { + t.Errorf("expected nil, got '%s'", val) + } + + err = idx.SetInternal([]byte("seqno"), []byte("7")) + if err != nil { + t.Error(err) + } + err = idx.SetInternal([]byte("status"), []byte("ready")) + if err != nil { + t.Error(err) + } + err = idx.DeleteInternal([]byte("status")) + if err != nil { + t.Error(err) + } + val, err = idx.GetInternal([]byte("status")) + if err != nil { + t.Error(err) + } + if val != nil { + t.Errorf("expected nil, got '%s'", val) + } + + val, err = idx.GetInternal([]byte("seqno")) + if err != nil { + t.Error(err) + } + if string(val) != "7" { + t.Errorf("expected '7', got '%s'", val) + } + + // close the index, open it again, and try some more things + err = idx.Close() + if err != nil { + t.Fatal(err) + } + + idx, err = Open(tmpIndexPath) + if err != nil { + t.Fatal(err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + count, err := idx.DocCount() + if err != nil { + t.Fatal(err) + } + if count != 2 { + t.Errorf("expected doc count 2, got %d", count) + } + + doc, err := idx.Document("a") + if err != nil { + t.Fatal(err) + } + foundNameField := false + doc.VisitFields(func(field index.Field) { + if field.Name() == "name" && string(field.Value()) == "marty" { + foundNameField = true + } + }) + if !foundNameField { + t.Errorf("expected to find field named 'name' with value 'marty'") + } + + fields, err := idx.Fields() + if err != nil { + t.Fatal(err) + } + expectedFields := map[string]bool{ + "_all": false, + "name": false, + "desc": false, + } + if len(fields) < len(expectedFields) { + t.Fatalf("expected %d fields got %d", len(expectedFields), len(fields)) + } + for _, f := range fields { + expectedFields[f] = true + } + for ef, efp := range expectedFields { + if !efp { + t.Errorf("field %s is missing", ef) + } + } +} + +func approxSame(actual, expected uint64) bool { + modulus := func(a, b uint64) uint64 { + if a > b { + return a - b + } + return b - a + } + + return float64(modulus(actual, expected))/float64(expected) < float64(0.30) +} + +func checkStatsOnIndexedBatch(indexPath string, indexMapping mapping.IndexMapping, + expectedVal uint64, +) error { + var wg sync.WaitGroup + var statValError error + + idx, err := NewUsing(indexPath, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil) + if err != nil { + return err + } + + batch, err := getBatchFromData(idx, "sample-data.json") + if err != nil { + return fmt.Errorf("failed to form a batch %v\n", err) + } + wg.Add(1) + batch.SetPersistedCallback(func(e error) { + defer wg.Done() + stats, _ := idx.StatsMap()["index"].(map[string]interface{}) + bytesWritten, _ := stats["num_bytes_written_at_index_time"].(uint64) + if !approxSame(bytesWritten, expectedVal) { + statValError = fmt.Errorf("expected bytes written is %d, got %v", expectedVal, + bytesWritten) + } + }) + err = idx.Batch(batch) + if err != nil { + return fmt.Errorf("failed to index batch %v\n", err) + } + wg.Wait() + idx.Close() + + return statValError +} + +func TestBytesWritten(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + + indexMapping := NewIndexMapping() + indexMapping.TypeField = "type" + indexMapping.DefaultAnalyzer = "en" + documentMapping := NewDocumentMapping() + indexMapping.AddDocumentMapping("hotel", documentMapping) + + indexMapping.DocValuesDynamic = false + indexMapping.StoreDynamic = false + + contentFieldMapping := NewTextFieldMapping() + contentFieldMapping.Index = true + contentFieldMapping.Store = false + contentFieldMapping.IncludeInAll = false + contentFieldMapping.IncludeTermVectors = false + contentFieldMapping.DocValues = false + + reviewsMapping := NewDocumentMapping() + reviewsMapping.AddFieldMappingsAt("content", contentFieldMapping) + documentMapping.AddSubDocumentMapping("reviews", reviewsMapping) + + typeFieldMapping := NewTextFieldMapping() + typeFieldMapping.Store = false + typeFieldMapping.IncludeInAll = false + typeFieldMapping.IncludeTermVectors = false + typeFieldMapping.DocValues = false + documentMapping.AddFieldMappingsAt("type", typeFieldMapping) + + err = checkStatsOnIndexedBatch(tmpIndexPath, indexMapping, 57273) + if err != nil { + t.Fatal(err) + } + cleanupTmpIndexPath(t, tmpIndexPath) + + contentFieldMapping.Store = true + tmpIndexPath1 := createTmpIndexPath(t) + + err := checkStatsOnIndexedBatch(tmpIndexPath1, indexMapping, 76069) + if err != nil { + t.Fatal(err) + } + cleanupTmpIndexPath(t, tmpIndexPath1) + + contentFieldMapping.Store = false + contentFieldMapping.IncludeInAll = true + tmpIndexPath2 := createTmpIndexPath(t) + + err = checkStatsOnIndexedBatch(tmpIndexPath2, indexMapping, 68875) + if err != nil { + t.Fatal(err) + } + cleanupTmpIndexPath(t, tmpIndexPath2) + + contentFieldMapping.IncludeInAll = false + contentFieldMapping.IncludeTermVectors = true + tmpIndexPath3 := createTmpIndexPath(t) + + err = checkStatsOnIndexedBatch(tmpIndexPath3, indexMapping, 78985) + if err != nil { + t.Fatal(err) + } + cleanupTmpIndexPath(t, tmpIndexPath3) + + contentFieldMapping.IncludeTermVectors = false + contentFieldMapping.DocValues = true + tmpIndexPath4 := createTmpIndexPath(t) + + err = checkStatsOnIndexedBatch(tmpIndexPath4, indexMapping, 64228) + if err != nil { + t.Fatal(err) + } + cleanupTmpIndexPath(t, tmpIndexPath4) +} + +func createIndexMappingOnSampleData() *mapping.IndexMappingImpl { + indexMapping := NewIndexMapping() + indexMapping.TypeField = "type" + indexMapping.DefaultAnalyzer = "en" + indexMapping.ScoringModel = index.DefaultScoringModel + documentMapping := NewDocumentMapping() + indexMapping.AddDocumentMapping("hotel", documentMapping) + indexMapping.StoreDynamic = false + indexMapping.DocValuesDynamic = false + contentFieldMapping := NewTextFieldMapping() + contentFieldMapping.Store = false + + reviewsMapping := NewDocumentMapping() + reviewsMapping.AddFieldMappingsAt("content", contentFieldMapping) + documentMapping.AddSubDocumentMapping("reviews", reviewsMapping) + + typeFieldMapping := NewTextFieldMapping() + typeFieldMapping.Store = false + documentMapping.AddFieldMappingsAt("type", typeFieldMapping) + + return indexMapping +} + +func TestBM25TFIDFScoring(t *testing.T) { + tmpIndexPath1 := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath1) + tmpIndexPath2 := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath2) + + indexMapping := createIndexMappingOnSampleData() + indexMapping.ScoringModel = index.BM25Scoring + indexBM25, err := NewUsing(tmpIndexPath1, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil) + if err != nil { + t.Fatal(err) + } + + indexMapping1 := createIndexMappingOnSampleData() + indexTFIDF, err := NewUsing(tmpIndexPath2, indexMapping1, Config.DefaultIndexType, Config.DefaultMemKVStore, nil) + if err != nil { + t.Fatal(err) + } + + defer func() { + err := indexBM25.Close() + if err != nil { + t.Fatal(err) + } + + err = indexTFIDF.Close() + if err != nil { + t.Fatal(err) + } + }() + + batch, err := getBatchFromData(indexBM25, "sample-data.json") + if err != nil { + t.Fatalf("failed to form a batch") + } + err = indexBM25.Batch(batch) + if err != nil { + t.Fatalf("failed to index batch %v\n", err) + } + query := NewMatchQuery("Hotel") + query.FieldVal = "name" + searchRequest := NewSearchRequestOptions(query, int(10), 0, true) + + resBM25, err := indexBM25.Search(searchRequest) + if err != nil { + t.Error(err) + } + + batch, err = getBatchFromData(indexTFIDF, "sample-data.json") + if err != nil { + t.Fatalf("failed to form a batch") + } + err = indexTFIDF.Batch(batch) + if err != nil { + t.Fatalf("failed to index batch %v\n", err) + } + + resTFIDF, err := indexTFIDF.Search(searchRequest) + if err != nil { + t.Error(err) + } + + for i, hit := range resTFIDF.Hits { + if hit.Score < resBM25.Hits[i].Score { + t.Fatalf("expected the score to be higher for BM25, got %v and %v", + resBM25.Hits[i].Score, hit.Score) + } + } +} + +func TestBM25GlobalScoring(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + indexMapping := createIndexMappingOnSampleData() + indexMapping.ScoringModel = index.BM25Scoring + idxSinglePartition, err := NewUsing(tmpIndexPath, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil) + if err != nil { + t.Fatal(err) + } + + defer func() { + err := idxSinglePartition.Close() + if err != nil { + t.Fatal(err) + } + }() + + batch, err := getBatchFromData(idxSinglePartition, "sample-data.json") + if err != nil { + t.Fatalf("failed to form a batch") + } + err = idxSinglePartition.Batch(batch) + if err != nil { + t.Fatalf("failed to index batch %v\n", err) + } + query := NewMatchQuery("Hotel") + query.FieldVal = "name" + searchRequest := NewSearchRequestOptions(query, int(10), 0, true) + + res, err := idxSinglePartition.Search(searchRequest) + if err != nil { + t.Error(err) + } + + singlePartHits := res.Hits + + dataset, _ := readDataFromFile("sample-data.json") + tmpIndexPath1 := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath1) + + idxPart1, err := NewUsing(tmpIndexPath1, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil) + if err != nil { + t.Fatal(err) + } + + defer func() { + err := idxPart1.Close() + if err != nil { + t.Fatal(err) + } + }() + + batch1 := idxPart1.NewBatch() + for _, doc := range dataset[:len(dataset)/2] { + err = batch1.Index(fmt.Sprintf("%d", doc["id"]), doc) + if err != nil { + t.Fatal(err) + } + } + err = idxPart1.Batch(batch1) + if err != nil { + t.Fatal(err) + } + + tmpIndexPath2 := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath2) + + idxPart2, err := NewUsing(tmpIndexPath2, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil) + if err != nil { + t.Fatal(err) + } + + defer func() { + err := idxPart2.Close() + if err != nil { + t.Fatal(err) + } + }() + + batch2 := idxPart2.NewBatch() + for _, doc := range dataset[len(dataset)/2:] { + err = batch2.Index(fmt.Sprintf("%d", doc["id"]), doc) + if err != nil { + t.Fatal(err) + } + } + err = idxPart2.Batch(batch2) + if err != nil { + t.Fatal(err) + } + + multiPartIndex := NewIndexAlias(idxPart1, idxPart2) + err = multiPartIndex.SetIndexMapping(indexMapping) + if err != nil { + t.Fatal(err) + } + + ctx := context.Background() + // this key is set to ensure that we have a consistent scoring at the index alias + // level (it forces a pre search phase which can have a small overhead) + ctx = context.WithValue(ctx, search.SearchTypeKey, search.GlobalScoring) + + res, err = multiPartIndex.SearchInContext(ctx, searchRequest) + if err != nil { + t.Error(err) + } + + for i, hit := range res.Hits { + if hit.Score != singlePartHits[i].Score { + t.Fatalf("expected the scores to be the same, got %v and %v", + hit.Score, singlePartHits[i].Score) + } + } +} + +func TestBytesRead(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + indexMapping := NewIndexMapping() + indexMapping.TypeField = "type" + indexMapping.DefaultAnalyzer = "en" + documentMapping := NewDocumentMapping() + indexMapping.AddDocumentMapping("hotel", documentMapping) + indexMapping.StoreDynamic = false + indexMapping.DocValuesDynamic = false + contentFieldMapping := NewTextFieldMapping() + contentFieldMapping.Store = false + + reviewsMapping := NewDocumentMapping() + reviewsMapping.AddFieldMappingsAt("content", contentFieldMapping) + documentMapping.AddSubDocumentMapping("reviews", reviewsMapping) + + typeFieldMapping := NewTextFieldMapping() + typeFieldMapping.Store = false + documentMapping.AddFieldMappingsAt("type", typeFieldMapping) + + idx, err := NewUsing(tmpIndexPath, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil) + if err != nil { + t.Fatal(err) + } + + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + batch, err := getBatchFromData(idx, "sample-data.json") + if err != nil { + t.Fatalf("failed to form a batch") + } + err = idx.Batch(batch) + if err != nil { + t.Fatalf("failed to index batch %v\n", err) + } + query := NewQueryStringQuery("united") + searchRequest := NewSearchRequestOptions(query, int(10), 0, true) + + res, err := idx.Search(searchRequest) + if err != nil { + t.Error(err) + } + stats, _ := idx.StatsMap()["index"].(map[string]interface{}) + prevBytesRead, _ := stats["num_bytes_read_at_query_time"].(uint64) + + expectedBytesRead := uint64(21574) + if supportForVectorSearch { + expectedBytesRead = 21984 + } + + if prevBytesRead != expectedBytesRead && res.Cost == prevBytesRead { + t.Fatalf("expected bytes read for query string %v, got %v", + expectedBytesRead, prevBytesRead) + } + + // subsequent queries on the same field results in lesser amount + // of bytes read because the segment static and dictionary is reused and not + // loaded from mmap'd filed + res, err = idx.Search(searchRequest) + if err != nil { + t.Error(err) + } + stats, _ = idx.StatsMap()["index"].(map[string]interface{}) + bytesRead, _ := stats["num_bytes_read_at_query_time"].(uint64) + if bytesRead-prevBytesRead != 66 && res.Cost == bytesRead-prevBytesRead { + t.Fatalf("expected bytes read for query string 66, got %v", + bytesRead-prevBytesRead) + } + prevBytesRead = bytesRead + + fuzz := NewFuzzyQuery("hotel") + fuzz.FieldVal = "reviews.content" + fuzz.Fuzziness = 2 + searchRequest = NewSearchRequest(fuzz) + res, err = idx.Search(searchRequest) + if err != nil { + t.Error(err) + } + stats, _ = idx.StatsMap()["index"].(map[string]interface{}) + bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) + if bytesRead-prevBytesRead != 8468 && res.Cost == bytesRead-prevBytesRead { + t.Fatalf("expected bytes read for fuzzy query is 8468, got %v", + bytesRead-prevBytesRead) + } + prevBytesRead = bytesRead + + typeFacet := NewFacetRequest("type", 2) + query = NewQueryStringQuery("united") + searchRequest = NewSearchRequestOptions(query, int(0), 0, true) + searchRequest.AddFacet("types", typeFacet) + res, err = idx.Search(searchRequest) + if err != nil { + t.Error(err) + } + + stats, _ = idx.StatsMap()["index"].(map[string]interface{}) + bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) + if !approxSame(bytesRead-prevBytesRead, 196) && res.Cost == bytesRead-prevBytesRead { + t.Fatalf("expected bytes read for faceted query is around 196, got %v", + bytesRead-prevBytesRead) + } + prevBytesRead = bytesRead + + min := float64(8660) + max := float64(8665) + numRangeQuery := NewNumericRangeQuery(&min, &max) + numRangeQuery.FieldVal = "id" + searchRequest = NewSearchRequestOptions(numRangeQuery, int(10), 0, true) + res, err = idx.Search(searchRequest) + if err != nil { + t.Error(err) + } + + stats, _ = idx.StatsMap()["index"].(map[string]interface{}) + bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) + if bytesRead-prevBytesRead != 924 && res.Cost == bytesRead-prevBytesRead { + t.Fatalf("expected bytes read for numeric range query is 924, got %v", + bytesRead-prevBytesRead) + } + prevBytesRead = bytesRead + + searchRequest = NewSearchRequestOptions(query, int(10), 0, true) + searchRequest.Highlight = &HighlightRequest{} + res, err = idx.Search(searchRequest) + if err != nil { + t.Error(err) + } + + stats, _ = idx.StatsMap()["index"].(map[string]interface{}) + bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) + if bytesRead-prevBytesRead != 105 && res.Cost == bytesRead-prevBytesRead { + t.Fatalf("expected bytes read for query with highlighter is 105, got %v", + bytesRead-prevBytesRead) + } + prevBytesRead = bytesRead + + disQuery := NewDisjunctionQuery(NewMatchQuery("hotel"), NewMatchQuery("united")) + searchRequest = NewSearchRequestOptions(disQuery, int(10), 0, true) + res, err = idx.Search(searchRequest) + if err != nil { + t.Error(err) + } + // expectation is that the bytes read is roughly equal to sum of sub queries in + // the disjunction query plus the segment loading portion for the second subquery + // since it's created afresh and not reused + stats, _ = idx.StatsMap()["index"].(map[string]interface{}) + bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) + if bytesRead-prevBytesRead != 120 && res.Cost == bytesRead-prevBytesRead { + t.Fatalf("expected bytes read for disjunction query is 120, got %v", + bytesRead-prevBytesRead) + } +} + +func TestBytesReadStored(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + indexMapping := NewIndexMapping() + indexMapping.TypeField = "type" + indexMapping.DefaultAnalyzer = "en" + documentMapping := NewDocumentMapping() + indexMapping.AddDocumentMapping("hotel", documentMapping) + + indexMapping.DocValuesDynamic = false + indexMapping.StoreDynamic = false + + contentFieldMapping := NewTextFieldMapping() + contentFieldMapping.Store = true + contentFieldMapping.IncludeInAll = false + contentFieldMapping.IncludeTermVectors = false + + reviewsMapping := NewDocumentMapping() + reviewsMapping.AddFieldMappingsAt("content", contentFieldMapping) + documentMapping.AddSubDocumentMapping("reviews", reviewsMapping) + + typeFieldMapping := NewTextFieldMapping() + typeFieldMapping.Store = false + typeFieldMapping.IncludeInAll = false + typeFieldMapping.IncludeTermVectors = false + documentMapping.AddFieldMappingsAt("type", typeFieldMapping) + idx, err := NewUsing(tmpIndexPath, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil) + if err != nil { + t.Fatal(err) + } + batch, err := getBatchFromData(idx, "sample-data.json") + if err != nil { + t.Fatalf("failed to form a batch %v\n", err) + } + err = idx.Batch(batch) + if err != nil { + t.Fatalf("failed to index batch %v\n", err) + } + query := NewTermQuery("hotel") + query.FieldVal = "reviews.content" + searchRequest := NewSearchRequestOptions(query, int(10), 0, true) + res, err := idx.Search(searchRequest) + if err != nil { + t.Error(err) + } + + stats, _ := idx.StatsMap()["index"].(map[string]interface{}) + bytesRead, _ := stats["num_bytes_read_at_query_time"].(uint64) + + expectedBytesRead := uint64(11435) + if supportForVectorSearch { + expectedBytesRead = 11845 + } + + if bytesRead != expectedBytesRead && bytesRead == res.Cost { + t.Fatalf("expected the bytes read stat to be around %v, got %v", expectedBytesRead, bytesRead) + } + prevBytesRead := bytesRead + + searchRequest = NewSearchRequestOptions(query, int(10), 0, true) + res, err = idx.Search(searchRequest) + if err != nil { + t.Error(err) + } + stats, _ = idx.StatsMap()["index"].(map[string]interface{}) + bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) + if bytesRead-prevBytesRead != 48 && bytesRead-prevBytesRead == res.Cost { + t.Fatalf("expected the bytes read stat to be around 48, got %v", bytesRead-prevBytesRead) + } + prevBytesRead = bytesRead + + searchRequest = NewSearchRequestOptions(query, int(10), 0, true) + searchRequest.Fields = []string{"*"} + res, err = idx.Search(searchRequest) + if err != nil { + t.Error(err) + } + + stats, _ = idx.StatsMap()["index"].(map[string]interface{}) + bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) + + if bytesRead-prevBytesRead != 26511 && bytesRead-prevBytesRead == res.Cost { + t.Fatalf("expected the bytes read stat to be around 26511, got %v", + bytesRead-prevBytesRead) + } + idx.Close() + cleanupTmpIndexPath(t, tmpIndexPath) + + // same type of querying but on field "type" + contentFieldMapping.Store = false + typeFieldMapping.Store = true + + tmpIndexPath1 := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath1) + + idx1, err := NewUsing(tmpIndexPath1, indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil) + if err != nil { + t.Fatal(err) + } + defer func() { + err := idx1.Close() + if err != nil { + t.Fatal(err) + } + }() + + batch, err = getBatchFromData(idx1, "sample-data.json") + if err != nil { + t.Fatalf("failed to form a batch %v\n", err) + } + err = idx1.Batch(batch) + if err != nil { + t.Fatalf("failed to index batch %v\n", err) + } + + query = NewTermQuery("hotel") + query.FieldVal = "type" + searchRequest = NewSearchRequestOptions(query, int(10), 0, true) + res, err = idx1.Search(searchRequest) + if err != nil { + t.Error(err) + } + + stats, _ = idx1.StatsMap()["index"].(map[string]interface{}) + bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) + + expectedBytesRead = uint64(3622) + if supportForVectorSearch { + expectedBytesRead = 4032 + } + + if bytesRead != expectedBytesRead && bytesRead == res.Cost { + t.Fatalf("expected the bytes read stat to be around %v, got %v", expectedBytesRead, bytesRead) + } + prevBytesRead = bytesRead + + res, err = idx1.Search(searchRequest) + if err != nil { + t.Error(err) + } + stats, _ = idx1.StatsMap()["index"].(map[string]interface{}) + bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) + if bytesRead-prevBytesRead != 47 && bytesRead-prevBytesRead == res.Cost { + t.Fatalf("expected the bytes read stat to be around 47, got %v", bytesRead-prevBytesRead) + } + prevBytesRead = bytesRead + + searchRequest.Fields = []string{"*"} + res, err = idx1.Search(searchRequest) + if err != nil { + t.Error(err) + } + + stats, _ = idx1.StatsMap()["index"].(map[string]interface{}) + bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) + if bytesRead-prevBytesRead != 77 && bytesRead-prevBytesRead == res.Cost { + t.Fatalf("expected the bytes read stat to be around 77, got %v", bytesRead-prevBytesRead) + } +} + +func readDataFromFile(fileName string) ([]map[string]interface{}, error) { + pwd, err := os.Getwd() + if err != nil { + return nil, err + } + path := filepath.Join(pwd, "data", "test", fileName) + + var dataset []map[string]interface{} + fileContent, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + err = json.Unmarshal(fileContent, &dataset) + if err != nil { + return nil, err + } + + return dataset, nil +} + +func getBatchFromData(idx Index, fileName string) (*Batch, error) { + dataset, err := readDataFromFile(fileName) + batch := idx.NewBatch() + for _, doc := range dataset { + err = batch.Index(fmt.Sprintf("%d", doc["id"]), doc) + if err != nil { + return nil, err + } + } + + return batch, err +} + +func TestIndexCreateNewOverExisting(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + + index, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + err = index.Close() + if err != nil { + t.Fatal(err) + } + + _, err = New(tmpIndexPath, NewIndexMapping()) + if err != ErrorIndexPathExists { + t.Fatalf("expected error index path exists, got %v", err) + } +} + +func TestIndexOpenNonExisting(t *testing.T) { + _, err := Open("doesnotexist") + if err != ErrorIndexPathDoesNotExist { + t.Fatalf("expected error index path does not exist, got %v", err) + } +} + +func TestIndexOpenMetaMissingOrCorrupt(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + index, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + err = index.Close() + if err != nil { + t.Fatal(err) + } + + tmpIndexPathMeta := filepath.Join(tmpIndexPath, "index_meta.json") + + // now intentionally change the storage type + err = os.WriteFile(tmpIndexPathMeta, []byte(`{"storage":"mystery"}`), 0o666) + if err != nil { + t.Fatal(err) + } + + _, err = Open(tmpIndexPath) + if err == nil { + t.Fatalf("expected error for unknown storage type, got %v", err) + } + + // now intentionally corrupt the metadata + err = os.WriteFile(tmpIndexPathMeta, []byte("corrupted"), 0o666) + if err != nil { + t.Fatal(err) + } + + _, err = Open(tmpIndexPath) + if err != ErrorIndexMetaCorrupt { + t.Fatalf("expected error index metadata corrupted, got %v", err) + } + + // now intentionally remove the metadata + err = os.Remove(tmpIndexPathMeta) + if err != nil { + t.Fatal(err) + } + + _, err = Open(tmpIndexPath) + if err != ErrorIndexMetaMissing { + t.Fatalf("expected error index metadata missing, got %v", err) + } +} + +func TestInMemIndex(t *testing.T) { + index, err := NewMemOnly(NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + err = index.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestClosedIndex(t *testing.T) { + index, err := NewMemOnly(NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + err = index.Close() + if err != nil { + t.Fatal(err) + } + + err = index.Index("test", "test") + if err != ErrorIndexClosed { + t.Errorf("expected error index closed, got %v", err) + } + + err = index.Delete("test") + if err != ErrorIndexClosed { + t.Errorf("expected error index closed, got %v", err) + } + + b := index.NewBatch() + err = index.Batch(b) + if err != ErrorIndexClosed { + t.Errorf("expected error index closed, got %v", err) + } + + _, err = index.Document("test") + if err != ErrorIndexClosed { + t.Errorf("expected error index closed, got %v", err) + } + + _, err = index.DocCount() + if err != ErrorIndexClosed { + t.Errorf("expected error index closed, got %v", err) + } + + _, err = index.Search(NewSearchRequest(NewTermQuery("test"))) + if err != ErrorIndexClosed { + t.Errorf("expected error index closed, got %v", err) + } + + _, err = index.Fields() + if err != ErrorIndexClosed { + t.Errorf("expected error index closed, got %v", err) + } +} + +type slowQuery struct { + actual query.Query + delay time.Duration +} + +func (s *slowQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + time.Sleep(s.delay) + return s.actual.Searcher(ctx, i, m, options) +} + +func TestSlowSearch(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + defer func() { + // reset logger back to normal + SetLog(log.New(io.Discard, "bleve", log.LstdFlags)) + }() + // set custom logger + var sdw sawDataWriter + SetLog(log.New(&sdw, "bleve", log.LstdFlags)) + + index, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + }() + + Config.SlowSearchLogThreshold = 1 * time.Minute + + query := NewTermQuery("water") + req := NewSearchRequest(query) + _, err = index.Search(req) + if err != nil { + t.Fatal(err) + } + + if sdw.sawData { + t.Errorf("expected to not see slow query logged, but did") + } + + sq := &slowQuery{ + actual: query, + delay: 50 * time.Millisecond, // on Windows timer resolution is 15ms + } + req.Query = sq + Config.SlowSearchLogThreshold = 1 * time.Microsecond + _, err = index.Search(req) + if err != nil { + t.Fatal(err) + } + + if !sdw.sawData { + t.Errorf("expected to see slow query logged, but didn't") + } +} + +type sawDataWriter struct { + sawData bool +} + +func (s *sawDataWriter) Write(p []byte) (n int, err error) { + s.sawData = true + return len(p), nil +} + +func TestStoredFieldPreserved(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + index, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + }() + + doca := map[string]interface{}{ + "name": "Marty", + "desc": "GopherCON India", + "bool": true, + "num": float64(1), + } + err = index.Index("a", doca) + if err != nil { + t.Error(err) + } + + q := NewTermQuery("marty") + req := NewSearchRequest(q) + req.Fields = []string{"name", "desc", "bool", "num"} + res, err := index.Search(req) + if err != nil { + t.Error(err) + } + + if len(res.Hits) != 1 { + t.Fatalf("expected 1 hit, got %d", len(res.Hits)) + } + if res.Hits[0].Fields["name"] != "Marty" { + t.Errorf("expected 'Marty' got '%s'", res.Hits[0].Fields["name"]) + } + if res.Hits[0].Fields["desc"] != "GopherCON India" { + t.Errorf("expected 'GopherCON India' got '%s'", res.Hits[0].Fields["desc"]) + } + if res.Hits[0].Fields["num"] != float64(1) { + t.Errorf("expected '1' got '%v'", res.Hits[0].Fields["num"]) + } + if res.Hits[0].Fields["bool"] != true { + t.Errorf("expected 'true' got '%v'", res.Hits[0].Fields["bool"]) + } +} + +func TestDict(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + index, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + + doca := map[string]interface{}{ + "name": "marty", + "desc": "gophercon india", + } + err = index.Index("a", doca) + if err != nil { + t.Error(err) + } + + docy := map[string]interface{}{ + "name": "jasper", + "desc": "clojure", + } + err = index.Index("y", docy) + if err != nil { + t.Error(err) + } + + docx := map[string]interface{}{ + "name": "rose", + "desc": "googler", + } + err = index.Index("x", docx) + if err != nil { + t.Error(err) + } + + dict, err := index.FieldDict("name") + if err != nil { + t.Error(err) + } + + terms := []string{} + de, err := dict.Next() + for err == nil && de != nil { + terms = append(terms, string(de.Term)) + de, err = dict.Next() + } + + expectedTerms := []string{"jasper", "marty", "rose"} + if !reflect.DeepEqual(terms, expectedTerms) { + t.Errorf("expected %v, got %v", expectedTerms, terms) + } + + err = dict.Close() + if err != nil { + t.Fatal(err) + } + + // test start and end range + dict, err = index.FieldDictRange("name", []byte("marty"), []byte("rose")) + if err != nil { + t.Error(err) + } + + terms = []string{} + de, err = dict.Next() + for err == nil && de != nil { + terms = append(terms, string(de.Term)) + de, err = dict.Next() + } + + expectedTerms = []string{"marty", "rose"} + if !reflect.DeepEqual(terms, expectedTerms) { + t.Errorf("expected %v, got %v", expectedTerms, terms) + } + + err = dict.Close() + if err != nil { + t.Fatal(err) + } + + docz := map[string]interface{}{ + "name": "prefix", + "desc": "bob cat cats catting dog doggy zoo", + } + err = index.Index("z", docz) + if err != nil { + t.Error(err) + } + + dict, err = index.FieldDictPrefix("desc", []byte("cat")) + if err != nil { + t.Error(err) + } + + terms = []string{} + de, err = dict.Next() + for err == nil && de != nil { + terms = append(terms, string(de.Term)) + de, err = dict.Next() + } + + expectedTerms = []string{"cat", "cats", "catting"} + if !reflect.DeepEqual(terms, expectedTerms) { + t.Errorf("expected %v, got %v", expectedTerms, terms) + } + + stats := index.Stats() + if stats == nil { + t.Errorf("expected IndexStat, got nil") + } + + err = dict.Close() + if err != nil { + t.Fatal(err) + } + + err = index.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestBatchString(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + index, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + }() + + batch := index.NewBatch() + err = batch.Index("a", []byte("{}")) + if err != nil { + t.Fatal(err) + } + batch.Delete("b") + batch.SetInternal([]byte("c"), []byte{}) + batch.DeleteInternal([]byte("d")) + + batchStr := batch.String() + if !strings.HasPrefix(batchStr, "Batch (2 ops, 2 internal ops)") { + t.Errorf("expected to start with Batch (2 ops, 2 internal ops), did not") + } + if !strings.Contains(batchStr, "INDEX - 'a'") { + t.Errorf("expected to contain INDEX - 'a', did not") + } + if !strings.Contains(batchStr, "DELETE - 'b'") { + t.Errorf("expected to contain DELETE - 'b', did not") + } + if !strings.Contains(batchStr, "SET INTERNAL - 'c'") { + t.Errorf("expected to contain SET INTERNAL - 'c', did not") + } + if !strings.Contains(batchStr, "DELETE INTERNAL - 'd'") { + t.Errorf("expected to contain DELETE INTERNAL - 'd', did not") + } +} + +func TestIndexMetadataRaceBug198(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + index, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + }() + + wg := sync.WaitGroup{} + wg.Add(1) + done := make(chan struct{}) + go func() { + for { + select { + case <-done: + wg.Done() + return + default: + _, err2 := index.DocCount() + if err2 != nil { + t.Error(err2) + wg.Done() + return + } + } + } + }() + + for i := 0; i < 100; i++ { + batch := index.NewBatch() + err = batch.Index("a", []byte("{}")) + if err != nil { + t.Fatal(err) + } + err = index.Batch(batch) + if err != nil { + t.Fatal(err) + } + } + + close(done) + wg.Wait() +} + +func TestSortMatchSearch(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + index, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + + names := []string{"Noam", "Uri", "David", "Yosef", "Eitan", "Itay", "Ariel", "Daniel", "Omer", "Yogev", "Yehonatan", "Moshe", "Mohammed", "Yusuf", "Omar"} + days := []string{"Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"} + numbers := []string{"One", "Two", "Three", "Four", "Five", "Six", "Seven", "Eight", "Nine", "Ten", "Eleven", "Twelve"} + b := index.NewBatch() + for i := 0; i < 200; i++ { + doc := make(map[string]interface{}) + doc["Name"] = names[i%len(names)] + doc["Day"] = days[i%len(days)] + doc["Number"] = numbers[i%len(numbers)] + err = b.Index(fmt.Sprintf("%d", i), doc) + if err != nil { + t.Fatal(err) + } + } + err = index.Batch(b) + if err != nil { + t.Fatal(err) + } + + req := NewSearchRequest(NewMatchQuery("One")) + req.SortBy([]string{"Day", "Name"}) + req.Fields = []string{"*"} + sr, err := index.Search(req) + if err != nil { + t.Fatal(err) + } + prev := "" + for _, hit := range sr.Hits { + val := hit.Fields["Day"].(string) + if prev > val { + t.Errorf("Hits must be sorted by 'Day'. Found '%s' before '%s'", prev, val) + } + prev = val + } + err = index.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestIndexCountMatchSearch(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + index, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func(i int) { + b := index.NewBatch() + for j := 0; j < 200; j++ { + id := fmt.Sprintf("%d", (i*200)+j) + + doc := struct { + Body string + }{ + Body: "match", + } + + err := b.Index(id, doc) + if err != nil { + t.Error(err) + wg.Done() + return + } + } + + err := index.Batch(b) + if err != nil { + t.Error(err) + wg.Done() + return + } + + wg.Done() + }(i) + } + wg.Wait() + + // search for something that should match all documents + sr, err := index.Search(NewSearchRequest(NewMatchQuery("match"))) + if err != nil { + t.Fatal(err) + } + + // get the index document count + dc, err := index.DocCount() + if err != nil { + t.Fatal(err) + } + + // make sure test is working correctly, doc count should 2000 + if dc != 2000 { + t.Errorf("expected doc count 2000, got %d", dc) + } + + // make sure our search found all the documents + if dc != sr.Total { + t.Errorf("expected search result total %d to match doc count %d", sr.Total, dc) + } + + err = index.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestBatchReset(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + index, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + + batch := index.NewBatch() + err = batch.Index("k1", struct { + Body string + }{ + Body: "v1", + }) + if err != nil { + t.Error(err) + } + batch.Delete("k2") + batch.SetInternal([]byte("k3"), []byte("v3")) + batch.DeleteInternal([]byte("k4")) + + if batch.Size() != 4 { + t.Logf("%v", batch) + t.Errorf("expected batch size 4, got %d", batch.Size()) + } + + batch.Reset() + + if batch.Size() != 0 { + t.Errorf("expected batch size 0 after reset, got %d", batch.Size()) + } + + err = index.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestDocumentFieldArrayPositions(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + idx, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + + // index a document with an array of strings + err = idx.Index("k", struct { + Messages []string + }{ + Messages: []string{ + "first", + "second", + "third", + "last", + }, + }) + if err != nil { + t.Fatal(err) + } + + // load the document + doc, err := idx.Document("k") + if err != nil { + t.Fatal(err) + } + + doc.VisitFields(func(f index.Field) { + if reflect.DeepEqual(f.Value(), []byte("first")) { + ap := f.ArrayPositions() + if len(ap) < 1 { + t.Errorf("expected an array position, got none") + return + } + if ap[0] != 0 { + t.Errorf("expected 'first' in array position 0, got %d", ap[0]) + } + } + if reflect.DeepEqual(f.Value(), []byte("second")) { + ap := f.ArrayPositions() + if len(ap) < 1 { + t.Errorf("expected an array position, got none") + return + } + if ap[0] != 1 { + t.Errorf("expected 'second' in array position 1, got %d", ap[0]) + } + } + if reflect.DeepEqual(f.Value(), []byte("third")) { + ap := f.ArrayPositions() + if len(ap) < 1 { + t.Errorf("expected an array position, got none") + return + } + if ap[0] != 2 { + t.Errorf("expected 'third' in array position 2, got %d", ap[0]) + } + } + if reflect.DeepEqual(f.Value(), []byte("last")) { + ap := f.ArrayPositions() + if len(ap) < 1 { + t.Errorf("expected an array position, got none") + return + } + if ap[0] != 3 { + t.Errorf("expected 'last' in array position 3, got %d", ap[0]) + } + } + }) + + // now index a document in the same field with a single string + err = idx.Index("k2", struct { + Messages string + }{ + Messages: "only", + }) + if err != nil { + t.Fatal(err) + } + + // load the document + doc, err = idx.Document("k2") + if err != nil { + t.Fatal(err) + } + + doc.VisitFields(func(f index.Field) { + if reflect.DeepEqual(f.Value(), []byte("only")) { + ap := f.ArrayPositions() + if len(ap) != 0 { + t.Errorf("expected no array positions, got %d", len(ap)) + return + } + } + }) + + err = idx.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestKeywordSearchBug207(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + f := NewTextFieldMapping() + f.Analyzer = keyword.Name + + m := NewIndexMapping() + m.DefaultMapping = NewDocumentMapping() + m.DefaultMapping.AddFieldMappingsAt("Body", f) + + index, err := New(tmpIndexPath, m) + if err != nil { + t.Fatal(err) + } + + doc1 := struct { + Body string + }{ + Body: "a555c3bb06f7a127cda000005", + } + + err = index.Index("a", doc1) + if err != nil { + t.Fatal(err) + } + + doc2 := struct { + Body string + }{ + Body: "555c3bb06f7a127cda000005", + } + + err = index.Index("b", doc2) + if err != nil { + t.Fatal(err) + } + + // now search for these terms + sreq := NewSearchRequest(NewTermQuery("a555c3bb06f7a127cda000005")) + sres, err := index.Search(sreq) + if err != nil { + t.Fatal(err) + } + if sres.Total != 1 { + t.Errorf("expected 1 result, got %d", sres.Total) + } + if sres.Hits[0].ID != "a" { + t.Errorf("expecated id 'a', got '%s'", sres.Hits[0].ID) + } + + sreq = NewSearchRequest(NewTermQuery("555c3bb06f7a127cda000005")) + sres, err = index.Search(sreq) + if err != nil { + t.Fatal(err) + } + if sres.Total != 1 { + t.Errorf("expected 1 result, got %d", sres.Total) + } + if sres.Hits[0].ID != "b" { + t.Errorf("expecated id 'b', got '%s'", sres.Hits[0].ID) + } + + // now do the same searches using query strings + sreq = NewSearchRequest(NewQueryStringQuery("Body:a555c3bb06f7a127cda000005")) + sres, err = index.Search(sreq) + if err != nil { + t.Fatal(err) + } + if sres.Total != 1 { + t.Errorf("expected 1 result, got %d", sres.Total) + } + if sres.Hits[0].ID != "a" { + t.Errorf("expecated id 'a', got '%s'", sres.Hits[0].ID) + } + + sreq = NewSearchRequest(NewQueryStringQuery(`Body:555c3bb06f7a127cda000005`)) + sres, err = index.Search(sreq) + if err != nil { + t.Fatal(err) + } + if sres.Total != 1 { + t.Errorf("expected 1 result, got %d", sres.Total) + } + if sres.Hits[0].ID != "b" { + t.Errorf("expecated id 'b', got '%s'", sres.Hits[0].ID) + } + + err = index.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestTermVectorArrayPositions(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + index, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + + // index a document with an array of strings + err = index.Index("k", struct { + Messages []string + }{ + Messages: []string{ + "first", + "second", + "third", + "last", + }, + }) + if err != nil { + t.Fatal(err) + } + + // search for this document in all field + tq := NewTermQuery("second") + tsr := NewSearchRequest(tq) + tsr.IncludeLocations = true + results, err := index.Search(tsr) + if err != nil { + t.Fatal(err) + } + if results.Total != 1 { + t.Fatalf("expected 1 result, got %d", results.Total) + } + if len(results.Hits[0].Locations["Messages"]["second"]) < 1 { + t.Fatalf("expected at least one location") + } + if len(results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions) < 1 { + t.Fatalf("expected at least one location array position") + } + if results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions[0] != 1 { + t.Fatalf("expected array position 1, got %d", results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions[0]) + } + + // repeat search for this document in Messages field + tq2 := NewTermQuery("third") + tq2.SetField("Messages") + tsr = NewSearchRequest(tq2) + tsr.IncludeLocations = true + results, err = index.Search(tsr) + if err != nil { + t.Fatal(err) + } + if results.Total != 1 { + t.Fatalf("expected 1 result, got %d", results.Total) + } + if len(results.Hits[0].Locations["Messages"]["third"]) < 1 { + t.Fatalf("expected at least one location") + } + if len(results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions) < 1 { + t.Fatalf("expected at least one location array position") + } + if results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions[0] != 2 { + t.Fatalf("expected array position 2, got %d", results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions[0]) + } + + err = index.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestDocumentStaticMapping(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + m := NewIndexMapping() + m.DefaultMapping = NewDocumentStaticMapping() + m.DefaultMapping.AddFieldMappingsAt("Text", NewTextFieldMapping()) + m.DefaultMapping.AddFieldMappingsAt("Date", NewDateTimeFieldMapping()) + m.DefaultMapping.AddFieldMappingsAt("Numeric", NewNumericFieldMapping()) + + index, err := New(tmpIndexPath, m) + if err != nil { + t.Fatal(err) + } + + doc1 := struct { + Text string + IgnoredText string + Numeric float64 + IgnoredNumeric float64 + Date time.Time + IgnoredDate time.Time + }{ + Text: "valid text", + IgnoredText: "ignored text", + Numeric: 10, + IgnoredNumeric: 20, + Date: time.Unix(1, 0), + IgnoredDate: time.Unix(2, 0), + } + + err = index.Index("a", doc1) + if err != nil { + t.Fatal(err) + } + + fields, err := index.Fields() + if err != nil { + t.Fatal(err) + } + sort.Strings(fields) + expectedFields := []string{"Date", "Numeric", "Text", "_all"} + if len(fields) < len(expectedFields) { + t.Fatalf("invalid field count: %d", len(fields)) + } + for i, expected := range expectedFields { + if expected != fields[i] { + t.Fatalf("unexpected field[%d]: %s", i, fields[i]) + } + } + + err = index.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestIndexEmptyDocId(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + index, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + }() + + doc := map[string]interface{}{ + "body": "nodocid", + } + + err = index.Index("", doc) + if err != ErrorEmptyID { + t.Errorf("expect index empty doc id to fail") + } + + err = index.Delete("") + if err != ErrorEmptyID { + t.Errorf("expect delete empty doc id to fail") + } + + batch := index.NewBatch() + err = batch.Index("", doc) + if err != ErrorEmptyID { + t.Errorf("expect index empty doc id in batch to fail") + } + + batch.Delete("") + if batch.Size() > 0 { + t.Errorf("expect delete empty doc id in batch to be ignored") + } +} + +func TestDateTimeFieldMappingIssue287(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + f := NewDateTimeFieldMapping() + + m := NewIndexMapping() + m.DefaultMapping = NewDocumentMapping() + m.DefaultMapping.AddFieldMappingsAt("Date", f) + + index, err := New(tmpIndexPath, m) + if err != nil { + t.Fatal(err) + } + + type doc struct { + Date time.Time + } + + now := time.Now() + + // 3hr ago to 1hr ago + for i := 0; i < 3; i++ { + d := doc{now.Add(time.Duration((i - 3)) * time.Hour)} + + err = index.Index(strconv.FormatInt(int64(i), 10), d) + if err != nil { + t.Fatal(err) + } + } + + // search range across all docs + start := now.Add(-4 * time.Hour) + end := now + sreq := NewSearchRequest(NewDateRangeQuery(start, end)) + sres, err := index.Search(sreq) + if err != nil { + t.Fatal(err) + } + if sres.Total != 3 { + t.Errorf("expected 3 results, got %d", sres.Total) + } + + // search range includes only oldest + start = now.Add(-4 * time.Hour) + end = now.Add(-121 * time.Minute) + sreq = NewSearchRequest(NewDateRangeQuery(start, end)) + sres, err = index.Search(sreq) + if err != nil { + t.Fatal(err) + } + if sres.Total != 1 { + t.Errorf("expected 1 results, got %d", sres.Total) + } + if sres.Total > 0 && sres.Hits[0].ID != "0" { + t.Errorf("expecated id '0', got '%s'", sres.Hits[0].ID) + } + + // search range includes only newest + start = now.Add(-61 * time.Minute) + end = now + sreq = NewSearchRequest(NewDateRangeQuery(start, end)) + sres, err = index.Search(sreq) + if err != nil { + t.Fatal(err) + } + if sres.Total != 1 { + t.Errorf("expected 1 results, got %d", sres.Total) + } + if sres.Total > 0 && sres.Hits[0].ID != "2" { + t.Errorf("expecated id '2', got '%s'", sres.Hits[0].ID) + } + + err = index.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestDocumentFieldArrayPositionsBug295(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + index, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + + // index a document with an array of strings + err = index.Index("k", struct { + Messages []string + Another string + MoreData []string + }{ + Messages: []string{ + "bleve", + "bleve", + }, + Another: "text", + MoreData: []string{ + "a", + "b", + "c", + "bleve", + }, + }) + if err != nil { + t.Fatal(err) + } + + // search for it in the messages field + tq := NewTermQuery("bleve") + tq.SetField("Messages") + tsr := NewSearchRequest(tq) + tsr.IncludeLocations = true + results, err := index.Search(tsr) + if err != nil { + t.Fatal(err) + } + if results.Total != 1 { + t.Fatalf("expected 1 result, got %d", results.Total) + } + if len(results.Hits[0].Locations["Messages"]["bleve"]) != 2 { + t.Fatalf("expected 2 locations of 'bleve', got %d", len(results.Hits[0].Locations["Messages"]["bleve"])) + } + if results.Hits[0].Locations["Messages"]["bleve"][0].ArrayPositions[0] != 0 { + t.Errorf("expected array position to be 0") + } + if results.Hits[0].Locations["Messages"]["bleve"][1].ArrayPositions[0] != 1 { + t.Errorf("expected array position to be 1") + } + + // search for it in all + tq = NewTermQuery("bleve") + tsr = NewSearchRequest(tq) + tsr.IncludeLocations = true + results, err = index.Search(tsr) + if err != nil { + t.Fatal(err) + } + if results.Total != 1 { + t.Fatalf("expected 1 result, got %d", results.Total) + } + if len(results.Hits[0].Locations["Messages"]["bleve"]) != 2 { + t.Fatalf("expected 2 locations of 'bleve', got %d", len(results.Hits[0].Locations["Messages"]["bleve"])) + } + if results.Hits[0].Locations["Messages"]["bleve"][0].ArrayPositions[0] != 0 { + t.Errorf("expected array position to be 0") + } + if results.Hits[0].Locations["Messages"]["bleve"][1].ArrayPositions[0] != 1 { + t.Errorf("expected array position to be 1") + } + + err = index.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestBooleanFieldMappingIssue109(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + m := NewIndexMapping() + m.DefaultMapping = NewDocumentMapping() + m.DefaultMapping.AddFieldMappingsAt("Bool", NewBooleanFieldMapping()) + + index, err := New(tmpIndexPath, m) + if err != nil { + t.Fatal(err) + } + + type doc struct { + Bool bool + } + err = index.Index("true", &doc{Bool: true}) + if err != nil { + t.Fatal(err) + } + err = index.Index("false", &doc{Bool: false}) + if err != nil { + t.Fatal(err) + } + + q := NewBoolFieldQuery(true) + q.SetField("Bool") + sreq := NewSearchRequest(q) + sres, err := index.Search(sreq) + if err != nil { + t.Fatal(err) + } + if sres.Total != 1 { + t.Errorf("expected 1 results, got %d", sres.Total) + } + + q = NewBoolFieldQuery(false) + q.SetField("Bool") + sreq = NewSearchRequest(q) + sres, err = index.Search(sreq) + if err != nil { + t.Fatal(err) + } + if sres.Total != 1 { + t.Errorf("expected 1 results, got %d", sres.Total) + } + + sreq = NewSearchRequest(NewBoolFieldQuery(true)) + sres, err = index.Search(sreq) + if err != nil { + t.Fatal(err) + } + if sres.Total != 1 { + t.Errorf("expected 1 results, got %d", sres.Total) + } + + err = index.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestSearchTimeout(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + index, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + }() + + // first run a search with an absurdly long timeout (should succeed) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + query := NewTermQuery("water") + req := NewSearchRequest(query) + _, err = index.SearchInContext(ctx, req) + if err != nil { + t.Fatal(err) + } + + // now run a search again with an absurdly low timeout (should timeout) + ctx, cancel = context.WithTimeout(context.Background(), 1*time.Microsecond) + defer cancel() + sq := &slowQuery{ + actual: query, + delay: 50 * time.Millisecond, // on Windows timer resolution is 15ms + } + req.Query = sq + _, err = index.SearchInContext(ctx, req) + if err != context.DeadlineExceeded { + t.Fatalf("expected %v, got: %v", context.DeadlineExceeded, err) + } + + // now run a search with a long timeout, but with a long query, and cancel it + ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) + sq = &slowQuery{ + actual: query, + delay: 100 * time.Millisecond, // on Windows timer resolution is 15ms + } + req = NewSearchRequest(sq) + cancel() + _, err = index.SearchInContext(ctx, req) + if err != context.Canceled { + t.Fatalf("expected %v, got: %v", context.Canceled, err) + } +} + +// TestConfigCache exposes a concurrent map write with go 1.6 +func TestConfigCache(t *testing.T) { + for i := 0; i < 100; i++ { + go func() { + _, err := Config.Cache.HighlighterNamed(Config.DefaultHighlighter) + if err != nil { + t.Error(err) + } + }() + } +} + +func TestBatchRaceBug260(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + i, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + defer func() { + err := i.Close() + if err != nil { + t.Fatal(err) + } + }() + b := i.NewBatch() + err = b.Index("1", 1) + if err != nil { + t.Fatal(err) + } + err = i.Batch(b) + if err != nil { + t.Fatal(err) + } + b.Reset() + err = b.Index("2", 2) + if err != nil { + t.Fatal(err) + } + err = i.Batch(b) + if err != nil { + t.Fatal(err) + } + b.Reset() +} + +func BenchmarkBatchOverhead(b *testing.B) { + tmpIndexPath := createTmpIndexPath(b) + defer cleanupTmpIndexPath(b, tmpIndexPath) + m := NewIndexMapping() + i, err := NewUsing(tmpIndexPath, m, Config.DefaultIndexType, null.Name, nil) + if err != nil { + b.Fatal(err) + } + for n := 0; n < b.N; n++ { + // put 1000 items in a batch + batch := i.NewBatch() + for i := 0; i < 1000; i++ { + err = batch.Index(fmt.Sprintf("%d", i), map[string]interface{}{"name": "bleve"}) + if err != nil { + b.Fatal(err) + } + } + err = i.Batch(batch) + if err != nil { + b.Fatal(err) + } + batch.Reset() + } +} + +func TestOpenReadonlyMultiple(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + // build an index and close it + index, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + + doca := map[string]interface{}{ + "name": "marty", + "desc": "gophercon india", + } + err = index.Index("a", doca) + if err != nil { + t.Fatal(err) + } + + err = index.Close() + if err != nil { + t.Fatal(err) + } + + // now open it read-only + index, err = OpenUsing(tmpIndexPath, map[string]interface{}{ + "read_only": true, + }) + if err != nil { + t.Fatal(err) + } + + // now open it again + index2, err := OpenUsing(tmpIndexPath, map[string]interface{}{ + "read_only": true, + }) + if err != nil { + t.Fatal(err) + } + + err = index.Close() + if err != nil { + t.Fatal(err) + } + err = index2.Close() + if err != nil { + t.Fatal(err) + } +} + +// TestBug408 tests for VERY large values of size, even though actual result +// set may be reasonable size +func TestBug408(t *testing.T) { + type TestStruct struct { + ID string `json:"id"` + UserID *string `json:"user_id"` + } + + docMapping := NewDocumentMapping() + docMapping.AddFieldMappingsAt("id", NewTextFieldMapping()) + docMapping.AddFieldMappingsAt("user_id", NewTextFieldMapping()) + + indexMapping := NewIndexMapping() + indexMapping.DefaultMapping = docMapping + + index, err := NewMemOnly(indexMapping) + if err != nil { + t.Fatal(err) + } + + numToTest := 10 + matchUserID := "match" + noMatchUserID := "no_match" + matchingDocIds := make(map[string]struct{}) + + for i := 0; i < numToTest; i++ { + ds := &TestStruct{"id_" + strconv.Itoa(i), nil} + if i%2 == 0 { + ds.UserID = &noMatchUserID + } else { + ds.UserID = &matchUserID + matchingDocIds[ds.ID] = struct{}{} + } + err = index.Index(ds.ID, ds) + if err != nil { + t.Fatal(err) + } + } + + cnt, err := index.DocCount() + if err != nil { + t.Fatal(err) + } + if int(cnt) != numToTest { + t.Fatalf("expected %d documents in index, got %d", numToTest, cnt) + } + + q := NewTermQuery(matchUserID) + q.SetField("user_id") + searchReq := NewSearchRequestOptions(q, math.MaxInt32, 0, false) + results, err := index.Search(searchReq) + if err != nil { + t.Fatal(err) + } + if int(results.Total) != numToTest/2 { + t.Fatalf("expected %d search hits, got %d", numToTest/2, results.Total) + } + + for _, result := range results.Hits { + if _, found := matchingDocIds[result.ID]; !found { + t.Fatalf("document with ID %s not in results as expected", result.ID) + } + } +} + +func TestIndexAdvancedCountMatchSearch(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + index, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + + var wg sync.WaitGroup + errChan := make(chan error, 10) + for i := 0; i < 10; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + b := index.NewBatch() + for j := 0; j < 200; j++ { + id := fmt.Sprintf("%d", (i*200)+j) + + doc := document.NewDocument(id) + doc.AddField(document.NewTextField("body", []uint64{}, []byte("match"))) + doc.AddField(document.NewCompositeField("_all", true, []string{}, []string{})) + + err := b.IndexAdvanced(doc) + if err != nil { + errChan <- err + return + } + } + err := index.Batch(b) + if err != nil { + errChan <- err + return + } + }(i) + } + wg.Wait() + + close(errChan) + for err := range errChan { + if err != nil { + t.Fatal(err) + } + } + + // search for something that should match all documents + sr, err := index.Search(NewSearchRequest(NewMatchQuery("match"))) + if err != nil { + t.Fatal(err) + } + + // get the index document count + dc, err := index.DocCount() + if err != nil { + t.Fatal(err) + } + + // make sure test is working correctly, doc count should 2000 + if dc != 2000 { + t.Errorf("expected doc count 2000, got %d", dc) + } + + // make sure our search found all the documents + if dc != sr.Total { + t.Errorf("expected search result total %d to match doc count %d", sr.Total, dc) + } + + err = index.Close() + if err != nil { + t.Fatal(err) + } +} + +func benchmarkSearchOverhead(indexType string, b *testing.B) { + tmpIndexPath := createTmpIndexPath(b) + defer cleanupTmpIndexPath(b, tmpIndexPath) + + index, err := NewUsing(tmpIndexPath, NewIndexMapping(), + indexType, Config.DefaultKVStore, nil) + if err != nil { + b.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + b.Fatal(err) + } + }() + + elements := []string{"air", "water", "fire", "earth"} + for j := 0; j < 10000; j++ { + err = index.Index(fmt.Sprintf("%d", j), + map[string]interface{}{"name": elements[j%len(elements)]}) + if err != nil { + b.Fatal(err) + } + } + + query1 := NewTermQuery("water") + query2 := NewTermQuery("fire") + query := NewDisjunctionQuery(query1, query2) + req := NewSearchRequest(query) + + b.ResetTimer() + + for n := 0; n < b.N; n++ { + _, err = index.Search(req) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkUpsidedownSearchOverhead(b *testing.B) { + benchmarkSearchOverhead(upsidedown.Name, b) +} + +func BenchmarkScorchSearchOverhead(b *testing.B) { + benchmarkSearchOverhead(scorch.Name, b) +} + +func TestSearchQueryCallback(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + index, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + }() + + query := NewTermQuery("water") + req := NewSearchRequest(query) + + expErr := fmt.Errorf("MEM_LIMIT_EXCEEDED") + f := func(size uint64) error { + // the intended usage of this callback is to see the estimated + // memory usage before executing, and possibly abort early + // in this test we simulate returning such an error + return expErr + } + + ctx := context.WithValue(context.Background(), SearchQueryStartCallbackKey, SearchQueryStartCallbackFn(f)) + _, err = index.SearchInContext(ctx, req) + if err != expErr { + t.Fatalf("Expected: %v, Got: %v", expErr, err) + } +} + +func TestBatchMerge(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + idx, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + doca := map[string]interface{}{ + "name": "scorch", + "desc": "gophercon india", + "nation": "india", + } + + batchA := idx.NewBatch() + err = batchA.Index("a", doca) + if err != nil { + t.Error(err) + } + batchA.SetInternal([]byte("batchkA"), []byte("batchvA")) + + docb := map[string]interface{}{ + "name": "moss", + "desc": "gophercon MV", + } + + batchB := idx.NewBatch() + err = batchB.Index("b", docb) + if err != nil { + t.Error(err) + } + batchB.SetInternal([]byte("batchkB"), []byte("batchvB")) + + docC := map[string]interface{}{ + "name": "blahblah", + "desc": "inProgress", + "country": "usa", + } + + batchC := idx.NewBatch() + err = batchC.Index("c", docC) + if err != nil { + t.Error(err) + } + batchC.SetInternal([]byte("batchkC"), []byte("batchvC")) + batchC.SetInternal([]byte("batchkB"), []byte("batchvBNew")) + batchC.Delete("a") + batchC.DeleteInternal([]byte("batchkA")) + + batchA.Merge(batchB) + + if batchA.Size() != 4 { + t.Errorf("expected batch size 4, got %d", batchA.Size()) + } + + batchA.Merge(batchC) + + if batchA.Size() != 6 { + t.Errorf("expected batch size 6, got %d", batchA.Size()) + } + + err = idx.Batch(batchA) + if err != nil { + t.Fatal(err) + } + + // close the index, open it again, and try some more things + err = idx.Close() + if err != nil { + t.Fatal(err) + } + + idx, err = Open(tmpIndexPath) + if err != nil { + t.Fatal(err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + count, err := idx.DocCount() + if err != nil { + t.Fatal(err) + } + if count != 2 { + t.Errorf("expected doc count 2, got %d", count) + } + + doc, err := idx.Document("c") + if err != nil { + t.Fatal(err) + } + + val, err := idx.GetInternal([]byte("batchkB")) + if err != nil { + t.Fatal(err) + } + if val == nil || string(val) != "batchvBNew" { + t.Errorf("expected val: batchvBNew , got %s", val) + } + + val, err = idx.GetInternal([]byte("batchkA")) + if err != nil { + t.Fatal(err) + } + if val != nil { + t.Errorf("expected nil, got %s", val) + } + + foundNameField := false + doc.VisitFields(func(field index.Field) { + if field.Name() == "name" && string(field.Value()) == "blahblah" { + foundNameField = true + } + }) + if !foundNameField { + t.Errorf("expected to find field named 'name' with value 'blahblah'") + } + + fields, err := idx.Fields() + if err != nil { + t.Fatal(err) + } + + expectedFields := map[string]bool{ + "_all": false, + "name": false, + "desc": false, + "country": false, + } + if len(fields) < len(expectedFields) { + t.Fatalf("expected %d fields got %d", len(expectedFields), len(fields)) + } + + for _, f := range fields { + expectedFields[f] = true + } + + for ef, efp := range expectedFields { + if !efp { + t.Errorf("field %s is missing", ef) + } + } +} + +func TestBug1096(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + // use default mapping + mapping := NewIndexMapping() + + // create a scorch index with default SAFE batches + var idx Index + idx, err = NewUsing(tmpIndexPath, mapping, "scorch", "scorch", nil) + if err != nil { + log.Fatal(err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + // create a single batch instance that we will reuse + // this should be safe because we have single goroutine + // and we always wait for batch execution to finish + batch := idx.NewBatch() + + // number of batches to execute + for i := 0; i < 10; i++ { + + // number of documents to put into the batch + for j := 0; j < 91; j++ { + + // create a doc id 0-90 (important so that we get id's 9 and 90) + // this could duplicate something already in the index + // this too should be OK and update the item in the index + id := fmt.Sprintf("%d", j) + + err = batch.Index(id, map[string]interface{}{ + "name": id, + "batch": fmt.Sprintf("%d", i), + }) + if err != nil { + log.Fatal(err) + } + } + + // execute the batch + err = idx.Batch(batch) + if err != nil { + log.Fatal(err) + } + + // reset the batch before reusing it + batch.Reset() + } + + // search for docs having name starting with the number 9 + q := NewWildcardQuery("9*") + q.SetField("name") + req := NewSearchRequestOptions(q, 1000, 0, false) + req.Fields = []string{"*"} + var res *SearchResult + res, err = idx.Search(req) + if err != nil { + log.Fatal(err) + } + + // we expect only 2 hits, for docs 9 and 90 + if res.Total > 2 { + t.Fatalf("expected only 2 hits '9' and '90', got %v", res) + } +} + +func TestDataRaceBug1092(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + // use default mapping + mapping := NewIndexMapping() + + var idx Index + idx, err = NewUsing(tmpIndexPath, mapping, upsidedown.Name, boltdb.Name, nil) + if err != nil { + log.Fatal(err) + } + defer func() { + cerr := idx.Close() + if cerr != nil { + t.Fatal(cerr) + } + }() + + batch := idx.NewBatch() + for i := 0; i < 10; i++ { + err = idx.Batch(batch) + if err != nil { + t.Error(err) + } + + batch.Reset() + } +} + +func TestBatchRaceBug1149(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + i, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + defer func() { + err := i.Close() + if err != nil { + t.Fatal(err) + } + }() + testBatchRaceBug1149(t, i) +} + +func TestBatchRaceBug1149Scorch(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + i, err := NewUsing(tmpIndexPath, NewIndexMapping(), "scorch", "scorch", nil) + if err != nil { + t.Fatal(err) + } + defer func() { + err := i.Close() + if err != nil { + t.Fatal(err) + } + }() + testBatchRaceBug1149(t, i) +} + +func testBatchRaceBug1149(t *testing.T, i Index) { + b := i.NewBatch() + b.Delete("1") + err = i.Batch(b) + if err != nil { + t.Fatal(err) + } + b.Reset() + err = i.Batch(b) + if err != nil { + t.Fatal(err) + } + b.Reset() +} + +func TestOptimisedConjunctionSearchHits(t *testing.T) { + scorch.OptimizeDisjunctionUnadorned = false + defer func() { + scorch.OptimizeDisjunctionUnadorned = true + }() + + defer func() { + err := os.RemoveAll("testidx") + if err != nil { + t.Fatal(err) + } + }() + + idx, err := NewUsing("testidx", NewIndexMapping(), "scorch", "scorch", nil) + if err != nil { + t.Fatal(err) + } + doca := map[string]interface{}{ + "country": "united", + "name": "Mercure Hotel", + "directions": "B560 and B56 Follow signs to the M56", + } + docb := map[string]interface{}{ + "country": "united", + "name": "Mercure Altrincham Bowdon Hotel", + "directions": "A570 and A57 Follow signs to the M56 Manchester Airport", + } + docc := map[string]interface{}{ + "country": "india united", + "name": "Sonoma Hotel", + "directions": "Northwest", + } + docd := map[string]interface{}{ + "country": "United Kingdom", + "name": "Cresta Court Hotel", + "directions": "junction of A560 and A56", + } + + b := idx.NewBatch() + err = b.Index("a", doca) + if err != nil { + t.Error(err) + } + err = b.Index("b", docb) + if err != nil { + t.Error(err) + } + err = b.Index("c", docc) + if err != nil { + t.Error(err) + } + err = b.Index("d", docd) + if err != nil { + t.Error(err) + } + // execute the batch + err = idx.Batch(b) + if err != nil { + log.Fatal(err) + } + + mq := NewMatchQuery("united") + mq.SetField("country") + + cq := NewConjunctionQuery(mq) + + mq1 := NewMatchQuery("hotel") + mq1.SetField("name") + cq.AddQuery(mq1) + + mq2 := NewMatchQuery("56") + mq2.SetField("directions") + mq2.SetFuzziness(1) + cq.AddQuery(mq2) + + req := NewSearchRequest(cq) + req.Score = "none" + + res, err := idx.Search(req) + if err != nil { + t.Fatal(err) + } + hitsWithOutScore := res.Total + + req = NewSearchRequest(cq) + req.Score = "" + + res, err = idx.Search(req) + if err != nil { + t.Fatal(err) + } + hitsWithScore := res.Total + + if hitsWithOutScore != hitsWithScore { + t.Errorf("expected %d hits without score, got %d", hitsWithScore, hitsWithOutScore) + } + + err = idx.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestIndexMappingDocValuesDynamic(t *testing.T) { + im := NewIndexMapping() + // DocValuesDynamic's default is true + // Now explicitly set it to false + im.DocValuesDynamic = false + + // Next, retrieve the JSON dump of the index mapping + var data []byte + data, err = json.Marshal(im) + if err != nil { + t.Fatal(err) + } + + // Now, edit an unrelated setting in the index mapping + var m map[string]interface{} + err = json.Unmarshal(data, &m) + if err != nil { + t.Fatal(err) + } + m["index_dynamic"] = false + data, err = json.Marshal(m) + if err != nil { + t.Fatal(err) + } + + // Unmarshal back the changes into the index mapping struct + if err = im.UnmarshalJSON(data); err != nil { + t.Fatal(err) + } + + // Expect DocValuesDynamic to remain false! + if im.DocValuesDynamic { + t.Fatalf("Expected DocValuesDynamic to remain false after the index mapping edit") + } +} + +func TestCopyIndex(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + idx, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + doca := map[string]interface{}{ + "name": "tester", + "desc": "gophercon india testing", + } + err = idx.Index("a", doca) + if err != nil { + t.Error(err) + } + + docy := map[string]interface{}{ + "name": "jasper", + "desc": "clojure", + } + err = idx.Index("y", docy) + if err != nil { + t.Error(err) + } + + err = idx.Delete("y") + if err != nil { + t.Error(err) + } + + docx := map[string]interface{}{ + "name": "rose", + "desc": "xoogler", + } + err = idx.Index("x", docx) + if err != nil { + t.Error(err) + } + + err = idx.SetInternal([]byte("status"), []byte("pending")) + if err != nil { + t.Error(err) + } + + docb := map[string]interface{}{ + "name": "sree", + "desc": "cbft janitor", + } + batch := idx.NewBatch() + err = batch.Index("b", docb) + if err != nil { + t.Error(err) + } + batch.Delete("x") + batch.SetInternal([]byte("batchi"), []byte("batchv")) + batch.DeleteInternal([]byte("status")) + err = idx.Batch(batch) + if err != nil { + t.Error(err) + } + val, err := idx.GetInternal([]byte("batchi")) + if err != nil { + t.Error(err) + } + if string(val) != "batchv" { + t.Errorf("expected 'batchv', got '%s'", val) + } + val, err = idx.GetInternal([]byte("status")) + if err != nil { + t.Error(err) + } + if val != nil { + t.Errorf("expected nil, got '%s'", val) + } + + err = idx.SetInternal([]byte("seqno"), []byte("7")) + if err != nil { + t.Error(err) + } + err = idx.SetInternal([]byte("status"), []byte("ready")) + if err != nil { + t.Error(err) + } + err = idx.DeleteInternal([]byte("status")) + if err != nil { + t.Error(err) + } + val, err = idx.GetInternal([]byte("status")) + if err != nil { + t.Error(err) + } + if val != nil { + t.Errorf("expected nil, got '%s'", val) + } + + val, err = idx.GetInternal([]byte("seqno")) + if err != nil { + t.Error(err) + } + if string(val) != "7" { + t.Errorf("expected '7', got '%s'", val) + } + + count, err := idx.DocCount() + if err != nil { + t.Fatal(err) + } + if count != 2 { + t.Errorf("expected doc count 2, got %d", count) + } + + doc, err := idx.Document("a") + if err != nil { + t.Fatal(err) + } + foundNameField := false + doc.VisitFields(func(field index.Field) { + if field.Name() == "name" && string(field.Value()) == "tester" { + foundNameField = true + } + }) + if !foundNameField { + t.Errorf("expected to find field named 'name' with value 'tester'") + } + + fields, err := idx.Fields() + if err != nil { + t.Fatal(err) + } + expectedFields := map[string]bool{ + "_all": false, + "name": false, + "desc": false, + } + if len(fields) < len(expectedFields) { + t.Fatalf("expected %d fields got %d", len(expectedFields), len(fields)) + } + for _, f := range fields { + expectedFields[f] = true + } + for ef, efp := range expectedFields { + if !efp { + t.Errorf("field %s is missing", ef) + } + } + + // now create a copy of the index, and repeat assertions on it + copyableIndex, ok := idx.(IndexCopyable) + if !ok { + t.Fatal("index doesn't support copy") + } + backupIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, backupIndexPath) + + err = copyableIndex.CopyTo(FileSystemDirectory(backupIndexPath)) + if err != nil { + t.Fatalf("error copying the index: %v", err) + } + + // open the copied index + idxCopied, err := Open(backupIndexPath) + if err != nil { + t.Fatalf("unable to open copy index") + } + defer func() { + err := idxCopied.Close() + if err != nil { + t.Fatalf("error closing copy index: %v", err) + } + }() + + // assertions on copied index + copyCount, err := idxCopied.DocCount() + if err != nil { + t.Fatal(err) + } + if copyCount != 2 { + t.Errorf("expected doc count 2, got %d", copyCount) + } + + copyDoc, err := idxCopied.Document("a") + if err != nil { + t.Fatal(err) + } + copyFoundNameField := false + copyDoc.VisitFields(func(field index.Field) { + if field.Name() == "name" && string(field.Value()) == "tester" { + copyFoundNameField = true + } + }) + if !copyFoundNameField { + t.Errorf("expected copy index to find field named 'name' with value 'tester'") + } + + copyFields, err := idx.Fields() + if err != nil { + t.Fatal(err) + } + copyExpectedFields := map[string]bool{ + "_all": false, + "name": false, + "desc": false, + } + if len(copyFields) < len(copyExpectedFields) { + t.Fatalf("expected %d fields got %d", len(copyExpectedFields), len(copyFields)) + } + for _, f := range copyFields { + copyExpectedFields[f] = true + } + for ef, efp := range copyExpectedFields { + if !efp { + t.Errorf("copy field %s is missing", ef) + } + } +} + +func TestFuzzyScoring(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + mp := NewIndexMapping() + mp.DefaultAnalyzer = "simple" + idx, err := New(tmpIndexPath, mp) + if err != nil { + t.Fatal(err) + } + + batch := idx.NewBatch() + + docs := []map[string]interface{}{ + { + "textField": "ab", + }, + { + "textField": "abc", + }, + { + "textField": "abcd", + }, + } + + for _, doc := range docs { + err := batch.Index(fmt.Sprintf("%v", doc["textField"]), doc) + if err != nil { + t.Fatal(err) + } + } + + err = idx.Batch(batch) + if err != nil { + t.Fatal(err) + } + + query := NewFuzzyQuery("ab") + query.Fuzziness = 2 + searchRequest := NewSearchRequestOptions(query, 10, 0, true) + res, err := idx.Search(searchRequest) + if err != nil { + t.Error(err) + } + + maxScore := res.Hits[0].Score + + for i, hit := range res.Hits { + if maxScore/float64(i+1) != hit.Score { + t.Errorf("expected score - %f, got score - %f", maxScore/float64(i+1), hit.Score) + } + } + + err = idx.Close() + if err != nil { + t.Fatal(err) + } +} diff --git a/index_update.go b/index_update.go new file mode 100644 index 0000000..f16b640 --- /dev/null +++ b/index_update.go @@ -0,0 +1,596 @@ +// Copyright (c) 2025 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "fmt" + "reflect" + + "github.com/blevesearch/bleve/v2/mapping" + index "github.com/blevesearch/bleve_index_api" +) + +// Store all the fields that interact with the data +// from a document path +type pathInfo struct { + fieldMapInfo []*fieldMapInfo + dynamic bool + path string + analyser string + parentPath string +} + +// Store the field information with respect to the +// document paths +type fieldMapInfo struct { + fieldMapping *mapping.FieldMapping + analyzer string + datetimeParser string + rootName string + parent *pathInfo +} + +// Compare two index mappings to identify all of the updatable changes +func DeletedFields(ori, upd *mapping.IndexMappingImpl) (map[string]*index.UpdateFieldInfo, error) { + // Compare all of the top level fields in an index mapping + err := compareMappings(ori, upd) + if err != nil { + return nil, err + } + + // Check for new mappings present in the type mappings + // of the updated compared to the original + for name, updDMapping := range upd.TypeMapping { + err = checkUpdatedMapping(ori.TypeMapping[name], updDMapping) + if err != nil { + return nil, err + } + } + + // Check for new mappings present in the default mappings + // of the updated compared to the original + err = checkUpdatedMapping(ori.DefaultMapping, upd.DefaultMapping) + if err != nil { + return nil, err + } + + oriPaths := make(map[string]*pathInfo) + updPaths := make(map[string]*pathInfo) + + // Go through each mapping present in the original + // and consolidate according to the document paths + for name, oriDMapping := range ori.TypeMapping { + addPathInfo(oriPaths, "", oriDMapping, ori, nil, name) + } + addPathInfo(oriPaths, "", ori.DefaultMapping, ori, nil, "") + + // Go through each mapping present in the updated + // and consolidate according to the document paths + for name, updDMapping := range upd.TypeMapping { + addPathInfo(updPaths, "", updDMapping, upd, nil, name) + } + addPathInfo(updPaths, "", upd.DefaultMapping, upd, nil, "") + + // Compare all components of custom analysis currently in use + err = compareCustomComponents(oriPaths, updPaths, ori, upd) + if err != nil { + return nil, err + } + + // Compare both the mappings based on the document paths + // and create a list of index, docvalues, store differences + // for every single field possible + fieldInfo := make(map[string]*index.UpdateFieldInfo, len(oriPaths)) + for path, info := range oriPaths { + err = addFieldInfo(fieldInfo, info, updPaths[path]) + if err != nil { + return nil, err + } + } + + // Remove entries from the list with no changes between the + // original and the updated mapping + for name, info := range fieldInfo { + if !info.Deleted && !info.Index && !info.DocValues && !info.Store { + delete(fieldInfo, name) + } + // A field cannot be completely deleted with any dynamic value turned on + if info.Deleted { + if upd.IndexDynamic { + return nil, fmt.Errorf("mapping cannot be removed when index dynamic is true") + } + if upd.StoreDynamic { + return nil, fmt.Errorf("mapping cannot be removed when store dynamic is true") + } + if upd.DocValuesDynamic { + return nil, fmt.Errorf("mapping cannot be removed when docvalues dynamic is true") + } + } + } + return fieldInfo, nil +} + +// Ensures none of the top level index mapping fields have changed +func compareMappings(ori, upd *mapping.IndexMappingImpl) error { + if ori.TypeField != upd.TypeField && + (len(ori.TypeMapping) != 0 || len(upd.TypeMapping) != 0) { + return fmt.Errorf("type field cannot be changed when type mappings are present") + } + + if ori.DefaultType != upd.DefaultType { + return fmt.Errorf("default type cannot be changed") + } + + if ori.IndexDynamic != upd.IndexDynamic { + return fmt.Errorf("index dynamic cannot be changed") + } + + if ori.StoreDynamic != upd.StoreDynamic { + return fmt.Errorf("store dynamic cannot be changed") + } + + if ori.DocValuesDynamic != upd.DocValuesDynamic { + return fmt.Errorf("docvalues dynamic cannot be changed") + } + + if ori.DefaultAnalyzer != upd.DefaultAnalyzer && upd.IndexDynamic { + return fmt.Errorf("default analyser cannot be changed if index dynamic is true") + } + + if ori.DefaultDateTimeParser != upd.DefaultDateTimeParser && upd.IndexDynamic { + return fmt.Errorf("default datetime parser cannot be changed if index dynamic is true") + } + + // Scoring model changes between "", "tf-idf" and "bm25" require no index changes to be made + if ori.ScoringModel != upd.ScoringModel { + if ori.ScoringModel != "" && ori.ScoringModel != index.TFIDFScoring && ori.ScoringModel != index.BM25Scoring || + upd.ScoringModel != "" && upd.ScoringModel != index.TFIDFScoring && upd.ScoringModel != index.BM25Scoring { + return fmt.Errorf("scoring model can only be changed between \"\", %q and %q", index.TFIDFScoring, index.BM25Scoring) + } + } + + return nil +} + +// Ensures updated document mapping does not contain new +// field mappings or document mappings +func checkUpdatedMapping(ori, upd *mapping.DocumentMapping) error { + // Check to verify both original and updated are not nil + // and are enabled before proceeding + if ori == nil { + if upd == nil || !upd.Enabled { + return nil + } + return fmt.Errorf("updated index mapping contains new properties") + } + + if upd == nil || !upd.Enabled { + return nil + } + + if ori.Nested != upd.Nested { + return fmt.Errorf("nested property cannot be changed") + } + + var err error + // Recursively go through the child mappings + for name, updDMapping := range upd.Properties { + err = checkUpdatedMapping(ori.Properties[name], updDMapping) + if err != nil { + return err + } + } + + // Simple checks to ensure no new field mappings present + // in updated + // Create a map of original field names for O(1) lookup + oriFieldNames := make(map[string]bool, len(ori.Fields)) + for _, fMapping := range ori.Fields { + oriFieldNames[fMapping.Name] = true + } + + for _, updFMapping := range upd.Fields { + if !oriFieldNames[updFMapping.Name] { + return fmt.Errorf("updated index mapping contains new fields") + } + } + + return nil +} + +// Adds all of the field mappings while maintaining a tree of the document structure +// to ensure traversal and verification is possible incase of multiple mappings defined +// for a single field or multiple document fields' data getting written to a single zapx field +func addPathInfo(paths map[string]*pathInfo, name string, mp *mapping.DocumentMapping, + im *mapping.IndexMappingImpl, parent *pathInfo, rootName string) { + // Early exit if mapping has been disabled + // Comparisions later on will be done with a nil object + if !mp.Enabled { + return + } + + // Consolidate path information like index dynamic across multiple + // mappings if path is the same + var pInfo *pathInfo + if val, ok := paths[name]; ok { + pInfo = val + } else { + pInfo = &pathInfo{ + fieldMapInfo: make([]*fieldMapInfo, 0), + } + pInfo.dynamic = mp.Dynamic && im.IndexDynamic + pInfo.analyser = im.AnalyzerNameForPath(name) + } + + pInfo.dynamic = (pInfo.dynamic || mp.Dynamic) && im.IndexDynamic + pInfo.path = name + if parent != nil { + pInfo.parentPath = parent.path + } + + // Recursively add path information for all child mappings + for cName, cMapping := range mp.Properties { + pathName := cName + if name != "" { + pathName = name + "." + cName + } + addPathInfo(paths, pathName, cMapping, im, pInfo, rootName) + } + + // Add field mapping information keeping the document structure intact + for _, fMap := range mp.Fields { + fieldMapInfo := &fieldMapInfo{ + fieldMapping: fMap, + rootName: rootName, + parent: pInfo, + } + pInfo.fieldMapInfo = append(pInfo.fieldMapInfo, fieldMapInfo) + } + + paths[name] = pInfo +} + +// Compares all of the custom analysis components in use +func compareCustomComponents(oriPaths, updPaths map[string]*pathInfo, ori, upd *mapping.IndexMappingImpl) error { + // Compare all analysers currently in use + err := compareAnalysers(oriPaths, updPaths, ori, upd) + if err != nil { + return err + } + + // Compare all datetime parsers currently in use + err = compareDateTimeParsers(oriPaths, updPaths, ori, upd) + if err != nil { + return err + } + + // Compare all synonum sources + err = compareSynonymSources(ori, upd) + if err != nil { + return err + } + + // Compare all char filters, tokenizers, token filters and token maps + err = compareAnalyserSubcomponents(ori, upd) + if err != nil { + return err + } + + return nil +} + +// Compares all analysers currently in use +// Standard analysers not in custom analysis are not compared +// Analysers in custom analysis but not in use are not compared +func compareAnalysers(oriPaths, updPaths map[string]*pathInfo, ori, upd *mapping.IndexMappingImpl) error { + oriAnalyzers := make(map[string]interface{}) + updAnalyzers := make(map[string]interface{}) + + extractAnalyzers := func(paths map[string]*pathInfo, customAnalyzers map[string]map[string]interface{}, + analyzers map[string]interface{}, indexMapping *mapping.IndexMappingImpl) { + for path, info := range paths { + for _, fInfo := range info.fieldMapInfo { + if fInfo.fieldMapping.Type == "text" { + analyzerName := indexMapping.AnalyzerNameForPath(path) + fInfo.analyzer = analyzerName + if val, ok := customAnalyzers[analyzerName]; ok { + analyzers[analyzerName] = val + } + } + } + } + } + + extractAnalyzers(oriPaths, ori.CustomAnalysis.Analyzers, oriAnalyzers, ori) + extractAnalyzers(updPaths, upd.CustomAnalysis.Analyzers, updAnalyzers, upd) + + for name, anUpd := range updAnalyzers { + if anOri, ok := oriAnalyzers[name]; ok { + if !reflect.DeepEqual(anUpd, anOri) { + return fmt.Errorf("analyser %s changed while being used by fields", name) + } + } else { + return fmt.Errorf("analyser %s newly added to an existing field", name) + } + } + + return nil +} + +// Compares all date time parsers currently in use +// Date time parsers in custom analysis but not in use are not compared +func compareDateTimeParsers(oriPaths, updPaths map[string]*pathInfo, ori, upd *mapping.IndexMappingImpl) error { + oriDateTimeParsers := make(map[string]interface{}) + updDateTimeParsers := make(map[string]interface{}) + + extractDateTimeParsers := func(paths map[string]*pathInfo, customParsers map[string]map[string]interface{}, + parsers map[string]interface{}, indexMapping *mapping.IndexMappingImpl) { + for _, info := range paths { + for _, fInfo := range info.fieldMapInfo { + if fInfo.fieldMapping.Type == "datetime" { + parserName := fInfo.fieldMapping.DateFormat + if parserName == "" { + parserName = indexMapping.DefaultDateTimeParser + } + fInfo.datetimeParser = parserName + if val, ok := customParsers[parserName]; ok { + parsers[parserName] = val + } + } + } + } + } + + extractDateTimeParsers(oriPaths, ori.CustomAnalysis.DateTimeParsers, oriDateTimeParsers, ori) + extractDateTimeParsers(updPaths, upd.CustomAnalysis.DateTimeParsers, updDateTimeParsers, upd) + + for name, dtUpd := range updDateTimeParsers { + if dtOri, ok := oriDateTimeParsers[name]; ok { + if !reflect.DeepEqual(dtUpd, dtOri) { + return fmt.Errorf("datetime parser %s changed while being used by fields", name) + } + } else { + return fmt.Errorf("datetime parser %s added to an existing field", name) + } + } + + return nil +} + +// Compares all synonym sources +// Synonym sources currently not in use are also compared +func compareSynonymSources(ori, upd *mapping.IndexMappingImpl) error { + if !reflect.DeepEqual(ori.CustomAnalysis.SynonymSources, upd.CustomAnalysis.SynonymSources) { + return fmt.Errorf("synonym sources cannot be changed") + } + + return nil +} + +// Compares all char filters, tokenizers, token filters and token maps +// Components not currently in use are also compared +func compareAnalyserSubcomponents(ori, upd *mapping.IndexMappingImpl) error { + if !reflect.DeepEqual(ori.CustomAnalysis.CharFilters, upd.CustomAnalysis.CharFilters) { + return fmt.Errorf("char filters cannot be changed") + } + + if !reflect.DeepEqual(ori.CustomAnalysis.TokenFilters, upd.CustomAnalysis.TokenFilters) { + return fmt.Errorf("token filters cannot be changed") + } + + if !reflect.DeepEqual(ori.CustomAnalysis.TokenMaps, upd.CustomAnalysis.TokenMaps) { + return fmt.Errorf("token maps cannot be changed") + } + + if !reflect.DeepEqual(ori.CustomAnalysis.Tokenizers, upd.CustomAnalysis.Tokenizers) { + return fmt.Errorf("tokenizers cannot be changed") + } + + return nil +} + +// Compare all of the fields at a particular document path and add its field information +func addFieldInfo(fInfo map[string]*index.UpdateFieldInfo, ori, upd *pathInfo) error { + var info *index.UpdateFieldInfo + var err error + + // Assume deleted or disabled mapping if upd is nil. Checks for ori being nil + // or upd having mappings not in orihave already been done before this stage + if upd == nil { + for _, oriFMapInfo := range ori.fieldMapInfo { + info, err = compareFieldMapping(oriFMapInfo.fieldMapping, nil) + if err != nil { + return err + } + err = validateFieldInfo(info, fInfo, ori, oriFMapInfo) + if err != nil { + return err + } + } + } else { + if upd.dynamic && ori.analyser != upd.analyser { + return fmt.Errorf("analyser has been changed for a dynamic mapping") + } + for _, oriFMapInfo := range ori.fieldMapInfo { + var updFMap *mapping.FieldMapping + var updAnalyser string + var updDatetimeParser string + + // For multiple fields at a single document path, compare + // only with the matching ones + for _, updFMapInfo := range upd.fieldMapInfo { + if oriFMapInfo.rootName == updFMapInfo.rootName && + oriFMapInfo.fieldMapping.Name == updFMapInfo.fieldMapping.Name { + updFMap = updFMapInfo.fieldMapping + if updFMap.Type == "text" { + updAnalyser = updFMapInfo.analyzer + } else if updFMap.Type == "datetime" { + updDatetimeParser = updFMapInfo.datetimeParser + } + } + } + // Compare analyser, datetime parser and synonym source before comparing + // the field mapping as it might not have this information + if updAnalyser != "" && oriFMapInfo.analyzer != updAnalyser { + return fmt.Errorf("analyser has been changed for a text field") + } + if updDatetimeParser != "" && oriFMapInfo.datetimeParser != updDatetimeParser { + return fmt.Errorf("datetime parser has been changed for a date time field") + } + info, err = compareFieldMapping(oriFMapInfo.fieldMapping, updFMap) + if err != nil { + return err + } + + // Validate to ensure change is possible + // Needed if multiple mappings are aliased to the same field + err = validateFieldInfo(info, fInfo, ori, oriFMapInfo) + if err != nil { + return err + } + } + } + + return nil +} + +// Compares two field mappings against each other, checking for changes in index, store, doc values +// and complete deletiion of the mapping while noting that the changes made are doable based on +// other values like includeInAll and dynamic +// first return argument gives an empty fieldInfo if no changes detected +// second return argument gives a flag indicating whether any changes, if detected, are doable or if +// update is impossible +// third argument is an error explaining exactly why the change is not possible +func compareFieldMapping(original, updated *mapping.FieldMapping) (*index.UpdateFieldInfo, error) { + rv := &index.UpdateFieldInfo{} + + if updated == nil { + if original != nil && !original.IncludeInAll { + rv.Deleted = true + return rv, nil + } else if original == nil { + return nil, fmt.Errorf("both field mappings cannot be nil") + } + return nil, fmt.Errorf("deleted field present in '_all' field") + } else if original == nil { + return nil, fmt.Errorf("matching field not found in original index mapping") + } + + if original.Type != updated.Type { + return nil, fmt.Errorf("field type cannot be updated") + } + if original.Type == "text" { + if original.Analyzer != updated.Analyzer { + return nil, fmt.Errorf("analyzer cannot be updated for text fields") + } + } + if original.Type == "datetime" { + if original.DateFormat != updated.DateFormat { + return nil, fmt.Errorf("dateFormat cannot be updated for datetime fields") + } + } + if original.Type == "vector" || original.Type == "vector_base64" { + if original.Dims != updated.Dims { + return nil, fmt.Errorf("dimensions cannot be updated for vector and vector_base64 fields") + } + if original.Similarity != updated.Similarity { + return nil, fmt.Errorf("similarity cannot be updated for vector and vector_base64 fields") + } + if original.VectorIndexOptimizedFor != updated.VectorIndexOptimizedFor { + return nil, fmt.Errorf("vectorIndexOptimizedFor cannot be updated for vector and vector_base64 fields") + } + if original.GPU != updated.GPU { + return nil, fmt.Errorf("gpu cannot be updated for vector and vector_base64 fields") + } + } + if original.IncludeInAll != updated.IncludeInAll { + return nil, fmt.Errorf("includeInAll cannot be changed") + } + if original.IncludeTermVectors != updated.IncludeTermVectors { + return nil, fmt.Errorf("includeTermVectors cannot be changed") + } + if original.SkipFreqNorm != updated.SkipFreqNorm { + return nil, fmt.Errorf("skipFreqNorm cannot be changed") + } + + // Updating is not possible if store changes from true + // to false when the field is included in _all + if original.Store != updated.Store { + if updated.Store { + return nil, fmt.Errorf("store cannot be changed from false to true") + } else if updated.IncludeInAll { + return nil, fmt.Errorf("store cannot be changed if field present in `_all' field") + } else { + rv.Store = true + } + } + + // Updating is not possible if index changes from true + // to false when the field is included in _all + if original.Index != updated.Index { + if updated.Index { + return nil, fmt.Errorf("index cannot be changed from false to true") + } else if updated.IncludeInAll { + return nil, fmt.Errorf("index cannot be changed if field present in `_all' field") + } else { + rv.Index = true + rv.DocValues = true + } + } + + // Updating is not possible if docvalues changes from true + // to false when the field is included in _all + if original.DocValues != updated.DocValues { + if updated.DocValues { + return nil, fmt.Errorf("docvalues cannot be changed from false to true") + } else if updated.IncludeInAll { + return nil, fmt.Errorf("docvalues cannot be changed if field present in `_all' field") + } else { + rv.DocValues = true + } + } + + return rv, nil +} + +// After identifying changes, validate against the existing changes incase of duplicate fields. +// In such a situation, any conflicting changes found will abort the update process +func validateFieldInfo(newInfo *index.UpdateFieldInfo, fInfo map[string]*index.UpdateFieldInfo, + ori *pathInfo, oriFMapInfo *fieldMapInfo) error { + // Determine field name + fieldName := oriFMapInfo.fieldMapping.Name + if fieldName == "" { + fieldName = oriFMapInfo.parent.path + } + + // Construct full name with parent path + var name string + if oriFMapInfo.parent.parentPath == "" { + name = fieldName + } else { + name = oriFMapInfo.parent.parentPath + "." + fieldName + } + if (newInfo.Deleted || newInfo.Index || newInfo.DocValues || newInfo.Store) && ori.dynamic { + return fmt.Errorf("updated field is under a dynamic property") + } + if oldInfo, ok := fInfo[name]; ok { + if !reflect.DeepEqual(oldInfo, newInfo) { + return fmt.Errorf("updated field impossible to verify because multiple mappings point to the same field name") + } + } else { + fInfo[name] = newInfo + } + return nil +} diff --git a/index_update_test.go b/index_update_test.go new file mode 100644 index 0000000..25a812b --- /dev/null +++ b/index_update_test.go @@ -0,0 +1,3250 @@ +// Copyright (c) 2025 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "context" + "encoding/json" + "fmt" + "math/rand" + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" + "github.com/blevesearch/bleve/v2/analysis/analyzer/simple" + "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" + "github.com/blevesearch/bleve/v2/analysis/datetime/percent" + "github.com/blevesearch/bleve/v2/analysis/datetime/sanitized" + "github.com/blevesearch/bleve/v2/analysis/lang/en" + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/letter" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/whitespace" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/index/scorch/mergeplan" + "github.com/blevesearch/bleve/v2/mapping" + index "github.com/blevesearch/bleve_index_api" +) + +func TestCompareFieldMapping(t *testing.T) { + tests := []struct { + original *mapping.FieldMapping + updated *mapping.FieldMapping + indexFieldInfo *index.UpdateFieldInfo + err bool + }{ + { // both nil => error + original: nil, + updated: nil, + indexFieldInfo: nil, + err: true, + }, + { // updated nil => delete all + original: &mapping.FieldMapping{}, + updated: nil, + indexFieldInfo: &index.UpdateFieldInfo{ + Deleted: true, + }, + err: false, + }, + { // type changed => not updatable + original: &mapping.FieldMapping{ + Type: "text", + }, + updated: &mapping.FieldMapping{ + Type: "datetime", + }, + indexFieldInfo: nil, + err: true, + }, + { // synonym source changed for text => updatable + original: &mapping.FieldMapping{ + Type: "text", + SynonymSource: "a", + }, + updated: &mapping.FieldMapping{ + Type: "text", + SynonymSource: "b", + }, + indexFieldInfo: &index.UpdateFieldInfo{}, + err: false, + }, + { // analyser changed for text => not updatable + original: &mapping.FieldMapping{ + Type: "text", + Analyzer: "a", + }, + updated: &mapping.FieldMapping{ + Type: "text", + Analyzer: "b", + }, + indexFieldInfo: nil, + err: true, + }, + { // dims changed for vector => not updatable + original: &mapping.FieldMapping{ + Type: "vector", + Dims: 128, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "memory-efficient", + }, + updated: &mapping.FieldMapping{ + Type: "vector", + Dims: 1024, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "memory-efficient", + }, + indexFieldInfo: nil, + err: true, + }, + { // similarity changed for vectorbase64 => not updatable + original: &mapping.FieldMapping{ + Type: "vector_base64", + Similarity: "l2_norm", + Dims: 128, + VectorIndexOptimizedFor: "memory-efficient", + }, + updated: &mapping.FieldMapping{ + Type: "vector_base64", + Similarity: "dot_product", + Dims: 128, + VectorIndexOptimizedFor: "memory-efficient", + }, + indexFieldInfo: nil, + err: true, + }, + { // vectorindexoptimizedfor chagned for vector => not updatable + original: &mapping.FieldMapping{ + Type: "vector", + Similarity: "dot_product", + Dims: 128, + VectorIndexOptimizedFor: "memory-efficient", + }, + updated: &mapping.FieldMapping{ + Type: "vector", + Similarity: "dot_product", + Dims: 128, + VectorIndexOptimizedFor: "latency", + }, + indexFieldInfo: nil, + err: true, + }, + { // gpu changed for vector => not updatable + original: &mapping.FieldMapping{ + Type: "vector", + Similarity: "dot_product", + Dims: 128, + VectorIndexOptimizedFor: "memory-efficient", + GPU: false, + }, + updated: &mapping.FieldMapping{ + Type: "vector", + Similarity: "dot_product", + Dims: 128, + VectorIndexOptimizedFor: "memory-efficient", + GPU: true, + }, + indexFieldInfo: nil, + err: true, + }, + { // gpu changed for vectorbase64 => not updatable + original: &mapping.FieldMapping{ + Type: "vector_base64", + Similarity: "dot_product", + Dims: 128, + VectorIndexOptimizedFor: "memory-efficient", + GPU: false, + }, + updated: &mapping.FieldMapping{ + Type: "vector_base64", + Similarity: "dot_product", + Dims: 128, + VectorIndexOptimizedFor: "memory-efficient", + GPU: true, + }, + indexFieldInfo: nil, + err: true, + }, + { // includeinall changed => not updatable + original: &mapping.FieldMapping{ + Type: "numeric", + IncludeInAll: true, + }, + updated: &mapping.FieldMapping{ + Type: "numeric", + IncludeInAll: false, + }, + indexFieldInfo: nil, + err: true, + }, + { //includetermvectors changed => not updatable + original: &mapping.FieldMapping{ + Type: "numeric", + IncludeTermVectors: false, + }, + updated: &mapping.FieldMapping{ + Type: "numeric", + IncludeTermVectors: true, + }, + indexFieldInfo: nil, + err: true, + }, + { // store changed after all checks => updatable with store delete + original: &mapping.FieldMapping{ + Type: "numeric", + SkipFreqNorm: true, + }, + updated: &mapping.FieldMapping{ + Type: "numeric", + SkipFreqNorm: false, + }, + indexFieldInfo: nil, + err: true, + }, + { // index changed after all checks => updatable with index and docvalues delete + original: &mapping.FieldMapping{ + Type: "geopoint", + Index: true, + }, + updated: &mapping.FieldMapping{ + Type: "geopoint", + Index: false, + }, + indexFieldInfo: &index.UpdateFieldInfo{ + Index: true, + DocValues: true, + }, + err: false, + }, + { // docvalues changed after all checks => docvalues delete + original: &mapping.FieldMapping{ + Type: "numeric", + DocValues: true, + }, + updated: &mapping.FieldMapping{ + Type: "numeric", + DocValues: false, + }, + indexFieldInfo: &index.UpdateFieldInfo{ + DocValues: true, + }, + err: false, + }, + { // no relavent changes => continue but no op + original: &mapping.FieldMapping{ + Name: "", + Type: "datetime", + Analyzer: "a", + Store: true, + Index: false, + IncludeTermVectors: true, + IncludeInAll: false, + DateFormat: "a", + DocValues: false, + SkipFreqNorm: true, + Dims: 128, + Similarity: "dot_product", + VectorIndexOptimizedFor: "memory-efficient", + SynonymSource: "a", + }, + updated: &mapping.FieldMapping{ + Name: "", + Type: "datetime", + Analyzer: "b", + Store: true, + Index: false, + IncludeTermVectors: true, + IncludeInAll: false, + DateFormat: "a", + DocValues: false, + SkipFreqNorm: true, + Dims: 256, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "latency", + SynonymSource: "b", + }, + indexFieldInfo: &index.UpdateFieldInfo{}, + err: false, + }, + } + + for i, test := range tests { + rv, err := compareFieldMapping(test.original, test.updated) + + if err == nil && test.err || err != nil && !test.err { + t.Errorf("Unexpected error value for test %d, expecting %t, got %v\n", i, test.err, err) + } + if rv == nil && test.indexFieldInfo != nil || rv != nil && test.indexFieldInfo == nil || !reflect.DeepEqual(rv, test.indexFieldInfo) { + t.Errorf("Unexpected index field info value for test %d, expecting %+v, got %+v, err %v", i, test.indexFieldInfo, rv, err) + } + } +} + +func TestCompareMappings(t *testing.T) { + tests := []struct { + original *mapping.IndexMappingImpl + updated *mapping.IndexMappingImpl + err bool + }{ + { // changed type field when non empty mappings are present => error + original: &mapping.IndexMappingImpl{ + TypeField: "a", + TypeMapping: map[string]*mapping.DocumentMapping{ + "a": {}, + "b": {}, + }, + }, + updated: &mapping.IndexMappingImpl{ + TypeField: "b", + TypeMapping: map[string]*mapping.DocumentMapping{ + "a": {}, + "b": {}, + }, + }, + err: true, + }, + { // changed default type => error + original: &mapping.IndexMappingImpl{ + DefaultType: "a", + }, + updated: &mapping.IndexMappingImpl{ + DefaultType: "b", + }, + err: true, + }, + { // changed default analyzer => analyser true + original: &mapping.IndexMappingImpl{ + DefaultAnalyzer: "a", + }, + updated: &mapping.IndexMappingImpl{ + DefaultAnalyzer: "b", + }, + err: false, + }, + { // changed default datetimeparser => datetimeparser true + original: &mapping.IndexMappingImpl{ + DefaultDateTimeParser: "a", + }, + updated: &mapping.IndexMappingImpl{ + DefaultDateTimeParser: "b", + }, + err: false, + }, + { // changed default synonym source => synonym source true + original: &mapping.IndexMappingImpl{ + DefaultSynonymSource: "a", + }, + updated: &mapping.IndexMappingImpl{ + DefaultSynonymSource: "b", + }, + err: false, + }, + { // changed default field => false + original: &mapping.IndexMappingImpl{ + DefaultField: "a", + }, + updated: &mapping.IndexMappingImpl{ + DefaultField: "b", + }, + err: false, + }, + { // changed index dynamic => error + original: &mapping.IndexMappingImpl{ + IndexDynamic: true, + }, + updated: &mapping.IndexMappingImpl{ + IndexDynamic: false, + }, + err: true, + }, + { // changed store dynamic => error + original: &mapping.IndexMappingImpl{ + StoreDynamic: false, + }, + updated: &mapping.IndexMappingImpl{ + StoreDynamic: true, + }, + err: true, + }, + { // changed docvalues dynamic => error + original: &mapping.IndexMappingImpl{ + DocValuesDynamic: true, + }, + updated: &mapping.IndexMappingImpl{ + DocValuesDynamic: false, + }, + err: true, + }, + } + + for i, test := range tests { + err := compareMappings(test.original, test.updated) + + if err == nil && test.err || err != nil && !test.err { + t.Errorf("Unexpected error value for test %d, expecting %t, got %v\n", i, test.err, err) + } + } +} + +func TestCompareAnalysers(t *testing.T) { + + ori := mapping.NewIndexMapping() + ori.DefaultMapping.AddFieldMappingsAt("a", NewTextFieldMapping()) + ori.DefaultMapping.AddFieldMappingsAt("b", NewTextFieldMapping()) + ori.DefaultMapping.AddFieldMappingsAt("c", NewTextFieldMapping()) + ori.DefaultMapping.Properties["b"].DefaultAnalyzer = "3xbla" + ori.DefaultMapping.Properties["c"].DefaultAnalyzer = simple.Name + + upd := mapping.NewIndexMapping() + upd.DefaultMapping.AddFieldMappingsAt("a", NewTextFieldMapping()) + upd.DefaultMapping.AddFieldMappingsAt("b", NewTextFieldMapping()) + upd.DefaultMapping.AddFieldMappingsAt("c", NewTextFieldMapping()) + upd.DefaultMapping.Properties["b"].DefaultAnalyzer = "3xbla" + upd.DefaultMapping.Properties["c"].DefaultAnalyzer = simple.Name + + if err := ori.AddCustomAnalyzer("3xbla", map[string]interface{}{ + "type": custom.Name, + "tokenizer": whitespace.Name, + "token_filters": []interface{}{lowercase.Name, "stop_en"}, + }); err != nil { + t.Fatal(err) + } + + if err := upd.AddCustomAnalyzer("3xbla", map[string]interface{}{ + "type": custom.Name, + "tokenizer": whitespace.Name, + "token_filters": []interface{}{lowercase.Name, "stop_en"}, + }); err != nil { + t.Fatal(err) + } + + oriPaths := map[string]*pathInfo{ + "a": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "text", + }, + }, + }, + dynamic: false, + path: "a", + parentPath: "", + }, + "b": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "text", + }, + }, + }, + dynamic: false, + path: "b", + parentPath: "", + }, + "c": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "text", + }, + }, + }, + dynamic: false, + path: "c", + parentPath: "", + }, + } + + updPaths := map[string]*pathInfo{ + "a": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "text", + }, + }, + }, + dynamic: false, + path: "a", + parentPath: "", + }, + "b": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "text", + }, + }, + }, + dynamic: false, + path: "b", + parentPath: "", + }, + "c": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "text", + }, + }, + }, + dynamic: false, + path: "c", + parentPath: "", + }, + } + + // Test case has identical analysers for text fields + err := compareAnalysers(oriPaths, updPaths, ori, upd) + if err != nil { + t.Errorf("Expected error to be nil, got %v", err) + } + + ori2 := mapping.NewIndexMapping() + ori2.DefaultMapping.AddFieldMappingsAt("a", NewTextFieldMapping()) + ori2.DefaultMapping.AddFieldMappingsAt("b", NewTextFieldMapping()) + ori2.DefaultMapping.AddFieldMappingsAt("c", NewTextFieldMapping()) + ori2.DefaultMapping.Properties["b"].DefaultAnalyzer = "3xbla" + ori2.DefaultMapping.Properties["c"].DefaultAnalyzer = simple.Name + + upd2 := mapping.NewIndexMapping() + upd2.DefaultMapping.AddFieldMappingsAt("a", NewTextFieldMapping()) + upd2.DefaultMapping.AddFieldMappingsAt("b", NewTextFieldMapping()) + upd2.DefaultMapping.AddFieldMappingsAt("c", NewTextFieldMapping()) + upd2.DefaultMapping.Properties["b"].DefaultAnalyzer = "3xbla" + upd2.DefaultMapping.Properties["c"].DefaultAnalyzer = simple.Name + + if err := ori2.AddCustomAnalyzer("3xbla", map[string]interface{}{ + "type": custom.Name, + "tokenizer": whitespace.Name, + "token_filters": []interface{}{lowercase.Name, "stop_en"}, + }); err != nil { + t.Fatal(err) + } + + if err := upd2.AddCustomAnalyzer("3xbla", map[string]interface{}{ + "type": custom.Name, + "tokenizer": letter.Name, + "token_filters": []interface{}{lowercase.Name, "stop_en"}, + }); err != nil { + t.Fatal(err) + } + + // Test case has different custom analyser for field "b" + err = compareAnalysers(oriPaths, updPaths, ori2, upd2) + if err == nil { + t.Errorf("Expected error, got nil") + } +} + +func TestCompareDatetimeParsers(t *testing.T) { + + ori := mapping.NewIndexMapping() + ori.DefaultMapping.AddFieldMappingsAt("a", NewDateTimeFieldMapping()) + ori.DefaultMapping.AddFieldMappingsAt("b", NewDateTimeFieldMapping()) + ori.DefaultMapping.AddFieldMappingsAt("c", NewDateTimeFieldMapping()) + ori.DefaultMapping.Properties["b"].Fields[0].DateFormat = "customDT" + ori.DefaultMapping.Properties["c"].Fields[0].DateFormat = percent.Name + + upd := mapping.NewIndexMapping() + upd.DefaultMapping.AddFieldMappingsAt("a", NewDateTimeFieldMapping()) + upd.DefaultMapping.AddFieldMappingsAt("b", NewDateTimeFieldMapping()) + upd.DefaultMapping.AddFieldMappingsAt("c", NewDateTimeFieldMapping()) + upd.DefaultMapping.Properties["b"].Fields[0].DateFormat = "customDT" + upd.DefaultMapping.Properties["c"].Fields[0].DateFormat = percent.Name + + err := ori.AddCustomDateTimeParser("customDT", map[string]interface{}{ + "type": sanitized.Name, + "layouts": []interface{}{ + "02/01/2006 15:04:05", + "2006/01/02 3:04PM", + }, + }) + if err != nil { + t.Fatal(err) + } + + err = upd.AddCustomDateTimeParser("customDT", map[string]interface{}{ + "type": sanitized.Name, + "layouts": []interface{}{ + "02/01/2006 15:04:05", + "2006/01/02 3:04PM", + }, + }) + if err != nil { + t.Fatal(err) + } + + oriPaths := map[string]*pathInfo{ + "a": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "datetime", + }, + }, + }, + dynamic: false, + path: "a", + parentPath: "", + }, + "b": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "datetime", + DateFormat: "customDT", + }, + }, + }, + dynamic: false, + path: "b", + parentPath: "", + }, + "c": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "datetime", + }, + }, + }, + dynamic: false, + path: "c", + parentPath: "", + }, + } + + updPaths := map[string]*pathInfo{ + "a": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "datetime", + }, + }, + }, + dynamic: false, + path: "a", + parentPath: "", + }, + "b": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "datetime", + DateFormat: "customDT", + }, + }, + }, + dynamic: false, + path: "b", + parentPath: "", + }, + "c": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "datetime", + }, + }, + }, + dynamic: false, + path: "c", + parentPath: "", + }, + } + + // Test case has identical datetime parsers for all fields + err = compareDateTimeParsers(oriPaths, updPaths, ori, upd) + if err != nil { + t.Fatalf("Expected error to be nil, got %v", err) + } + + ori2 := mapping.NewIndexMapping() + ori2.DefaultMapping.AddFieldMappingsAt("a", NewDateTimeFieldMapping()) + ori2.DefaultMapping.AddFieldMappingsAt("b", NewDateTimeFieldMapping()) + ori2.DefaultMapping.AddFieldMappingsAt("c", NewDateTimeFieldMapping()) + ori2.DefaultMapping.Properties["b"].Fields[0].DateFormat = "customDT" + ori2.DefaultMapping.Properties["c"].Fields[0].DateFormat = percent.Name + + upd2 := mapping.NewIndexMapping() + upd2.DefaultMapping.AddFieldMappingsAt("a", NewDateTimeFieldMapping()) + upd2.DefaultMapping.AddFieldMappingsAt("b", NewDateTimeFieldMapping()) + upd2.DefaultMapping.AddFieldMappingsAt("c", NewDateTimeFieldMapping()) + upd2.DefaultMapping.Properties["b"].Fields[0].DateFormat = "customDT" + upd2.DefaultMapping.Properties["c"].Fields[0].DateFormat = percent.Name + + err = ori2.AddCustomDateTimeParser("customDT", map[string]interface{}{ + "type": sanitized.Name, + "layouts": []interface{}{ + "02/01/2006 15:04:05", + "2006/01/02 3:04PM", + }, + }) + if err != nil { + t.Fatal(err) + } + + err = upd2.AddCustomDateTimeParser("customDT", map[string]interface{}{ + "type": sanitized.Name, + "layouts": []interface{}{ + "02/01/2006 15:04:05", + "2006/01/02", + }, + }) + if err != nil { + t.Fatal(err) + } + + // test case has different custom datetime parser for field "b" + err = compareDateTimeParsers(oriPaths, updPaths, ori2, upd2) + if err == nil { + t.Errorf("Expected error, got nil") + } +} + +func TestCompareSynonymSources(t *testing.T) { + + ori := mapping.NewIndexMapping() + ori.DefaultMapping.AddFieldMappingsAt("a", NewTextFieldMapping()) + ori.DefaultMapping.AddFieldMappingsAt("b", NewTextFieldMapping()) + ori.DefaultMapping.DefaultSynonymSource = "syn1" + ori.DefaultMapping.Properties["b"].Fields[0].SynonymSource = "syn2" + + upd := mapping.NewIndexMapping() + upd.DefaultMapping.AddFieldMappingsAt("a", NewTextFieldMapping()) + upd.DefaultMapping.AddFieldMappingsAt("b", NewTextFieldMapping()) + upd.DefaultMapping.DefaultSynonymSource = "syn1" + upd.DefaultMapping.Properties["b"].Fields[0].SynonymSource = "syn2" + + err := ori.AddSynonymSource("syn1", map[string]interface{}{ + "collection": "col1", + "analyzer": simple.Name, + }) + if err != nil { + t.Fatal(err) + } + err = ori.AddSynonymSource("syn2", map[string]interface{}{ + "collection": "col2", + "analyzer": standard.Name, + }) + if err != nil { + t.Fatal(err) + } + + err = upd.AddSynonymSource("syn1", map[string]interface{}{ + "collection": "col1", + "analyzer": simple.Name, + }) + if err != nil { + t.Fatal(err) + } + err = upd.AddSynonymSource("syn2", map[string]interface{}{ + "collection": "col2", + "analyzer": standard.Name, + }) + if err != nil { + t.Fatal(err) + } + + // Test case has identical synonym sources + err = compareSynonymSources(ori, upd) + if err != nil { + t.Errorf("Expected error to be nil, got %v", err) + } + + ori2 := mapping.NewIndexMapping() + ori2.DefaultMapping.AddFieldMappingsAt("a", NewTextFieldMapping()) + ori2.DefaultMapping.AddFieldMappingsAt("b", NewTextFieldMapping()) + ori2.DefaultMapping.DefaultSynonymSource = "syn1" + ori2.DefaultMapping.Properties["b"].Fields[0].SynonymSource = "syn2" + + upd2 := mapping.NewIndexMapping() + upd2.DefaultMapping.AddFieldMappingsAt("a", NewTextFieldMapping()) + upd2.DefaultMapping.AddFieldMappingsAt("b", NewTextFieldMapping()) + upd2.DefaultMapping.DefaultSynonymSource = "syn1" + upd2.DefaultMapping.Properties["b"].Fields[0].SynonymSource = "syn2" + + err = ori2.AddSynonymSource("syn1", map[string]interface{}{ + "collection": "col1", + "analyzer": simple.Name, + }) + if err != nil { + t.Fatal(err) + } + err = ori2.AddSynonymSource("syn2", map[string]interface{}{ + "collection": "col2", + "analyzer": standard.Name, + }) + if err != nil { + t.Fatal(err) + } + + err = upd2.AddSynonymSource("syn1", map[string]interface{}{ + "collection": "col1", + "analyzer": simple.Name, + }) + if err != nil { + t.Fatal(err) + } + err = upd2.AddSynonymSource("syn2", map[string]interface{}{ + "collection": "col3", + "analyzer": standard.Name, + }) + if err != nil { + t.Fatal(err) + } + + // test case has different synonym sources + err = compareSynonymSources(ori2, upd2) + if err == nil { + t.Errorf("Expected error, got nil") + } +} + +func TestDeletedFields(t *testing.T) { + tests := []struct { + original *mapping.IndexMappingImpl + updated *mapping.IndexMappingImpl + fieldInfo map[string]*index.UpdateFieldInfo + err bool + }{ + { + // changed default analyzer with index dynamic + // => error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{}, + DefaultMapping: &mapping.DocumentMapping{}, + DefaultAnalyzer: standard.Name, + IndexDynamic: true, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{}, + DefaultMapping: &mapping.DocumentMapping{}, + DefaultAnalyzer: simple.Name, + IndexDynamic: true, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: nil, + err: true, + }, + { + // changed default analyzer within a mapping with index dynamic + // => error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{}, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: true, + DefaultAnalyzer: standard.Name, + }, + DefaultAnalyzer: "", + IndexDynamic: true, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{}, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: true, + DefaultAnalyzer: simple.Name, + }, + IndexDynamic: true, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: nil, + err: true, + }, + { + // changed default datetime parser with index dynamic + // => error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{}, + DefaultMapping: &mapping.DocumentMapping{}, + DefaultDateTimeParser: percent.Name, + IndexDynamic: true, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{}, + DefaultMapping: &mapping.DocumentMapping{}, + DefaultDateTimeParser: sanitized.Name, + IndexDynamic: true, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: nil, + err: true, + }, + { + // no change between original and updated having type and default mapping + // => empty fieldInfo with no error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: map[string]*index.UpdateFieldInfo{}, + err: false, + }, + { + // no changes in type mappings and default mapping disabled with changes + // => empty fieldInfo with no error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: false, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: false, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "d": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: map[string]*index.UpdateFieldInfo{}, + err: false, + }, + { + // new type mappings in updated => error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: nil, + err: true, + }, + { + // new mappings in default mapping => error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{}, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{}, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: nil, + err: true, + }, + { + // fully removed mapping in type with some dynamic => error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: true, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: nil, + err: true, + }, + { + // semi removed mapping in default with some dynamic + // proper fieldInfo with no errors + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: false, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: map[string]*index.UpdateFieldInfo{ + "b": { + Index: true, + DocValues: true, + }, + }, + err: false, + }, + { + // two fields from diff paths with removed content matching + // => relavent fieldInfo + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: false, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: false, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: map[string]*index.UpdateFieldInfo{ + "a": { + Index: true, + DocValues: true, + }, + }, + err: false, + }, + { + // two fields from diff paths with removed content not matching + // => error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: false, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: nil, + err: true, + }, + { + // two fields from the same path => relavent fieldInfo + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Name: "a", + Type: "numeric", + Index: true, + Store: true, + }, + { + Name: "b", + Type: "numeric", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Name: "a", + Type: "numeric", + Index: false, + Store: true, + }, + { + Name: "b", + Type: "numeric", + Index: true, + Store: false, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: map[string]*index.UpdateFieldInfo{ + "a": { + Index: true, + DocValues: true, + }, + "b": { + Store: true, + }, + }, + err: false, + }, + { + // one store, one index, one dynamic and one all removed in type and default + // => relavent fieldInfo without error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Store: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map3": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + DocValues: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "d": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + Store: true, + DocValues: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: false, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Store: false, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map3": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + DocValues: false, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: map[string]*index.UpdateFieldInfo{ + "a": { + Index: true, + DocValues: true, + }, + "b": { + Store: true, + }, + "c": { + DocValues: true, + }, + "d": { + Deleted: true, + }, + }, + err: false, + }, + } + + for i, test := range tests { + info, err := DeletedFields(test.original, test.updated) + + if err == nil && test.err || err != nil && !test.err { + t.Errorf("Unexpected error value for test %d, expecting %t, got %v\n", i, test.err, err) + } + if info == nil && test.fieldInfo != nil || info != nil && test.fieldInfo == nil || !reflect.DeepEqual(info, test.fieldInfo) { + t.Errorf("Unexpected default info value for test %d, expecting %+v, got %+v, err %v", i, test.fieldInfo, info, err) + } + } +} + +func TestIndexUpdateText(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + indexMappingBefore := mapping.NewIndexMapping() + indexMappingBefore.TypeMapping = map[string]*mapping.DocumentMapping{} + indexMappingBefore.DefaultMapping = &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "d": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + } + indexMappingBefore.IndexDynamic = false + indexMappingBefore.StoreDynamic = false + indexMappingBefore.DocValuesDynamic = false + + index, err := New(tmpIndexPath, indexMappingBefore) + if err != nil { + t.Fatal(err) + } + doc1 := map[string]interface{}{"a": "xyz", "b": "abc", "c": "def", "d": "ghi"} + doc2 := map[string]interface{}{"a": "uvw", "b": "rst", "c": "klm", "d": "pqr"} + doc3 := map[string]interface{}{"a": "xyz", "b": "def", "c": "abc", "d": "mno"} + batch := index.NewBatch() + err = batch.Index("001", doc1) + if err != nil { + t.Fatal(err) + } + err = batch.Index("002", doc2) + if err != nil { + t.Fatal(err) + } + err = batch.Index("003", doc3) + if err != nil { + t.Fatal(err) + } + err = index.Batch(batch) + if err != nil { + t.Fatal(err) + } + err = index.Close() + if err != nil { + t.Fatal(err) + } + + indexMappingAfter := mapping.NewIndexMapping() + indexMappingAfter.TypeMapping = map[string]*mapping.DocumentMapping{} + indexMappingAfter.DefaultMapping = &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: false, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: false, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + } + indexMappingAfter.IndexDynamic = false + indexMappingAfter.StoreDynamic = false + indexMappingAfter.DocValuesDynamic = false + + mappingString, err := json.Marshal(indexMappingAfter) + if err != nil { + t.Fatal(err) + } + + config := map[string]interface{}{ + "updated_mapping": string(mappingString), + } + + index, err = OpenUsing(tmpIndexPath, config) + if err != nil { + t.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + }() + + q1 := NewSearchRequest(NewQueryStringQuery("a:*")) + q1.Fields = append(q1.Fields, "a") + res1, err := index.Search(q1) + if err != nil { + t.Fatal(err) + } + if len(res1.Hits) != 3 { + t.Errorf("Expected 3 hits, got %d\n", len(res1.Hits)) + } + if len(res1.Hits[0].Fields) != 1 { + t.Errorf("Expected 1 field, got %d\n", len(res1.Hits[0].Fields)) + } + q2 := NewSearchRequest(NewQueryStringQuery("b:*")) + q2.Fields = append(q2.Fields, "b") + res2, err := index.Search(q2) + if err != nil { + t.Fatal(err) + } + if len(res2.Hits) != 0 { + t.Errorf("Expected 0 hits, got %d\n", len(res2.Hits)) + } + q3 := NewSearchRequest(NewQueryStringQuery("c:*")) + q3.Fields = append(q3.Fields, "c") + res3, err := index.Search(q3) + if err != nil { + t.Fatal(err) + } + if len(res3.Hits) != 3 { + t.Errorf("Expected 3 hits, got %d\n", len(res3.Hits)) + } + if len(res3.Hits[0].Fields) != 0 { + t.Errorf("Expected 0 fields, got %d\n", len(res3.Hits[0].Fields)) + } + q4 := NewSearchRequest(NewQueryStringQuery("d:*")) + q4.Fields = append(q4.Fields, "d") + res4, err := index.Search(q4) + if err != nil { + t.Fatal(err) + } + if len(res4.Hits) != 0 { + t.Errorf("Expected 0 hits, got %d\n", len(res4.Hits)) + } +} + +func TestIndexUpdateSynonym(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + synonymCollection := "collection1" + synonymSourceName := "english" + analyzer := en.AnalyzerName + synonymSourceConfig := map[string]interface{}{ + "collection": synonymCollection, + "analyzer": analyzer, + } + + a := mapping.NewTextFieldMapping() + a.Analyzer = analyzer + a.SynonymSource = synonymSourceName + a.IncludeInAll = false + + b := mapping.NewTextFieldMapping() + b.Analyzer = analyzer + b.SynonymSource = synonymSourceName + b.IncludeInAll = false + + c := mapping.NewTextFieldMapping() + c.Analyzer = analyzer + c.SynonymSource = synonymSourceName + c.IncludeInAll = false + + indexMappingBefore := mapping.NewIndexMapping() + indexMappingBefore.DefaultMapping.AddFieldMappingsAt("a", a) + indexMappingBefore.DefaultMapping.AddFieldMappingsAt("b", b) + indexMappingBefore.DefaultMapping.AddFieldMappingsAt("c", c) + err := indexMappingBefore.AddSynonymSource(synonymSourceName, synonymSourceConfig) + if err != nil { + t.Fatal(err) + } + + indexMappingBefore.IndexDynamic = false + indexMappingBefore.StoreDynamic = false + indexMappingBefore.DocValuesDynamic = false + + index, err := New(tmpIndexPath, indexMappingBefore) + if err != nil { + t.Fatal(err) + } + + doc1 := map[string]interface{}{ + "a": `The hardworking employee consistently strives to exceed expectations. + His industrious nature makes him a valuable asset to any team. + His conscientious attention to detail ensures that projects are completed efficiently and accurately. + He remains persistent even in the face of challenges.`, + "b": `The hardworking employee consistently strives to exceed expectations. + His industrious nature makes him a valuable asset to any team. + His conscientious attention to detail ensures that projects are completed efficiently and accurately. + He remains persistent even in the face of challenges.`, + "c": `The hardworking employee consistently strives to exceed expectations. + His industrious nature makes him a valuable asset to any team. + His conscientious attention to detail ensures that projects are completed efficiently and accurately. + He remains persistent even in the face of challenges.`, + } + doc2 := map[string]interface{}{ + "a": `The tranquil surroundings of the retreat provide a perfect escape from the hustle and bustle of city life. + Guests enjoy the peaceful atmosphere, which is perfect for relaxation and rejuvenation. + The calm environment offers the ideal place to meditate and connect with nature. + Even the most stressed individuals find themselves feeling relaxed and at ease.`, + "b": `The tranquil surroundings of the retreat provide a perfect escape from the hustle and bustle of city life. + Guests enjoy the peaceful atmosphere, which is perfect for relaxation and rejuvenation. + The calm environment offers the ideal place to meditate and connect with nature. + Even the most stressed individuals find themselves feeling relaxed and at ease.`, + "c": `The tranquil surroundings of the retreat provide a perfect escape from the hustle and bustle of city life. + Guests enjoy the peaceful atmosphere, which is perfect for relaxation and rejuvenation. + The calm environment offers the ideal place to meditate and connect with nature. + Even the most stressed individuals find themselves feeling relaxed and at ease.`, + } + synDoc1 := &SynonymDefinition{Synonyms: []string{"hardworking", "industrious", "conscientious", "persistent", "focused", "devoted"}} + synDoc2 := &SynonymDefinition{Synonyms: []string{"tranquil", "peaceful", "calm", "relaxed", "unruffled"}} + + batch := index.NewBatch() + err = batch.IndexSynonym("001", synonymCollection, synDoc1) + if err != nil { + t.Fatal(err) + } + err = batch.IndexSynonym("002", synonymCollection, synDoc2) + if err != nil { + t.Fatal(err) + } + err = batch.Index("003", doc1) + if err != nil { + t.Fatal(err) + } + err = batch.Index("004", doc2) + if err != nil { + t.Fatal(err) + } + err = index.Batch(batch) + if err != nil { + t.Fatal(err) + } + err = index.Close() + if err != nil { + t.Fatal(err) + } + + indexMappingAfter := mapping.NewIndexMapping() + indexMappingAfter.DefaultMapping.AddFieldMappingsAt("a", a) + b.Index = false + indexMappingAfter.DefaultMapping.AddFieldMappingsAt("b", b) + err = indexMappingAfter.AddSynonymSource(synonymSourceName, synonymSourceConfig) + if err != nil { + t.Fatal(err) + } + + indexMappingAfter.IndexDynamic = false + indexMappingAfter.StoreDynamic = false + indexMappingAfter.DocValuesDynamic = false + + mappingString, err := json.Marshal(indexMappingAfter) + if err != nil { + t.Fatal(err) + } + config := map[string]interface{}{ + "updated_mapping": string(mappingString), + } + + index, err = OpenUsing(tmpIndexPath, config) + if err != nil { + t.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + }() + + q1 := NewSearchRequest(NewQueryStringQuery("a:devoted")) + res1, err := index.Search(q1) + if err != nil { + t.Fatal(err) + } + if len(res1.Hits) != 1 { + t.Errorf("Expected 1 hit, got %d\n", len(res1.Hits)) + } + + q2 := NewSearchRequest(NewQueryStringQuery("b:devoted")) + res2, err := index.Search(q2) + if err != nil { + t.Fatal(err) + } + if len(res2.Hits) != 0 { + t.Errorf("Expected 0 hits, got %d\n", len(res2.Hits)) + } + + q3 := NewSearchRequest(NewQueryStringQuery("c:unruffled")) + res3, err := index.Search(q3) + if err != nil { + t.Fatal(err) + } + if len(res3.Hits) != 0 { + t.Errorf("Expected 0 hits, got %d\n", len(res3.Hits)) + } +} + +func TestIndexUpdateMerge(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + indexMappingBefore := mapping.NewIndexMapping() + indexMappingBefore.TypeMapping = map[string]*mapping.DocumentMapping{} + indexMappingBefore.DefaultMapping = &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "d": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + } + indexMappingBefore.IndexDynamic = false + indexMappingBefore.StoreDynamic = false + indexMappingBefore.DocValuesDynamic = false + + index, err := New(tmpIndexPath, indexMappingBefore) + if err != nil { + t.Fatal(err) + } + + numDocsPerBatch := 1000 + numBatches := 10 + + var batch *Batch + doc := make(map[string]interface{}) + const letters = "abcdefghijklmnopqrstuvwxyz" + + randStr := func() string { + result := make([]byte, 3) + for i := 0; i < 3; i++ { + result[i] = letters[rand.Intn(len(letters))] + } + return string(result) + } + for i := 0; i < numBatches; i++ { + batch = index.NewBatch() + for j := 0; j < numDocsPerBatch; j++ { + doc["a"] = randStr() + doc["b"] = randStr() + doc["c"] = randStr() + doc["d"] = randStr() + err = batch.Index(fmt.Sprintf("%d", i*numDocsPerBatch+j), doc) + if err != nil { + t.Fatal(err) + } + } + err = index.Batch(batch) + if err != nil { + t.Fatal(err) + } + } + + err = index.Close() + if err != nil { + t.Fatal(err) + } + + indexMappingAfter := mapping.NewIndexMapping() + indexMappingAfter.TypeMapping = map[string]*mapping.DocumentMapping{} + indexMappingAfter.DefaultMapping = &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: false, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: false, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + } + indexMappingAfter.IndexDynamic = false + indexMappingAfter.StoreDynamic = false + indexMappingAfter.DocValuesDynamic = false + + mappingString, err := json.Marshal(indexMappingAfter) + if err != nil { + t.Fatal(err) + } + config := map[string]interface{}{ + "updated_mapping": string(mappingString), + } + + index, err = OpenUsing(tmpIndexPath, config) + if err != nil { + t.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + }() + + impl, ok := index.(*indexImpl) + if !ok { + t.Fatalf("Typecasting index to indexImpl failed") + } + sindex, ok := impl.i.(*scorch.Scorch) + if !ok { + t.Fatalf("Typecasting index to scorch index failed") + } + + err = sindex.ForceMerge(context.Background(), &mergeplan.SingleSegmentMergePlanOptions) + if err != nil { + t.Fatal(err) + } + + q1 := NewSearchRequest(NewQueryStringQuery("a:*")) + q1.Fields = append(q1.Fields, "a") + + res1, err := index.Search(q1) + if err != nil { + t.Fatal(err) + } + if len(res1.Hits) != 10 { + t.Errorf("Expected 10 hits, got %d\n", len(res1.Hits)) + } + if len(res1.Hits[0].Fields) != 1 { + t.Errorf("Expected 1 field, got %d\n", len(res1.Hits[0].Fields)) + } + q2 := NewSearchRequest(NewQueryStringQuery("b:*")) + q2.Fields = append(q2.Fields, "b") + res2, err := index.Search(q2) + if err != nil { + t.Fatal(err) + } + if len(res2.Hits) != 0 { + t.Errorf("Expected 0 hits, got %d\n", len(res2.Hits)) + } + q3 := NewSearchRequest(NewQueryStringQuery("c:*")) + q3.Fields = append(q3.Fields, "c") + res3, err := index.Search(q3) + if err != nil { + t.Fatal(err) + } + if len(res3.Hits) != 10 { + t.Errorf("Expected 10 hits, got %d\n", len(res3.Hits)) + } + if len(res3.Hits[0].Fields) != 0 { + t.Errorf("Expected 0 fields, got %d\n", len(res3.Hits[0].Fields)) + } + q4 := NewSearchRequest(NewQueryStringQuery("d:*")) + q4.Fields = append(q4.Fields, "d") + res4, err := index.Search(q4) + if err != nil { + t.Fatal(err) + } + if len(res4.Hits) != 0 { + t.Errorf("Expected 0 hits, got %d\n", len(res4.Hits)) + } +} + +func BenchmarkIndexUpdateText(b *testing.B) { + + tmpIndexPath := createTmpIndexPath(b) + defer cleanupTmpIndexPath(b, tmpIndexPath) + + indexMappingBefore := mapping.NewIndexMapping() + indexMappingBefore.TypeMapping = map[string]*mapping.DocumentMapping{} + indexMappingBefore.DefaultMapping = &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + } + indexMappingBefore.IndexDynamic = false + indexMappingBefore.StoreDynamic = false + indexMappingBefore.DocValuesDynamic = false + + index, err := New(tmpIndexPath, indexMappingBefore) + if err != nil { + b.Fatal(err) + } + + numDocsPerBatch := 1000 + numBatches := 5 + + var batch *Batch + doc := make(map[string]interface{}) + const letters = "abcdefghijklmnopqrstuvwxyz" + + randStr := func() string { + result := make([]byte, 3) + for i := 0; i < 3; i++ { + result[i] = letters[rand.Intn(len(letters))] + } + return string(result) + } + for i := 0; i < numBatches; i++ { + batch = index.NewBatch() + for j := 0; j < numDocsPerBatch; j++ { + doc["a"] = randStr() + err = batch.Index(fmt.Sprintf("%d", i*numDocsPerBatch+j), doc) + if err != nil { + b.Fatal(err) + } + } + err = index.Batch(batch) + if err != nil { + b.Fatal(err) + } + } + + err = index.Close() + if err != nil { + b.Fatal(err) + } + + indexMappingAfter := mapping.NewIndexMapping() + indexMappingAfter.TypeMapping = map[string]*mapping.DocumentMapping{} + indexMappingAfter.DefaultMapping = &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: false, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + } + indexMappingAfter.IndexDynamic = false + indexMappingAfter.StoreDynamic = false + indexMappingAfter.DocValuesDynamic = false + + mappingString, err := json.Marshal(indexMappingAfter) + if err != nil { + b.Fatal(err) + } + config := map[string]interface{}{ + "updated_mapping": string(mappingString), + } + + index, err = OpenUsing(tmpIndexPath, config) + if err != nil { + b.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + b.Fatal(err) + } + }() + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + q := NewQueryStringQuery("a:*") + req := NewSearchRequest(q) + if _, err = index.Search(req); err != nil { + b.Fatal(err) + } + } +} + +func TestIndexUpdateNestedMapping(t *testing.T) { + // Helper: create a mapping with optional nested structure + createCompanyMapping := func(nestedEmployees, nestedDepartments, nestedProjects, nestedLocations bool) *mapping.IndexMappingImpl { + rv := mapping.NewIndexMapping() + companyMapping := mapping.NewDocumentMapping() + + // Basic fields + companyMapping.AddFieldMappingsAt("id", mapping.NewTextFieldMapping()) + companyMapping.AddFieldMappingsAt("name", mapping.NewTextFieldMapping()) + + var deptMapping *mapping.DocumentMapping + // Departments nested conditionally + if !nestedDepartments { + deptMapping = mapping.NewDocumentMapping() + } else { + deptMapping = mapping.NewNestedDocumentMapping() + } + deptMapping.AddFieldMappingsAt("name", mapping.NewTextFieldMapping()) + deptMapping.AddFieldMappingsAt("budget", mapping.NewNumericFieldMapping()) + + // Employees nested conditionally + var empMapping *mapping.DocumentMapping + if !nestedEmployees { + empMapping = mapping.NewNestedDocumentMapping() + } else { + empMapping = mapping.NewDocumentMapping() + } + empMapping.AddFieldMappingsAt("name", mapping.NewTextFieldMapping()) + empMapping.AddFieldMappingsAt("role", mapping.NewTextFieldMapping()) + deptMapping.AddSubDocumentMapping("employees", empMapping) + + // Projects nested conditionally + var projMapping *mapping.DocumentMapping + if !nestedProjects { + projMapping = mapping.NewNestedDocumentMapping() + } else { + projMapping = mapping.NewDocumentMapping() + } + projMapping.AddFieldMappingsAt("title", mapping.NewTextFieldMapping()) + projMapping.AddFieldMappingsAt("status", mapping.NewTextFieldMapping()) + deptMapping.AddSubDocumentMapping("projects", projMapping) + + companyMapping.AddSubDocumentMapping("departments", deptMapping) + + // Locations nested conditionally + var locMapping *mapping.DocumentMapping + if nestedLocations { + locMapping = mapping.NewNestedDocumentMapping() + } else { + locMapping = mapping.NewDocumentMapping() + } + locMapping.AddFieldMappingsAt("address", mapping.NewTextFieldMapping()) + locMapping.AddFieldMappingsAt("city", mapping.NewTextFieldMapping()) + + companyMapping.AddSubDocumentMapping("locations", locMapping) + + rv.DefaultMapping.AddSubDocumentMapping("company", companyMapping) + return rv + } + + tests := []struct { + name string + original *mapping.IndexMappingImpl + updated *mapping.IndexMappingImpl + expectErr bool + }{ + { + name: "No nested to all nested", + original: createCompanyMapping(false, false, false, false), + updated: createCompanyMapping(true, true, true, true), + expectErr: true, + }, + { + name: "No nested to mixed nested", + original: createCompanyMapping(false, false, false, false), + updated: createCompanyMapping(true, false, true, false), + expectErr: true, + }, + { + name: "No nested to mixed nested", + original: createCompanyMapping(false, false, false, false), + updated: createCompanyMapping(true, true, true, false), + expectErr: true, + }, + { + name: "Mixed nested to no nested", + original: createCompanyMapping(false, true, false, true), + updated: createCompanyMapping(false, false, true, true), + expectErr: true, + }, + { + name: "All nested to no nested", + original: createCompanyMapping(true, true, true, true), + updated: createCompanyMapping(false, false, false, false), + expectErr: true, + }, + { + name: "Mixed nested to all nested", + original: createCompanyMapping(true, false, true, false), + updated: createCompanyMapping(true, true, true, true), + expectErr: true, + }, + { + name: "All nested to mixed nested", + original: createCompanyMapping(true, true, true, true), + updated: createCompanyMapping(true, false, true, false), + expectErr: true, + }, + { + name: "No nested to no nested", + original: createCompanyMapping(false, false, false, false), + updated: createCompanyMapping(false, false, false, false), + expectErr: false, + }, + { + name: "All nested to all nested", + original: createCompanyMapping(true, true, true, true), + updated: createCompanyMapping(true, true, true, true), + expectErr: false, + }, + } + + for _, test := range tests { + _, err := DeletedFields(test.original, test.updated) + if (err != nil) != test.expectErr { + t.Errorf("Test '%s' unexpected error state: got %v, expectErr %t", test.name, err, test.expectErr) + } + } +} diff --git a/mapping.go b/mapping.go new file mode 100644 index 0000000..ba77f15 --- /dev/null +++ b/mapping.go @@ -0,0 +1,97 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import "github.com/blevesearch/bleve/v2/mapping" + +// NewIndexMapping creates a new IndexMapping that will use all the default indexing rules +func NewIndexMapping() *mapping.IndexMappingImpl { + return mapping.NewIndexMapping() +} + +// NewDocumentMapping returns a new document mapping +// with all the default values. +func NewDocumentMapping() *mapping.DocumentMapping { + return mapping.NewDocumentMapping() +} + +// NewDocumentStaticMapping returns a new document +// mapping that will not automatically index parts +// of a document without an explicit mapping. +func NewDocumentStaticMapping() *mapping.DocumentMapping { + return mapping.NewDocumentStaticMapping() +} + +// NewNestedDocumentMapping returns a new document mapping +// that will treat all objects as nested documents. +func NewNestedDocumentMapping() *mapping.DocumentMapping { + return mapping.NewNestedDocumentMapping() +} + +// NewNestedDocumentStaticMapping returns a new document mapping +// that will treat all objects as nested documents and +// will not automatically index parts of a nested document +// without an explicit mapping. +func NewNestedDocumentStaticMapping() *mapping.DocumentMapping { + return mapping.NewNestedDocumentStaticMapping() +} + +// NewDocumentDisabledMapping returns a new document +// mapping that will not perform any indexing. +func NewDocumentDisabledMapping() *mapping.DocumentMapping { + return mapping.NewDocumentDisabledMapping() +} + +// NewTextFieldMapping returns a default field mapping for text +func NewTextFieldMapping() *mapping.FieldMapping { + return mapping.NewTextFieldMapping() +} + +// NewKeywordFieldMapping returns a field mapping for text using the keyword +// analyzer, which essentially doesn't apply any specific text analysis. +func NewKeywordFieldMapping() *mapping.FieldMapping { + return mapping.NewKeywordFieldMapping() +} + +// NewNumericFieldMapping returns a default field mapping for numbers +func NewNumericFieldMapping() *mapping.FieldMapping { + return mapping.NewNumericFieldMapping() +} + +// NewDateTimeFieldMapping returns a default field mapping for dates +func NewDateTimeFieldMapping() *mapping.FieldMapping { + return mapping.NewDateTimeFieldMapping() +} + +// NewBooleanFieldMapping returns a default field mapping for booleans +func NewBooleanFieldMapping() *mapping.FieldMapping { + return mapping.NewBooleanFieldMapping() +} + +func NewGeoPointFieldMapping() *mapping.FieldMapping { + return mapping.NewGeoPointFieldMapping() +} + +func NewGeoShapeFieldMapping() *mapping.FieldMapping { + return mapping.NewGeoShapeFieldMapping() +} + +func NewGeoShapeV2FieldMapping() *mapping.FieldMapping { + return mapping.NewGeoShapeV2FieldMapping() +} + +func NewIPFieldMapping() *mapping.FieldMapping { + return mapping.NewIPFieldMapping() +} diff --git a/mapping/analysis.go b/mapping/analysis.go new file mode 100644 index 0000000..311e972 --- /dev/null +++ b/mapping/analysis.go @@ -0,0 +1,107 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapping + +type customAnalysis struct { + CharFilters map[string]map[string]interface{} `json:"char_filters,omitempty"` + Tokenizers map[string]map[string]interface{} `json:"tokenizers,omitempty"` + TokenMaps map[string]map[string]interface{} `json:"token_maps,omitempty"` + TokenFilters map[string]map[string]interface{} `json:"token_filters,omitempty"` + Analyzers map[string]map[string]interface{} `json:"analyzers,omitempty"` + DateTimeParsers map[string]map[string]interface{} `json:"date_time_parsers,omitempty"` + SynonymSources map[string]map[string]interface{} `json:"synonym_sources,omitempty"` +} + +func (c *customAnalysis) registerAll(i *IndexMappingImpl) error { + for name, config := range c.CharFilters { + _, err := i.cache.DefineCharFilter(name, config) + if err != nil { + return err + } + } + + if len(c.Tokenizers) > 0 { + // put all the names in map tracking work to do + todo := map[string]struct{}{} + for name := range c.Tokenizers { + todo[name] = struct{}{} + } + registered := 1 + errs := []error{} + // as long as we keep making progress, keep going + for len(todo) > 0 && registered > 0 { + registered = 0 + errs = []error{} + for name := range todo { + config := c.Tokenizers[name] + _, err := i.cache.DefineTokenizer(name, config) + if err != nil { + errs = append(errs, err) + } else { + delete(todo, name) + registered++ + } + } + } + + if len(errs) > 0 { + return errs[0] + } + } + for name, config := range c.TokenMaps { + _, err := i.cache.DefineTokenMap(name, config) + if err != nil { + return err + } + } + for name, config := range c.TokenFilters { + _, err := i.cache.DefineTokenFilter(name, config) + if err != nil { + return err + } + } + for name, config := range c.Analyzers { + _, err := i.cache.DefineAnalyzer(name, config) + if err != nil { + return err + } + } + for name, config := range c.DateTimeParsers { + _, err := i.cache.DefineDateTimeParser(name, config) + if err != nil { + return err + } + } + for name, config := range c.SynonymSources { + _, err := i.cache.DefineSynonymSource(name, config) + if err != nil { + return err + } + } + return nil +} + +func newCustomAnalysis() *customAnalysis { + rv := customAnalysis{ + CharFilters: make(map[string]map[string]interface{}), + Tokenizers: make(map[string]map[string]interface{}), + TokenMaps: make(map[string]map[string]interface{}), + TokenFilters: make(map[string]map[string]interface{}), + Analyzers: make(map[string]map[string]interface{}), + DateTimeParsers: make(map[string]map[string]interface{}), + SynonymSources: make(map[string]map[string]interface{}), + } + return &rv +} diff --git a/mapping/document.go b/mapping/document.go new file mode 100644 index 0000000..943056b --- /dev/null +++ b/mapping/document.go @@ -0,0 +1,718 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapping + +import ( + "encoding" + "encoding/json" + "fmt" + "net" + "reflect" + "time" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/util" +) + +// A DocumentMapping describes how a type of document +// should be indexed. +// As documents can be hierarchical, named sub-sections +// of documents are mapped using the same structure in +// the Properties field. +// Each value inside a document can be indexed 0 or more +// ways. These index entries are called fields and +// are stored in the Fields field. +// Entire sections of a document can be ignored or +// excluded by setting Enabled to false. +// If not explicitly mapped, default mapping operations +// are used. To disable this automatic handling, set +// Dynamic to false. +type DocumentMapping struct { + Enabled bool `json:"enabled"` + Dynamic bool `json:"dynamic"` + Properties map[string]*DocumentMapping `json:"properties,omitempty"` + Fields []*FieldMapping `json:"fields,omitempty"` + Nested bool `json:"nested,omitempty"` + DefaultAnalyzer string `json:"default_analyzer,omitempty"` + DefaultSynonymSource string `json:"default_synonym_source,omitempty"` + + // StructTagKey overrides "json" when looking for field names in struct tags + StructTagKey string `json:"struct_tag_key,omitempty"` +} + +func (dm *DocumentMapping) Validate(cache *registry.Cache, + path []string, fieldAliasCtx map[string]*FieldMapping, +) error { + var err error + if dm.DefaultAnalyzer != "" { + _, err := cache.AnalyzerNamed(dm.DefaultAnalyzer) + if err != nil { + return err + } + } + if dm.DefaultSynonymSource != "" { + _, err := cache.SynonymSourceNamed(dm.DefaultSynonymSource) + if err != nil { + return err + } + } + for propertyName, property := range dm.Properties { + err = property.Validate(cache, append(path, propertyName), fieldAliasCtx) + if err != nil { + return err + } + } + for _, field := range dm.Fields { + if field.Analyzer != "" { + _, err = cache.AnalyzerNamed(field.Analyzer) + if err != nil { + return err + } + } + if field.DateFormat != "" { + _, err = cache.DateTimeParserNamed(field.DateFormat) + if err != nil { + return err + } + } + if field.SynonymSource != "" { + _, err = cache.SynonymSourceNamed(field.SynonymSource) + if err != nil { + return err + } + } + err := validateFieldMapping(field, path, fieldAliasCtx) + if err != nil { + return err + } + } + return nil +} + +func validateFieldType(field *FieldMapping) error { + switch field.Type { + case "text", "datetime", "number", "boolean", "geopoint", "geoshape", "geoshape_v2", "IP": + return nil + default: + return fmt.Errorf("field: '%s', unknown field type: '%s'", + field.Name, field.Type) + } +} + +// analyzerNameForPath attempts to first find the field +// described by this path, then returns the analyzer +// configured for that field +func (dm *DocumentMapping) analyzerNameForPath(path string) string { + field := dm.fieldDescribedByPath(path) + if field != nil { + return field.Analyzer + } + return "" +} + +// synonymSourceForPath attempts to first find the field +// described by this path, then returns the analyzer +// configured for that field +func (dm *DocumentMapping) synonymSourceForPath(path string) string { + field := dm.fieldDescribedByPath(path) + if field != nil { + return field.SynonymSource + } + return "" +} + +func (dm *DocumentMapping) fieldDescribedByPath(path string) *FieldMapping { + pathElements := decodePath(path) + if len(pathElements) > 1 { + // easy case, there is more than 1 path element remaining + // the next path element must match a property name + // at this level + for propName, subDocMapping := range dm.Properties { + if propName == pathElements[0] { + return subDocMapping.fieldDescribedByPath(encodePath(pathElements[1:])) + } + } + } + + // either the path just had one element + // or it had multiple, but no match for the first element at this level + // look for match with full path + + // first look for property name with empty field + for propName, subDocMapping := range dm.Properties { + if propName == path { + // found property name match, now look at its fields + for _, field := range subDocMapping.Fields { + if field.Name == "" || field.Name == path { + // match + return field + } + } + } + } + // next, walk the properties again, looking for field overriding the name + for propName, subDocMapping := range dm.Properties { + if propName != path { + // property name isn't a match, but field name could override it + for _, field := range subDocMapping.Fields { + if field.Name == path { + return field + } + } + } + } + + return nil +} + +// documentMappingForPathElements returns the EXACT and closest matches for a sub +// document or for an explicitly mapped field; the closest most specific +// document mapping could be one that matches part of the provided path. +func (dm *DocumentMapping) documentMappingForPathElements(pathElements []string) ( + *DocumentMapping, *DocumentMapping, +) { + var pathElementsCopy []string + if len(pathElements) == 0 { + pathElementsCopy = []string{""} + } else { + pathElementsCopy = pathElements + } + current := dm +OUTER: + for i, pathElement := range pathElementsCopy { + if subDocMapping, exists := current.Properties[pathElement]; exists { + current = subDocMapping + continue OUTER + } + + // no subDocMapping matches this pathElement + // only if this is the last element check for field name + if i == len(pathElementsCopy)-1 { + for _, field := range current.Fields { + if field.Name == pathElement { + break + } + } + } + + return nil, current + } + return current, current +} + +// documentMappingForPath returns the EXACT and closest matches for a sub +// document or for an explicitly mapped field; the closest most specific +// document mapping could be one that matches part of the provided path. +func (dm *DocumentMapping) documentMappingForPath(path string) ( + *DocumentMapping, *DocumentMapping, +) { + pathElements := decodePath(path) + return dm.documentMappingForPathElements(pathElements) +} + +// NewDocumentMapping returns a new document mapping +// with all the default values. +func NewDocumentMapping() *DocumentMapping { + return &DocumentMapping{ + Enabled: true, + Dynamic: true, + } +} + +// NewNestedDocumentMapping returns a new document +// mapping that treats sub-documents as nested +// objects. +func NewNestedDocumentMapping() *DocumentMapping { + return &DocumentMapping{ + Nested: true, + Enabled: true, + Dynamic: true, + } +} + +// NewDocumentStaticMapping returns a new document +// mapping that will not automatically index parts +// of a document without an explicit mapping. +func NewDocumentStaticMapping() *DocumentMapping { + return &DocumentMapping{ + Enabled: true, + } +} + +// NewNestedDocumentStaticMapping returns a new document +// mapping that treats sub-documents as nested +// objects and will not automatically index parts +// of the nested document without an explicit mapping. +func NewNestedDocumentStaticMapping() *DocumentMapping { + return &DocumentMapping{ + Enabled: true, + Nested: true, + } +} + +// NewDocumentDisabledMapping returns a new document +// mapping that will not perform any indexing. +func NewDocumentDisabledMapping() *DocumentMapping { + return &DocumentMapping{} +} + +// AddSubDocumentMapping adds the provided DocumentMapping as a sub-mapping +// for the specified named subsection. +func (dm *DocumentMapping) AddSubDocumentMapping(property string, sdm *DocumentMapping) { + if dm.Properties == nil { + dm.Properties = make(map[string]*DocumentMapping) + } + dm.Properties[property] = sdm +} + +// AddFieldMappingsAt adds one or more FieldMappings +// at the named sub-document. If the named sub-document +// doesn't yet exist it is created for you. +// This is a convenience function to make most common +// mappings more concise. +// Otherwise, you would: +// +// subMapping := NewDocumentMapping() +// subMapping.AddFieldMapping(fieldMapping) +// parentMapping.AddSubDocumentMapping(property, subMapping) +func (dm *DocumentMapping) AddFieldMappingsAt(property string, fms ...*FieldMapping) { + if dm.Properties == nil { + dm.Properties = make(map[string]*DocumentMapping) + } + sdm, ok := dm.Properties[property] + if !ok { + sdm = NewDocumentMapping() + } + for _, fm := range fms { + sdm.AddFieldMapping(fm) + } + dm.Properties[property] = sdm +} + +// AddFieldMapping adds the provided FieldMapping for this section +// of the document. +func (dm *DocumentMapping) AddFieldMapping(fm *FieldMapping) { + if dm.Fields == nil { + dm.Fields = make([]*FieldMapping, 0) + } + dm.Fields = append(dm.Fields, fm) +} + +// UnmarshalJSON offers custom unmarshaling with optional strict validation +func (dm *DocumentMapping) UnmarshalJSON(data []byte) error { + var tmp map[string]json.RawMessage + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return err + } + + // set defaults for fields which might have been omitted + dm.Enabled = true + dm.Dynamic = true + + var invalidKeys []string + for k, v := range tmp { + switch k { + case "enabled": + err := util.UnmarshalJSON(v, &dm.Enabled) + if err != nil { + return err + } + case "dynamic": + err := util.UnmarshalJSON(v, &dm.Dynamic) + if err != nil { + return err + } + case "nested": + err := util.UnmarshalJSON(v, &dm.Nested) + if err != nil { + return err + } + case "default_analyzer": + err := util.UnmarshalJSON(v, &dm.DefaultAnalyzer) + if err != nil { + return err + } + case "default_synonym_source": + err := util.UnmarshalJSON(v, &dm.DefaultSynonymSource) + if err != nil { + return err + } + case "properties": + err := util.UnmarshalJSON(v, &dm.Properties) + if err != nil { + return err + } + case "fields": + err := util.UnmarshalJSON(v, &dm.Fields) + if err != nil { + return err + } + case "struct_tag_key": + err := util.UnmarshalJSON(v, &dm.StructTagKey) + if err != nil { + return err + } + default: + invalidKeys = append(invalidKeys, k) + } + } + + if MappingJSONStrict && len(invalidKeys) > 0 { + return fmt.Errorf("document mapping contains invalid keys: %v", invalidKeys) + } + + return nil +} + +func (dm *DocumentMapping) defaultAnalyzerName(path []string) string { + current := dm + rv := current.DefaultAnalyzer + for _, pathElement := range path { + var ok bool + current, ok = current.Properties[pathElement] + if !ok { + break + } + if current.DefaultAnalyzer != "" { + rv = current.DefaultAnalyzer + } + } + return rv +} + +func (dm *DocumentMapping) defaultSynonymSource(path []string) string { + current := dm + rv := current.DefaultSynonymSource + for _, pathElement := range path { + var ok bool + current, ok = current.Properties[pathElement] + if !ok { + break + } + if current.DefaultSynonymSource != "" { + rv = current.DefaultSynonymSource + } + } + return rv +} + +// baseType returns the base type of v by dereferencing pointers +func baseType(v interface{}) reflect.Type { + if v == nil { + return nil + } + t := reflect.TypeOf(v) + for t.Kind() == reflect.Pointer { + t = t.Elem() + } + return t +} + +func (dm *DocumentMapping) walkDocument(data interface{}, path []string, indexes []uint64, context *walkContext) { + // allow default "json" tag to be overridden + structTagKey := dm.StructTagKey + if structTagKey == "" { + structTagKey = "json" + } + + val := reflect.ValueOf(data) + if !val.IsValid() { + return + } + + typ := val.Type() + switch typ.Kind() { + case reflect.Map: + // FIXME can add support for other map keys in the future + if typ.Key().Kind() == reflect.String { + for _, key := range val.MapKeys() { + fieldName := key.String() + fieldVal := val.MapIndex(key).Interface() + dm.processProperty(fieldVal, append(path, fieldName), indexes, context) + } + } + case reflect.Struct: + for i := 0; i < val.NumField(); i++ { + field := typ.Field(i) + fieldName := field.Name + // anonymous fields of type struct can elide the type name + if field.Anonymous && field.Type.Kind() == reflect.Struct { + fieldName = "" + } + + // if the field has a name under the specified tag, prefer that + tag := field.Tag.Get(structTagKey) + tagFieldName := parseTagName(tag) + if tagFieldName == "-" { + continue + } + // allow tag to set field name to empty, only if anonymous + if field.Tag != "" && (tagFieldName != "" || field.Anonymous) { + fieldName = tagFieldName + } + + if val.Field(i).CanInterface() { + fieldVal := val.Field(i).Interface() + newpath := path + if fieldName != "" { + newpath = append(path, fieldName) + } + dm.processProperty(fieldVal, newpath, indexes, context) + } + } + case reflect.Slice, reflect.Array: + subDocMapping, _ := dm.documentMappingForPathElements(path) + allowNested := subDocMapping != nil && subDocMapping.Nested + for i := 0; i < val.Len(); i++ { + // for each array element, check if it can be represented as an interface + idxVal := val.Index(i) + // skip invalid values + if !idxVal.CanInterface() { + continue + } + // get the actual value in interface form + actual := idxVal.Interface() + // if nested mapping, only create nested document for object elements + if allowNested && actual != nil { + // check the kind of the actual value, is it an object (struct or map)? + typ := baseType(actual) + if typ == nil { + continue + } + kind := typ.Kind() + // only create nested docs for real JSON objects + if kind == reflect.Struct || kind == reflect.Map { + // Create nested document only for only object elements + nestedDocument := document.NewDocument( + fmt.Sprintf("%s_$%s_$%d", context.doc.ID(), encodePath(path), i)) + nestedContext := context.im.newWalkContext(nestedDocument, dm) + dm.processProperty(actual, path, append(indexes, uint64(i)), nestedContext) + context.doc.AddNestedDocument(nestedDocument) + continue + } + } + // non-nested mapping, or non-object element in nested mapping + // process the element normally + dm.processProperty(actual, path, append(indexes, uint64(i)), context) + } + case reflect.Ptr: + ptrElem := val.Elem() + if ptrElem.IsValid() && ptrElem.CanInterface() { + dm.processProperty(ptrElem.Interface(), path, indexes, context) + } + case reflect.String: + dm.processProperty(val.String(), path, indexes, context) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + dm.processProperty(float64(val.Int()), path, indexes, context) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + dm.processProperty(float64(val.Uint()), path, indexes, context) + case reflect.Float32, reflect.Float64: + dm.processProperty(float64(val.Float()), path, indexes, context) + case reflect.Bool: + dm.processProperty(val.Bool(), path, indexes, context) + } +} + +func (dm *DocumentMapping) processProperty(property interface{}, path []string, indexes []uint64, context *walkContext) { + // look to see if there is a mapping for this field + subDocMapping, closestDocMapping := dm.documentMappingForPathElements(path) + + // check to see if we even need to do further processing + if subDocMapping != nil && !subDocMapping.Enabled { + return + } + + propertyValue := reflect.ValueOf(property) + if !propertyValue.IsValid() { + // cannot do anything with the zero value + return + } + + pathString := encodePath(path) + propertyType := propertyValue.Type() + switch propertyType.Kind() { + case reflect.String: + propertyValueString := propertyValue.String() + if subDocMapping != nil { + // index by explicit mapping + for _, fieldMapping := range subDocMapping.Fields { + switch fieldMapping.Type { + case "geoshape": + fieldMapping.processGeoShape(property, pathString, path, indexes, context) + case "geopoint": + fieldMapping.processGeoPoint(property, pathString, path, indexes, context) + case "vector_base64": + fieldMapping.processVectorBase64(property, pathString, path, indexes, context) + case "geoshape_v2": + fieldMapping.processGeoShapeV2(property, pathString, path, context) + default: + fieldMapping.processString(propertyValueString, pathString, path, indexes, context) + } + } + } else if closestDocMapping.Dynamic { + // automatic indexing behavior + + // first see if it can be parsed by the default date parser + dateTimeParser := context.im.DateTimeParserNamed(context.im.DefaultDateTimeParser) + if dateTimeParser != nil { + parsedDateTime, layout, err := dateTimeParser.ParseDateTime(propertyValueString) + if err != nil { + // index as text + fieldMapping := newTextFieldMappingDynamic(context.im) + fieldMapping.processString(propertyValueString, pathString, path, indexes, context) + } else { + // index as datetime + fieldMapping := newDateTimeFieldMappingDynamic(context.im) + fieldMapping.processTime(parsedDateTime, layout, pathString, path, indexes, context) + } + } + } + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + dm.processProperty(float64(propertyValue.Int()), path, indexes, context) + return + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + dm.processProperty(float64(propertyValue.Uint()), path, indexes, context) + return + case reflect.Float64, reflect.Float32: + propertyValFloat := propertyValue.Float() + if subDocMapping != nil { + // index by explicit mapping + for _, fieldMapping := range subDocMapping.Fields { + fieldMapping.processFloat64(propertyValFloat, pathString, path, indexes, context) + } + } else if closestDocMapping.Dynamic { + // automatic indexing behavior + fieldMapping := newNumericFieldMappingDynamic(context.im) + fieldMapping.processFloat64(propertyValFloat, pathString, path, indexes, context) + } + case reflect.Bool: + propertyValBool := propertyValue.Bool() + if subDocMapping != nil { + // index by explicit mapping + for _, fieldMapping := range subDocMapping.Fields { + fieldMapping.processBoolean(propertyValBool, pathString, path, indexes, context) + } + } else if closestDocMapping.Dynamic { + // automatic indexing behavior + fieldMapping := newBooleanFieldMappingDynamic(context.im) + fieldMapping.processBoolean(propertyValBool, pathString, path, indexes, context) + } + case reflect.Struct: + switch property := property.(type) { + case time.Time: + // don't descend into the time struct + if subDocMapping != nil { + // index by explicit mapping + for _, fieldMapping := range subDocMapping.Fields { + fieldMapping.processTime(property, time.RFC3339, pathString, path, indexes, context) + } + } else if closestDocMapping.Dynamic { + fieldMapping := newDateTimeFieldMappingDynamic(context.im) + fieldMapping.processTime(property, time.RFC3339, pathString, path, indexes, context) + } + case encoding.TextMarshaler: + txt, err := property.MarshalText() + if err == nil && subDocMapping != nil { + // index by explicit mapping + for _, fieldMapping := range subDocMapping.Fields { + if fieldMapping.Type == "text" { + fieldMapping.processString(string(txt), pathString, path, indexes, context) + } + } + } + dm.walkDocument(property, path, indexes, context) + default: + if subDocMapping != nil { + for _, fieldMapping := range subDocMapping.Fields { + switch fieldMapping.Type { + case "geopoint": + fieldMapping.processGeoPoint(property, pathString, path, indexes, context) + case "geoshape": + fieldMapping.processGeoShape(property, pathString, path, indexes, context) + case "geoshape_v2": + fieldMapping.processGeoShapeV2(property, pathString, path, context) + } + } + } + dm.walkDocument(property, path, indexes, context) + } + case reflect.Map, reflect.Slice: + walkDocument := false + if subDocMapping != nil && len(subDocMapping.Fields) != 0 { + for _, fieldMapping := range subDocMapping.Fields { + switch fieldMapping.Type { + case "vector": + fieldMapping.processVector(property, pathString, path, + indexes, context) + case "geopoint": + fieldMapping.processGeoPoint(property, pathString, path, indexes, context) + walkDocument = true + case "IP": + ip, ok := property.(net.IP) + if ok { + fieldMapping.processIP(ip, pathString, path, indexes, context) + } + walkDocument = true + case "geoshape": + fieldMapping.processGeoShape(property, pathString, path, indexes, context) + walkDocument = true + case "geoshape_v2": + fieldMapping.processGeoShapeV2(property, pathString, path, context) + walkDocument = true + default: + walkDocument = true + } + } + } else { + walkDocument = true + } + if walkDocument { + dm.walkDocument(property, path, indexes, context) + } + case reflect.Ptr: + if !propertyValue.IsNil() { + switch property := property.(type) { + case encoding.TextMarshaler: + // ONLY process TextMarshaler if there is an explicit mapping + // AND all of the fields are of type text + // OTHERWISE process field without TextMarshaler + if subDocMapping != nil { + allFieldsText := true + for _, fieldMapping := range subDocMapping.Fields { + if fieldMapping.Type != "text" { + allFieldsText = false + break + } + } + txt, err := property.MarshalText() + if err == nil && allFieldsText { + txtStr := string(txt) + for _, fieldMapping := range subDocMapping.Fields { + fieldMapping.processString(txtStr, pathString, path, indexes, context) + } + return + } + } + dm.walkDocument(property, path, indexes, context) + default: + dm.walkDocument(property, path, indexes, context) + } + } + default: + dm.walkDocument(property, path, indexes, context) + } +} diff --git a/mapping/examples_test.go b/mapping/examples_test.go new file mode 100644 index 0000000..c45e1ae --- /dev/null +++ b/mapping/examples_test.go @@ -0,0 +1,61 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapping + +import "fmt" + +// Examples for Mapping related functions + +func ExampleDocumentMapping_AddSubDocumentMapping() { + // adds a document mapping for a property in a document + // useful for mapping nested documents + documentMapping := NewDocumentMapping() + subDocumentMapping := NewDocumentMapping() + documentMapping.AddSubDocumentMapping("Property", subDocumentMapping) + + fmt.Println(len(documentMapping.Properties)) + // Output: + // 1 +} + +func ExampleDocumentMapping_AddFieldMapping() { + // you can only add field mapping to those properties which already have a document mapping + documentMapping := NewDocumentMapping() + subDocumentMapping := NewDocumentMapping() + documentMapping.AddSubDocumentMapping("Property", subDocumentMapping) + + fieldMapping := NewTextFieldMapping() + fieldMapping.Analyzer = "en" + subDocumentMapping.AddFieldMapping(fieldMapping) + + fmt.Println(len(documentMapping.Properties["Property"].Fields)) + // Output: + // 1 +} + +func ExampleDocumentMapping_AddFieldMappingsAt() { + // you can only add field mapping to those properties which already have a document mapping + documentMapping := NewDocumentMapping() + subDocumentMapping := NewDocumentMapping() + documentMapping.AddSubDocumentMapping("NestedProperty", subDocumentMapping) + + fieldMapping := NewTextFieldMapping() + fieldMapping.Analyzer = "en" + documentMapping.AddFieldMappingsAt("NestedProperty", fieldMapping) + + fmt.Println(len(documentMapping.Properties["NestedProperty"].Fields)) + // Output: + // 1 +} diff --git a/mapping/field.go b/mapping/field.go new file mode 100644 index 0000000..bf8d4d4 --- /dev/null +++ b/mapping/field.go @@ -0,0 +1,580 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapping + +import ( + "encoding/json" + "fmt" + "net" + "time" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/geo/geojson" +) + +// control the default behavior for dynamic fields (those not explicitly mapped) +var ( + IndexDynamic = true + StoreDynamic = true + DocValuesDynamic = true // TODO revisit default? +) + +// A FieldMapping describes how a specific item +// should be put into the index. +type FieldMapping struct { + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + + // Analyzer specifies the name of the analyzer to use for this field. If + // Analyzer is empty, traverse the DocumentMapping tree toward the root and + // pick the first non-empty DefaultAnalyzer found. If there is none, use + // the IndexMapping.DefaultAnalyzer. + Analyzer string `json:"analyzer,omitempty"` + + // Store indicates whether to store field values in the index. Stored + // values can be retrieved from search results using SearchRequest.Fields. + Store bool `json:"store,omitempty"` + Index bool `json:"index,omitempty"` + + // IncludeTermVectors, if true, makes terms occurrences to be recorded for + // this field. It includes the term position within the terms sequence and + // the term offsets in the source document field. Term vectors are required + // to perform phrase queries or terms highlighting in source documents. + IncludeTermVectors bool `json:"include_term_vectors,omitempty"` + IncludeInAll bool `json:"include_in_all,omitempty"` + DateFormat string `json:"date_format,omitempty"` + + // DocValues, if true makes the index uninverting possible for this field + // It is useful for faceting and sorting queries. + DocValues bool `json:"docvalues,omitempty"` + + // SkipFreqNorm, if true, avoids the indexing of frequency and norm values + // of the tokens for this field. This option would be useful for saving + // the processing of freq/norm details when the default score based relevancy + // isn't needed. + SkipFreqNorm bool `json:"skip_freq_norm,omitempty"` + + // Dimensionality of the vector + Dims int `json:"dims,omitempty"` + + // Similarity is the similarity algorithm used for scoring + // field's content while performing search on it. + // See: index.SimilarityModels + Similarity string `json:"similarity,omitempty"` + + // Applicable to vector fields only - optimization string + VectorIndexOptimizedFor string `json:"vector_index_optimized_for,omitempty"` + + SynonymSource string `json:"synonym_source,omitempty"` + + // Applicable to vector fields only - enables GPU acceleration for indexing and searching + GPU bool `json:"gpu,omitempty"` +} + +// NewTextFieldMapping returns a default field mapping for text +func NewTextFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "text", + Store: true, + Index: true, + IncludeTermVectors: true, + IncludeInAll: true, + DocValues: true, + } +} + +func newTextFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { + rv := NewTextFieldMapping() + rv.Store = im.StoreDynamic + rv.Index = im.IndexDynamic + rv.DocValues = im.DocValuesDynamic + return rv +} + +// NewKeywordFieldMapping returns a default field mapping for text with analyzer "keyword". +func NewKeywordFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "text", + Analyzer: keyword.Name, + Store: true, + Index: true, + IncludeTermVectors: true, + IncludeInAll: true, + DocValues: true, + } +} + +// NewNumericFieldMapping returns a default field mapping for numbers +func NewNumericFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "number", + Store: true, + Index: true, + IncludeInAll: true, + DocValues: true, + } +} + +func newNumericFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { + rv := NewNumericFieldMapping() + rv.Store = im.StoreDynamic + rv.Index = im.IndexDynamic + rv.DocValues = im.DocValuesDynamic + return rv +} + +// NewDateTimeFieldMapping returns a default field mapping for dates +func NewDateTimeFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "datetime", + Store: true, + Index: true, + IncludeInAll: true, + DocValues: true, + } +} + +func newDateTimeFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { + rv := NewDateTimeFieldMapping() + rv.Store = im.StoreDynamic + rv.Index = im.IndexDynamic + rv.DocValues = im.DocValuesDynamic + return rv +} + +// NewBooleanFieldMapping returns a default field mapping for booleans +func NewBooleanFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "boolean", + Store: true, + Index: true, + IncludeInAll: true, + DocValues: true, + } +} + +func newBooleanFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { + rv := NewBooleanFieldMapping() + rv.Store = im.StoreDynamic + rv.Index = im.IndexDynamic + rv.DocValues = im.DocValuesDynamic + return rv +} + +// NewGeoPointFieldMapping returns a default field mapping for geo points +func NewGeoPointFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "geopoint", + Store: true, + Index: true, + IncludeInAll: true, + DocValues: true, + } +} + +// NewGeoShapeFieldMapping returns a default field mapping +// for geoshapes +func NewGeoShapeFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "geoshape", + Store: true, + Index: true, + IncludeInAll: true, + DocValues: true, + } +} + +// NewGeoShapeV2FieldMapping returns a default field mapping for +// geoshapes using the new GeoShapeV2 format +func NewGeoShapeV2FieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "geoshape_v2", + Store: false, + Index: true, + IncludeInAll: false, + DocValues: false, + } +} + +// NewIPFieldMapping returns a default field mapping for IP points +func NewIPFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "IP", + Store: true, + Index: true, + IncludeInAll: true, + } +} + +// Options returns the indexing options for this field. +func (fm *FieldMapping) Options() index.FieldIndexingOptions { + var rv index.FieldIndexingOptions + if fm.Store { + rv |= index.StoreField + } + if fm.Index { + rv |= index.IndexField + } + if fm.IncludeTermVectors { + rv |= index.IncludeTermVectors + } + if fm.DocValues { + rv |= index.DocValues + } + if fm.SkipFreqNorm { + rv |= index.SkipFreqNorm + } + if fm.GPU { + rv |= index.GPU + } + return rv +} + +func (fm *FieldMapping) processString(propertyValueString string, pathString string, path []string, indexes []uint64, context *walkContext) { + fieldName := getFieldName(pathString, path, fm) + options := fm.Options() + + switch fm.Type { + case "text": + analyzer := fm.analyzerForField(path, context) + field := document.NewTextFieldCustom(fieldName, indexes, []byte(propertyValueString), options, analyzer) + context.doc.AddField(field) + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } + case "datetime": + dateTimeFormat := context.im.DefaultDateTimeParser + if fm.DateFormat != "" { + dateTimeFormat = fm.DateFormat + } + dateTimeParser := context.im.DateTimeParserNamed(dateTimeFormat) + if dateTimeParser != nil { + parsedDateTime, layout, err := dateTimeParser.ParseDateTime(propertyValueString) + if err == nil { + fm.processTime(parsedDateTime, layout, pathString, path, indexes, context) + } + } + case "IP": + ip := net.ParseIP(propertyValueString) + if ip != nil { + fm.processIP(ip, pathString, path, indexes, context) + } + } +} + +func (fm *FieldMapping) processFloat64(propertyValFloat float64, pathString string, path []string, indexes []uint64, context *walkContext) { + fieldName := getFieldName(pathString, path, fm) + if fm.Type == "number" { + options := fm.Options() + field := document.NewNumericFieldWithIndexingOptions(fieldName, indexes, propertyValFloat, options) + context.doc.AddField(field) + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } + } +} + +func (fm *FieldMapping) processTime(propertyValueTime time.Time, layout string, pathString string, path []string, indexes []uint64, context *walkContext) { + fieldName := getFieldName(pathString, path, fm) + if fm.Type == "datetime" { + options := fm.Options() + field, err := document.NewDateTimeFieldWithIndexingOptions(fieldName, indexes, propertyValueTime, layout, options) + if err == nil { + context.doc.AddField(field) + } else { + logger.Printf("could not build date %v", err) + } + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } + } +} + +func (fm *FieldMapping) processBoolean(propertyValueBool bool, pathString string, path []string, indexes []uint64, context *walkContext) { + fieldName := getFieldName(pathString, path, fm) + if fm.Type == "boolean" { + options := fm.Options() + field := document.NewBooleanFieldWithIndexingOptions(fieldName, indexes, propertyValueBool, options) + context.doc.AddField(field) + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } + } +} + +func (fm *FieldMapping) processGeoPoint(propertyMightBeGeoPoint interface{}, pathString string, path []string, indexes []uint64, context *walkContext) { + lon, lat, found := geo.ExtractGeoPoint(propertyMightBeGeoPoint) + if found { + fieldName := getFieldName(pathString, path, fm) + options := fm.Options() + field := document.NewGeoPointFieldWithIndexingOptions(fieldName, indexes, lon, lat, options) + context.doc.AddField(field) + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } + } +} + +func (fm *FieldMapping) processIP(ip net.IP, pathString string, path []string, indexes []uint64, context *walkContext) { + fieldName := getFieldName(pathString, path, fm) + options := fm.Options() + field := document.NewIPFieldWithIndexingOptions(fieldName, indexes, ip, options) + context.doc.AddField(field) + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } +} + +// processGeoShape processes a property that might be a GeoJSON +// shape and adds the appropriate field to the document. +func (fm *FieldMapping) processGeoShape(propertyMightBeGeoShape interface{}, + pathString string, path []string, indexes []uint64, context *walkContext, +) { + fm.processGeoShapeInternal(propertyMightBeGeoShape, pathString, path, context, + func(fieldName string, shapes []*geojson.GeoShape, + options index.FieldIndexingOptions) document.Field { + field := document.NewGeometryCollectionFieldFromShapesWithIndexingOptions( + fieldName, indexes, shapes, options) + if field == nil { + // return an untyped nil, so that the caller's nil check works; + // returning the nil *GeoShapeField directly would produce a + // non-nil document.Field interface wrapping a nil pointer + return nil + } + return field + }, + func(fieldName string, shape *geojson.GeoShape, + options index.FieldIndexingOptions) document.Field { + field := document.NewGeoShapeFieldFromShapeWithIndexingOptions( + fieldName, indexes, shape, options) + if field == nil { + return nil + } + return field + }, + ) +} + +// processGeoShapeV2 processes a property that might be a GeoJSON +// shape and adds the appropriate field to the document using the new GeoShapeV2 format. +func (fm *FieldMapping) processGeoShapeV2(propertyMightBeGeoShape interface{}, + pathString string, path []string, context *walkContext, +) { + fm.processGeoShapeInternal(propertyMightBeGeoShape, pathString, path, context, + func(fieldName string, shapes []*geojson.GeoShape, + options index.FieldIndexingOptions) document.Field { + field := document.NewGeometryCollectionV2FieldFromShapesWithIndexingOptions( + fieldName, shapes, options) + if field == nil { + // return an untyped nil, so that the caller's nil check works; + // returning the nil *GeoShapeV2Field directly would produce a + // non-nil document.Field interface wrapping a nil pointer + return nil + } + return field + }, + func(fieldName string, shape *geojson.GeoShape, + options index.FieldIndexingOptions) document.Field { + field := document.NewGeoShapeV2FieldFromShapeWithIndexingOptions( + fieldName, shape, options) + if field == nil { + return nil + } + return field + }, + ) +} + +// processGeoShapeInternal is a helper function that processes +// a property that might be a GeoJSON shape and adds the appropriate +// field to the document. It takes two functions as parameters to handle +// the creation of fields for geometry collections and individual shapes. +func (fm *FieldMapping) processGeoShapeInternal( + propertyMightBeGeoShape interface{}, + pathString string, path []string, context *walkContext, + makeCollection func(fieldName string, shapes []*geojson.GeoShape, + options index.FieldIndexingOptions) document.Field, + makeShape func(fieldName string, shape *geojson.GeoShape, + options index.FieldIndexingOptions) document.Field, +) { + coordValue, shape, err := geo.ParseGeoShapeField(propertyMightBeGeoShape) + if err != nil { + return + } + + var field document.Field + var found bool + + if shape == geo.GeometryCollectionType { + var geoShapes []*geojson.GeoShape + geoShapes, found = geo.ExtractGeometryCollection(propertyMightBeGeoShape) + if found { + fieldName := getFieldName(pathString, path, fm) + field = makeCollection(fieldName, geoShapes, fm.Options()) + } + } else { + var geoShape *geojson.GeoShape + if shape == geo.CircleType { + geoShape, found = geo.ExtractCircle(propertyMightBeGeoShape) + } else { + geoShape, found = geo.ExtractGeoShapeCoordinates(coordValue, shape) + } + if found { + fieldName := getFieldName(pathString, path, fm) + field = makeShape(fieldName, geoShape, fm.Options()) + } + } + + // field is nil when the constructor failed to encode the shape, + // in which case nothing is added to the document + if found && field != nil { + context.doc.AddField(field) + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, field.Name()) + } + } +} + +func (fm *FieldMapping) analyzerForField(path []string, context *walkContext) analysis.Analyzer { + analyzerName := fm.Analyzer + if analyzerName == "" { + analyzerName = context.dm.defaultAnalyzerName(path) + if analyzerName == "" { + analyzerName = context.im.DefaultAnalyzer + } + } + return context.im.AnalyzerNamed(analyzerName) +} + +func getFieldName(pathString string, path []string, fieldMapping *FieldMapping) string { + fieldName := pathString + if fieldMapping.Name != "" { + parentName := "" + if len(path) > 1 { + parentName = encodePath(path[:len(path)-1]) + pathSeparator + } + fieldName = parentName + fieldMapping.Name + } + return fieldName +} + +// UnmarshalJSON offers custom unmarshaling with optional strict validation +func (fm *FieldMapping) UnmarshalJSON(data []byte) error { + var tmp map[string]json.RawMessage + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return err + } + + var invalidKeys []string + for k, v := range tmp { + switch k { + case "name": + err := util.UnmarshalJSON(v, &fm.Name) + if err != nil { + return err + } + case "type": + err := util.UnmarshalJSON(v, &fm.Type) + if err != nil { + return err + } + case "analyzer": + err := util.UnmarshalJSON(v, &fm.Analyzer) + if err != nil { + return err + } + case "store": + err := util.UnmarshalJSON(v, &fm.Store) + if err != nil { + return err + } + case "index": + err := util.UnmarshalJSON(v, &fm.Index) + if err != nil { + return err + } + case "include_term_vectors": + err := util.UnmarshalJSON(v, &fm.IncludeTermVectors) + if err != nil { + return err + } + case "include_in_all": + err := util.UnmarshalJSON(v, &fm.IncludeInAll) + if err != nil { + return err + } + case "date_format": + err := util.UnmarshalJSON(v, &fm.DateFormat) + if err != nil { + return err + } + case "docvalues": + err := util.UnmarshalJSON(v, &fm.DocValues) + if err != nil { + return err + } + case "skip_freq_norm": + err := util.UnmarshalJSON(v, &fm.SkipFreqNorm) + if err != nil { + return err + } + case "dims": + err := util.UnmarshalJSON(v, &fm.Dims) + if err != nil { + return err + } + case "similarity": + err := util.UnmarshalJSON(v, &fm.Similarity) + if err != nil { + return err + } + case "vector_index_optimized_for": + err := util.UnmarshalJSON(v, &fm.VectorIndexOptimizedFor) + if err != nil { + return err + } + case "synonym_source": + err := util.UnmarshalJSON(v, &fm.SynonymSource) + if err != nil { + return err + } + case "gpu": + err := util.UnmarshalJSON(v, &fm.GPU) + if err != nil { + return err + } + default: + invalidKeys = append(invalidKeys, k) + } + } + + if MappingJSONStrict && len(invalidKeys) > 0 { + return fmt.Errorf("field mapping contains invalid keys: %v", invalidKeys) + } + + return nil +} diff --git a/mapping/index.go b/mapping/index.go new file mode 100644 index 0000000..143ff5a --- /dev/null +++ b/mapping/index.go @@ -0,0 +1,653 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapping + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" + "github.com/blevesearch/bleve/v2/analysis/datetime/optional" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +var MappingJSONStrict = false + +const defaultTypeField = "_type" +const defaultType = "_default" +const defaultField = "_all" +const defaultAnalyzer = standard.Name +const defaultDateTimeParser = optional.Name + +// An IndexMappingImpl controls how objects are placed +// into an index. +// First the type of the object is determined. +// Once the type is know, the appropriate +// DocumentMapping is selected by the type. +// If no mapping was determined for that type, +// a DefaultMapping will be used. +type IndexMappingImpl struct { + TypeMapping map[string]*DocumentMapping `json:"types,omitempty"` + DefaultMapping *DocumentMapping `json:"default_mapping"` + TypeField string `json:"type_field"` + DefaultType string `json:"default_type"` + DefaultAnalyzer string `json:"default_analyzer"` + DefaultDateTimeParser string `json:"default_datetime_parser"` + DefaultSynonymSource string `json:"default_synonym_source,omitempty"` + ScoringModel string `json:"scoring_model,omitempty"` + DefaultField string `json:"default_field"` + StoreDynamic bool `json:"store_dynamic"` + IndexDynamic bool `json:"index_dynamic"` + DocValuesDynamic bool `json:"docvalues_dynamic"` + CustomAnalysis *customAnalysis `json:"analysis,omitempty"` + cache *registry.Cache +} + +// AddCustomCharFilter defines a custom char filter for use in this mapping +func (im *IndexMappingImpl) AddCustomCharFilter(name string, config map[string]interface{}) error { + _, err := im.cache.DefineCharFilter(name, config) + if err != nil { + return err + } + im.CustomAnalysis.CharFilters[name] = config + return nil +} + +// AddCustomTokenizer defines a custom tokenizer for use in this mapping +func (im *IndexMappingImpl) AddCustomTokenizer(name string, config map[string]interface{}) error { + _, err := im.cache.DefineTokenizer(name, config) + if err != nil { + return err + } + im.CustomAnalysis.Tokenizers[name] = config + return nil +} + +// AddCustomTokenMap defines a custom token map for use in this mapping +func (im *IndexMappingImpl) AddCustomTokenMap(name string, config map[string]interface{}) error { + _, err := im.cache.DefineTokenMap(name, config) + if err != nil { + return err + } + im.CustomAnalysis.TokenMaps[name] = config + return nil +} + +// AddCustomTokenFilter defines a custom token filter for use in this mapping +func (im *IndexMappingImpl) AddCustomTokenFilter(name string, config map[string]interface{}) error { + _, err := im.cache.DefineTokenFilter(name, config) + if err != nil { + return err + } + im.CustomAnalysis.TokenFilters[name] = config + return nil +} + +// AddCustomAnalyzer defines a custom analyzer for use in this mapping. The +// config map must have a "type" string entry to resolve the analyzer +// constructor. The constructor is invoked with the remaining entries and +// returned analyzer is registered in the IndexMapping. +// +// bleve comes with predefined analyzers, like +// github.com/blevesearch/bleve/analysis/analyzer/custom. They are +// available only if their package is imported by client code. To achieve this, +// use their metadata to fill configuration entries: +// +// import ( +// "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" +// "github.com/blevesearch/bleve/v2/analysis/char/html" +// "github.com/blevesearch/bleve/v2/analysis/token/lowercase" +// "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" +// ) +// +// m := bleve.NewIndexMapping() +// err := m.AddCustomAnalyzer("html", map[string]interface{}{ +// "type": custom.Name, +// "char_filters": []string{ +// html.Name, +// }, +// "tokenizer": unicode.Name, +// "token_filters": []string{ +// lowercase.Name, +// ... +// }, +// }) +func (im *IndexMappingImpl) AddCustomAnalyzer(name string, config map[string]interface{}) error { + _, err := im.cache.DefineAnalyzer(name, config) + if err != nil { + return err + } + im.CustomAnalysis.Analyzers[name] = config + return nil +} + +// AddCustomDateTimeParser defines a custom date time parser for use in this mapping +func (im *IndexMappingImpl) AddCustomDateTimeParser(name string, config map[string]interface{}) error { + _, err := im.cache.DefineDateTimeParser(name, config) + if err != nil { + return err + } + im.CustomAnalysis.DateTimeParsers[name] = config + return nil +} + +func (im *IndexMappingImpl) AddSynonymSource(name string, config map[string]interface{}) error { + _, err := im.cache.DefineSynonymSource(name, config) + if err != nil { + return err + } + im.CustomAnalysis.SynonymSources[name] = config + return nil +} + +// NewIndexMapping creates a new IndexMapping that will use all the default indexing rules +func NewIndexMapping() *IndexMappingImpl { + return &IndexMappingImpl{ + TypeMapping: make(map[string]*DocumentMapping), + DefaultMapping: NewDocumentMapping(), + TypeField: defaultTypeField, + DefaultType: defaultType, + DefaultAnalyzer: defaultAnalyzer, + DefaultDateTimeParser: defaultDateTimeParser, + DefaultField: defaultField, + IndexDynamic: IndexDynamic, + StoreDynamic: StoreDynamic, + DocValuesDynamic: DocValuesDynamic, + CustomAnalysis: newCustomAnalysis(), + cache: registry.NewCache(), + } +} + +// Validate will walk the entire structure ensuring the following +// explicitly named and default analyzers can be built +func (im *IndexMappingImpl) Validate() error { + _, err := im.cache.AnalyzerNamed(im.DefaultAnalyzer) + if err != nil { + return err + } + _, err = im.cache.DateTimeParserNamed(im.DefaultDateTimeParser) + if err != nil { + return err + } + if im.DefaultSynonymSource != "" { + _, err = im.cache.SynonymSourceNamed(im.DefaultSynonymSource) + if err != nil { + return err + } + } + // fieldAliasCtx is used to detect any field alias conflicts across the entire mapping + // the map will hold the fully qualified field name to FieldMapping, so we can + // check for conflicts as we validate each DocumentMapping. + fieldAliasCtx := make(map[string]*FieldMapping) + // ensure that the nested property is not set for top-level default mapping + if im.DefaultMapping.Nested { + return fmt.Errorf("default mapping cannot be nested") + } + err = im.DefaultMapping.Validate(im.cache, []string{}, fieldAliasCtx) + if err != nil { + return err + } + for name, docMapping := range im.TypeMapping { + // ensure that the nested property is not set for top-level mappings + if docMapping.Nested { + return fmt.Errorf("type mapping named: %s cannot be nested", name) + } + err = docMapping.Validate(im.cache, []string{}, fieldAliasCtx) + if err != nil { + return err + } + } + + if _, ok := index.SupportedScoringModels[im.ScoringModel]; !ok && im.ScoringModel != "" { + return fmt.Errorf("unsupported scoring model: %s", im.ScoringModel) + } + + return nil +} + +// AddDocumentMapping sets a custom document mapping for the specified type +func (im *IndexMappingImpl) AddDocumentMapping(doctype string, dm *DocumentMapping) { + im.TypeMapping[doctype] = dm +} + +func (im *IndexMappingImpl) mappingForType(docType string) *DocumentMapping { + docMapping := im.TypeMapping[docType] + if docMapping == nil { + docMapping = im.DefaultMapping + } + return docMapping +} + +// UnmarshalJSON offers custom unmarshaling with optional strict validation +func (im *IndexMappingImpl) UnmarshalJSON(data []byte) error { + + var tmp map[string]json.RawMessage + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return err + } + + // set defaults for fields which might have been omitted + im.cache = registry.NewCache() + im.CustomAnalysis = newCustomAnalysis() + im.TypeField = defaultTypeField + im.DefaultType = defaultType + im.DefaultAnalyzer = defaultAnalyzer + im.DefaultDateTimeParser = defaultDateTimeParser + im.DefaultField = defaultField + im.DefaultMapping = NewDocumentMapping() + im.TypeMapping = make(map[string]*DocumentMapping) + im.StoreDynamic = StoreDynamic + im.IndexDynamic = IndexDynamic + im.DocValuesDynamic = DocValuesDynamic + + var invalidKeys []string + for k, v := range tmp { + switch k { + case "analysis": + err := util.UnmarshalJSON(v, &im.CustomAnalysis) + if err != nil { + return err + } + case "type_field": + err := util.UnmarshalJSON(v, &im.TypeField) + if err != nil { + return err + } + case "default_type": + err := util.UnmarshalJSON(v, &im.DefaultType) + if err != nil { + return err + } + case "default_analyzer": + err := util.UnmarshalJSON(v, &im.DefaultAnalyzer) + if err != nil { + return err + } + case "default_datetime_parser": + err := util.UnmarshalJSON(v, &im.DefaultDateTimeParser) + if err != nil { + return err + } + case "default_synonym_source": + err := util.UnmarshalJSON(v, &im.DefaultSynonymSource) + if err != nil { + return err + } + case "default_field": + err := util.UnmarshalJSON(v, &im.DefaultField) + if err != nil { + return err + } + case "default_mapping": + err := util.UnmarshalJSON(v, &im.DefaultMapping) + if err != nil { + return err + } + case "types": + err := util.UnmarshalJSON(v, &im.TypeMapping) + if err != nil { + return err + } + case "store_dynamic": + err := util.UnmarshalJSON(v, &im.StoreDynamic) + if err != nil { + return err + } + case "index_dynamic": + err := util.UnmarshalJSON(v, &im.IndexDynamic) + if err != nil { + return err + } + case "docvalues_dynamic": + err := util.UnmarshalJSON(v, &im.DocValuesDynamic) + if err != nil { + return err + } + case "scoring_model": + err := util.UnmarshalJSON(v, &im.ScoringModel) + if err != nil { + return err + } + + default: + invalidKeys = append(invalidKeys, k) + } + } + + if MappingJSONStrict && len(invalidKeys) > 0 { + return fmt.Errorf("index mapping contains invalid keys: %v", invalidKeys) + } + + err = im.CustomAnalysis.registerAll(im) + if err != nil { + return err + } + + return nil +} + +func (im *IndexMappingImpl) determineType(data interface{}) string { + // first see if the object implements bleveClassifier + bleveClassifier, ok := data.(bleveClassifier) + if ok { + return bleveClassifier.BleveType() + } + // next see if the object implements Classifier + classifier, ok := data.(Classifier) + if ok { + return classifier.Type() + } + + // now see if we can find a type using the mapping + typ, ok := mustString(lookupPropertyPath(data, im.TypeField)) + if ok { + return typ + } + + return im.DefaultType +} + +func (im *IndexMappingImpl) MapDocument(doc *document.Document, data interface{}) error { + docType := im.determineType(data) + docMapping := im.mappingForType(docType) + if docMapping.Enabled { + walkContext := im.newWalkContext(doc, docMapping) + docMapping.walkDocument(data, []string{}, []uint64{}, walkContext) + + // see if the _all field was disabled + allMapping, _ := docMapping.documentMappingForPath("_all") + if allMapping == nil || allMapping.Enabled { + field := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, walkContext.excludedFromAll, index.IndexField|index.IncludeTermVectors) + doc.AddField(field) + } + doc.SetIndexed() + } + + return nil +} + +func (im *IndexMappingImpl) MapSynonymDocument(doc *document.Document, collection string, input []string, synonyms []string) error { + // determine all the synonym sources with the given collection + // and create a synonym field for each + err := im.SynonymSourceVisitor(func(name string, item analysis.SynonymSource) error { + if item.Collection() == collection { + // create a new field with the name of the synonym source + analyzer := im.AnalyzerNamed(item.Analyzer()) + if analyzer == nil { + return fmt.Errorf("unknown analyzer named: %s", item.Analyzer()) + } + field := document.NewSynonymField(name, analyzer, input, synonyms) + doc.AddField(field) + } + return nil + }) + return err +} + +type walkContext struct { + doc *document.Document + im *IndexMappingImpl + dm *DocumentMapping + excludedFromAll []string +} + +func (im *IndexMappingImpl) newWalkContext(doc *document.Document, dm *DocumentMapping) *walkContext { + return &walkContext{ + doc: doc, + im: im, + dm: dm, + excludedFromAll: []string{"_id"}, + } +} + +// AnalyzerNameForPath attempts to find the best analyzer to use with only a +// field name will walk all the document types, look for field mappings at the +// provided path, if one exists and it has an explicit analyzer that is +// returned. +func (im *IndexMappingImpl) AnalyzerNameForPath(path string) string { + // first we look for explicit mapping on the field + for _, docMapping := range im.TypeMapping { + analyzerName := docMapping.analyzerNameForPath(path) + if analyzerName != "" { + return analyzerName + } + } + + // now try the default mapping + pathMapping, _ := im.DefaultMapping.documentMappingForPath(path) + if pathMapping != nil { + if len(pathMapping.Fields) > 0 { + if pathMapping.Fields[0].Analyzer != "" { + return pathMapping.Fields[0].Analyzer + } + } + } + + // next we will try default analyzers for the path + pathDecoded := decodePath(path) + for _, docMapping := range im.TypeMapping { + if docMapping.Enabled { + rv := docMapping.defaultAnalyzerName(pathDecoded) + if rv != "" { + return rv + } + } + } + // now the default analyzer for the default mapping + if im.DefaultMapping.Enabled { + rv := im.DefaultMapping.defaultAnalyzerName(pathDecoded) + if rv != "" { + return rv + } + } + + return im.DefaultAnalyzer +} + +func (im *IndexMappingImpl) AnalyzerNamed(name string) analysis.Analyzer { + analyzer, err := im.cache.AnalyzerNamed(name) + if err != nil { + logger.Printf("error using analyzer named: %s", name) + return nil + } + return analyzer +} + +func (im *IndexMappingImpl) DateTimeParserNamed(name string) analysis.DateTimeParser { + if name == "" { + name = im.DefaultDateTimeParser + } + dateTimeParser, err := im.cache.DateTimeParserNamed(name) + if err != nil { + logger.Printf("error using datetime parser named: %s", name) + return nil + } + return dateTimeParser +} + +func (im *IndexMappingImpl) AnalyzeText(analyzerName string, text []byte) (analysis.TokenStream, error) { + analyzer, err := im.cache.AnalyzerNamed(analyzerName) + if err != nil { + return nil, err + } + return analyzer.Analyze(text), nil +} + +// FieldAnalyzer returns the name of the analyzer used on a field. +func (im *IndexMappingImpl) FieldAnalyzer(field string) string { + return im.AnalyzerNameForPath(field) +} + +// FieldMappingForPath returns the mapping for a specific field 'path'. +func (im *IndexMappingImpl) FieldMappingForPath(path string) FieldMapping { + if im.TypeMapping != nil { + for _, v := range im.TypeMapping { + fm := v.fieldDescribedByPath(path) + if fm != nil { + return *fm + } + } + } + + fm := im.DefaultMapping.fieldDescribedByPath(path) + if fm != nil { + return *fm + } + + return FieldMapping{} +} + +// wrapper to satisfy new interface + +func (im *IndexMappingImpl) DefaultSearchField() string { + return im.DefaultField +} + +func (im *IndexMappingImpl) SynonymSourceNamed(name string) analysis.SynonymSource { + syn, err := im.cache.SynonymSourceNamed(name) + if err != nil { + logger.Printf("error using synonym source named: %s", name) + return nil + } + return syn +} + +func (im *IndexMappingImpl) SynonymSourceForPath(path string) string { + // first we look for explicit mapping on the field + for _, docMapping := range im.TypeMapping { + synonymSource := docMapping.synonymSourceForPath(path) + if synonymSource != "" { + return synonymSource + } + } + + // now try the default mapping + pathMapping, _ := im.DefaultMapping.documentMappingForPath(path) + if pathMapping != nil { + if len(pathMapping.Fields) > 0 { + if pathMapping.Fields[0].SynonymSource != "" { + return pathMapping.Fields[0].SynonymSource + } + } + } + + // next we will try default synonym sources for the path + pathDecoded := decodePath(path) + for _, docMapping := range im.TypeMapping { + if docMapping.Enabled { + rv := docMapping.defaultSynonymSource(pathDecoded) + if rv != "" { + return rv + } + } + } + // now the default analyzer for the default mapping + if im.DefaultMapping.Enabled { + rv := im.DefaultMapping.defaultSynonymSource(pathDecoded) + if rv != "" { + return rv + } + } + + return im.DefaultSynonymSource +} + +// SynonymCount() returns the number of synonym sources defined in the mapping +func (im *IndexMappingImpl) SynonymCount() int { + return len(im.CustomAnalysis.SynonymSources) +} + +// SynonymSourceVisitor() allows a visitor to iterate over all synonym sources +func (im *IndexMappingImpl) SynonymSourceVisitor(visitor analysis.SynonymSourceVisitor) error { + err := im.cache.SynonymSources.VisitSynonymSources(visitor) + if err != nil { + return err + } + return nil +} + +func (im *IndexMappingImpl) buildNestedPrefixes() map[string]int { + prefixDepth := make(map[string]int) + var collectNestedFields func(dm *DocumentMapping, pathComponents []string, currentDepth int) + collectNestedFields = func(dm *DocumentMapping, pathComponents []string, currentDepth int) { + for name, docMapping := range dm.Properties { + newPathComponents := append(pathComponents, name) + if docMapping.Nested { + // This is a nested field boundary + newDepth := currentDepth + 1 + prefixDepth[strings.Join(newPathComponents, pathSeparator)] = newDepth + // Continue deeper with incremented depth + collectNestedFields(docMapping, newPathComponents, newDepth) + } else { + // Not nested, continue with same depth + collectNestedFields(docMapping, newPathComponents, currentDepth) + } + } + } + // Start from depth 0 (root) + if im.DefaultMapping != nil && im.DefaultMapping.Enabled { + collectNestedFields(im.DefaultMapping, []string{}, 0) + } + // Now do this for each type mapping + for _, docMapping := range im.TypeMapping { + if docMapping.Enabled { + collectNestedFields(docMapping, []string{}, 0) + } + } + return prefixDepth +} + +func (im *IndexMappingImpl) NestedDepth(fs search.FieldSet) (int, int) { + if im.cache == nil || im.cache.NestedPrefixes == nil { + return 0, 0 + } + + im.cache.NestedPrefixes.InitOnce(func() map[string]int { + return im.buildNestedPrefixes() + }) + + return im.cache.NestedPrefixes.NestedDepth(fs) +} + +func (im *IndexMappingImpl) CountNested() int { + if im.cache == nil || im.cache.NestedPrefixes == nil { + return 0 + } + + im.cache.NestedPrefixes.InitOnce(func() map[string]int { + return im.buildNestedPrefixes() + }) + + return im.cache.NestedPrefixes.CountNested() +} + +func (im *IndexMappingImpl) IntersectsPrefix(fs search.FieldSet) bool { + if im.cache == nil || im.cache.NestedPrefixes == nil { + return false + } + + im.cache.NestedPrefixes.InitOnce(func() map[string]int { + return im.buildNestedPrefixes() + }) + + return im.cache.NestedPrefixes.IntersectsPrefix(fs) +} diff --git a/mapping/mapping.go b/mapping/mapping.go new file mode 100644 index 0000000..7ff2f99 --- /dev/null +++ b/mapping/mapping.go @@ -0,0 +1,95 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapping + +import ( + "io" + "log" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/search" +) + +// A Classifier is an interface describing any object which knows how to +// identify its own type. Alternatively, if a struct already has a Type +// field or method in conflict, one can use BleveType instead. +type Classifier interface { + Type() string +} + +// A bleveClassifier is an interface describing any object which knows how +// to identify its own type. This is introduced as an alternative to the +// Classifier interface which often has naming conflicts with existing +// structures. +type bleveClassifier interface { + BleveType() string +} + +var logger = log.New(io.Discard, "bleve mapping ", log.LstdFlags) + +// SetLog sets the logger used for logging +// by default log messages are sent to io.Discard +func SetLog(l *log.Logger) { + logger = l +} + +type IndexMapping interface { + MapDocument(doc *document.Document, data interface{}) error + Validate() error + + DateTimeParserNamed(name string) analysis.DateTimeParser + + DefaultSearchField() string + + AnalyzerNameForPath(path string) string + AnalyzerNamed(name string) analysis.Analyzer + + FieldMappingForPath(path string) FieldMapping +} + +// A SynonymMapping extends the IndexMapping interface to provide +// additional methods for working with synonyms. +type SynonymMapping interface { + IndexMapping + + MapSynonymDocument(doc *document.Document, collection string, input []string, synonyms []string) error + + SynonymSourceForPath(path string) string + + SynonymSourceNamed(name string) analysis.SynonymSource + + SynonymCount() int + + SynonymSourceVisitor(visitor analysis.SynonymSourceVisitor) error +} + +// A NestedMapping extends the IndexMapping interface to provide +// additional methods for working with nested object mappings. +type NestedMapping interface { + // NestedDepth returns two values: + // - common: the highest nested level that is common to all given field paths, + // if 0 then there is no common nested level among the given field paths + // - max: the highest nested level that applies to at least one of the given field paths + // if 0 then none of the given field paths are nested + NestedDepth(fieldPaths search.FieldSet) (int, int) + + // IntersectsPrefix returns true if any of the given + // field paths intersect with a known nested prefix + IntersectsPrefix(fieldPaths search.FieldSet) bool + + // CountNested returns the number of nested object mappings + CountNested() int +} diff --git a/mapping/mapping_no_vectors.go b/mapping/mapping_no_vectors.go new file mode 100644 index 0000000..cbe9d81 --- /dev/null +++ b/mapping/mapping_no_vectors.go @@ -0,0 +1,44 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !vectors +// +build !vectors + +package mapping + +func NewVectorFieldMapping() *FieldMapping { + return nil +} + +func NewVectorBase64FieldMapping() *FieldMapping { + return nil +} + +func (fm *FieldMapping) processVector(propertyMightBeVector interface{}, + pathString string, path []string, indexes []uint64, context *walkContext) bool { + return false +} + +func (fm *FieldMapping) processVectorBase64(propertyMightBeVector interface{}, + pathString string, path []string, indexes []uint64, context *walkContext) { + +} + +// ----------------------------------------------------------------------------- +// document validation functions + +func validateFieldMapping(field *FieldMapping, path []string, + fieldAliasCtx map[string]*FieldMapping) error { + return validateFieldType(field) +} diff --git a/mapping/mapping_test.go b/mapping/mapping_test.go new file mode 100644 index 0000000..092794b --- /dev/null +++ b/mapping/mapping_test.go @@ -0,0 +1,1260 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapping + +import ( + "encoding/json" + "fmt" + index "github.com/blevesearch/bleve_index_api" + "reflect" + "strconv" + "testing" + "time" + + "github.com/blevesearch/bleve/v2/analysis/tokenizer/exception" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/regexp" + "github.com/blevesearch/bleve/v2/document" +) + +var mappingSource = []byte(`{ + "types": { + "beer": { + "properties": { + "name": { + "fields": [ + { + "name": "name", + "type": "text", + "analyzer": "standard", + "store": true, + "index": true, + "include_term_vectors": true, + "include_in_all": true, + "docvalues": true + } + ] + } + } + }, + "brewery": { + } + }, + "type_field": "_type", + "default_type": "_default" +}`) + +func buildMapping() IndexMapping { + nameFieldMapping := NewTextFieldMapping() + nameFieldMapping.Name = "name" + nameFieldMapping.Analyzer = "standard" + + beerMapping := NewDocumentMapping() + beerMapping.AddFieldMappingsAt("name", nameFieldMapping) + + breweryMapping := NewDocumentMapping() + + mapping := NewIndexMapping() + mapping.AddDocumentMapping("beer", beerMapping) + mapping.AddDocumentMapping("brewery", breweryMapping) + + return mapping +} + +func TestUnmarshalMappingJSON(t *testing.T) { + mapping := buildMapping() + + var indexMapping IndexMappingImpl + err := json.Unmarshal(mappingSource, &indexMapping) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(&indexMapping, mapping) { + t.Errorf("expected %#v,\n got %#v", mapping, &indexMapping) + } +} + +func TestMappingStructWithJSONTags(t *testing.T) { + + mapping := buildMapping() + + x := struct { + NoJSONTag string + Name string `json:"name"` + }{ + Name: "marty", + } + + doc := document.NewDocument("1") + err := mapping.MapDocument(doc, x) + if err != nil { + t.Fatal(err) + } + foundJSONName := false + foundNoJSONName := false + count := 0 + for _, f := range doc.Fields { + if f.Name() == "name" { + foundJSONName = true + } + if f.Name() == "NoJSONTag" { + foundNoJSONName = true + } + count++ + } + if !foundJSONName { + t.Errorf("expected to find field named 'name'") + } + if !foundNoJSONName { + t.Errorf("expected to find field named 'NoJSONTag'") + } + if count != 2 { + t.Errorf("expected to find 2 find, found %d", count) + } +} + +func TestMappingStructWithJSONTagsOneDisabled(t *testing.T) { + + mapping := buildMapping() + + x := struct { + Name string `json:"name"` + Title string `json:"-"` + NoJSONTag string + }{ + Name: "marty", + } + + doc := document.NewDocument("1") + err := mapping.MapDocument(doc, x) + if err != nil { + t.Fatal(err) + } + foundJSONName := false + foundNoJSONName := false + count := 0 + for _, f := range doc.Fields { + if f.Name() == "name" { + foundJSONName = true + } + if f.Name() == "NoJSONTag" { + foundNoJSONName = true + } + count++ + } + if !foundJSONName { + t.Errorf("expected to find field named 'name'") + } + if !foundNoJSONName { + t.Errorf("expected to find field named 'NoJSONTag'") + } + if count != 2 { + t.Errorf("expected to find 2 find, found %d", count) + } +} + +func TestMappingStructWithAlternateTags(t *testing.T) { + + mapping := buildMapping() + mapping.(*IndexMappingImpl).DefaultMapping.StructTagKey = "bleve" + + x := struct { + NoBLEVETag string + Name string `bleve:"name"` + }{ + Name: "marty", + } + + doc := document.NewDocument("1") + err := mapping.MapDocument(doc, x) + if err != nil { + t.Fatal(err) + } + foundBLEVEName := false + foundNoBLEVEName := false + count := 0 + for _, f := range doc.Fields { + if f.Name() == "name" { + foundBLEVEName = true + } + if f.Name() == "NoBLEVETag" { + foundNoBLEVEName = true + } + count++ + } + if !foundBLEVEName { + t.Errorf("expected to find field named 'name'") + } + if !foundNoBLEVEName { + t.Errorf("expected to find field named 'NoBLEVETag'") + } + if count != 2 { + t.Errorf("expected to find 2 find, found %d", count) + } +} + +func TestMappingStructWithAlternateTagsTwoDisabled(t *testing.T) { + + mapping := buildMapping() + mapping.(*IndexMappingImpl).DefaultMapping.StructTagKey = "bleve" + + x := struct { + Name string `json:"-" bleve:"name"` + Title string `json:"-" bleve:"-"` + NoBLEVETag string `json:"-"` + Extra string `json:"extra" bleve:"-"` + }{ + Name: "marty", + } + + doc := document.NewDocument("1") + err := mapping.MapDocument(doc, x) + if err != nil { + t.Fatal(err) + } + foundBLEVEName := false + foundNoBLEVEName := false + count := 0 + for _, f := range doc.Fields { + if f.Name() == "name" { + foundBLEVEName = true + } + if f.Name() == "NoBLEVETag" { + foundNoBLEVEName = true + } + count++ + } + if !foundBLEVEName { + t.Errorf("expected to find field named 'name'") + } + if !foundNoBLEVEName { + t.Errorf("expected to find field named 'NoBLEVETag'") + } + if count != 2 { + t.Errorf("expected to find 2 find, found %d", count) + } +} + +func TestMappingStructWithPointerToString(t *testing.T) { + + mapping := buildMapping() + + name := "marty" + + x := struct { + Name *string + }{ + Name: &name, + } + + doc := document.NewDocument("1") + err := mapping.MapDocument(doc, x) + if err != nil { + t.Fatal(err) + } + found := false + count := 0 + for _, f := range doc.Fields { + if f.Name() == "Name" { + found = true + } + count++ + } + if !found { + t.Errorf("expected to find field named 'Name'") + } + if count != 1 { + t.Errorf("expected to find 1 find, found %d", count) + } +} + +func TestMappingJSONWithNull(t *testing.T) { + + mapping := NewIndexMapping() + + jsonbytes := []byte(`{"name":"marty", "age": null}`) + var jsondoc interface{} + err := json.Unmarshal(jsonbytes, &jsondoc) + if err != nil { + t.Fatal(err) + } + + doc := document.NewDocument("1") + err = mapping.MapDocument(doc, jsondoc) + if err != nil { + t.Fatal(err) + } + found := false + count := 0 + for _, f := range doc.Fields { + if f.Name() == "name" { + found = true + } + count++ + } + if !found { + t.Errorf("expected to find field named 'name'") + } + if count != 1 { + t.Errorf("expected to find 1 find, found %d", count) + } +} + +func TestMappingForPath(t *testing.T) { + + enFieldMapping := NewTextFieldMapping() + enFieldMapping.Analyzer = "en" + + docMappingA := NewDocumentMapping() + docMappingA.AddFieldMappingsAt("name", enFieldMapping) + + customMapping := NewTextFieldMapping() + customMapping.Analyzer = "xyz" + customMapping.Name = "nameCustom" + + subDocMappingB := NewDocumentMapping() + customFieldX := NewTextFieldMapping() + customFieldX.Analyzer = "analyzerx" + subDocMappingB.AddFieldMappingsAt("desc", customFieldX) + + docMappingA.AddFieldMappingsAt("author", enFieldMapping, customMapping) + docMappingA.AddSubDocumentMapping("child", subDocMappingB) + + mapping := NewIndexMapping() + mapping.AddDocumentMapping("a", docMappingA) + + analyzerName := mapping.AnalyzerNameForPath("name") + if analyzerName != enFieldMapping.Analyzer { + t.Errorf("expected '%s' got '%s'", enFieldMapping.Analyzer, analyzerName) + } + + analyzerName = mapping.AnalyzerNameForPath("nameCustom") + if analyzerName != customMapping.Analyzer { + t.Errorf("expected '%s' got '%s'", customMapping.Analyzer, analyzerName) + } + + analyzerName = mapping.AnalyzerNameForPath("child.desc") + if analyzerName != customFieldX.Analyzer { + t.Errorf("expected '%s' got '%s'", customFieldX.Analyzer, analyzerName) + } + +} + +func TestMappingWithTokenizerDeps(t *testing.T) { + + tokNoDeps := map[string]interface{}{ + "type": regexp.Name, + "regexp": "", + } + + tokDepsL1 := map[string]interface{}{ + "type": exception.Name, + "tokenizer": "a", + "exceptions": []string{".*"}, + } + + // this tests a 1-level dependency + // it is run 100 times to increase the + // likelihood that it fails along time way + // (depends on key order iteration in map) + for i := 0; i < 100; i++ { + + m := NewIndexMapping() + ca := customAnalysis{ + Tokenizers: map[string]map[string]interface{}{ + "a": tokNoDeps, + "b": tokDepsL1, + }, + } + err := ca.registerAll(m) + if err != nil { + t.Fatal(err) + } + } + + tokDepsL2 := map[string]interface{}{ + "type": "exception", + "tokenizer": "b", + "exceptions": []string{".*"}, + } + + // now test a second-level dependency + for i := 0; i < 100; i++ { + + m := NewIndexMapping() + ca := customAnalysis{ + Tokenizers: map[string]map[string]interface{}{ + "a": tokNoDeps, + "b": tokDepsL1, + "c": tokDepsL2, + }, + } + err := ca.registerAll(m) + if err != nil { + t.Fatal(err) + } + } + + tokUnsatisfied := map[string]interface{}{ + "type": "exception", + "tokenizer": "e", + } + + // now make sure an unsatisfied dep still + // results in an error + m := NewIndexMapping() + ca := customAnalysis{ + Tokenizers: map[string]map[string]interface{}{ + "a": tokNoDeps, + "b": tokDepsL1, + "c": tokDepsL2, + "d": tokUnsatisfied, + }, + } + err := ca.registerAll(m) + if err == nil { + t.Fatal(err) + } +} + +func TestEnablingDisablingStoringDynamicFields(t *testing.T) { + + // first verify that with system defaults, dynamic field is stored + data := map[string]interface{}{ + "name": "bleve", + } + doc := document.NewDocument("x") + mapping := NewIndexMapping() + err := mapping.MapDocument(doc, data) + if err != nil { + t.Fatal(err) + } + for _, field := range doc.Fields { + if field.Name() == "name" && !field.Options().IsStored() { + t.Errorf("expected field 'name' to be stored, isn't") + } + } + + // now change system level defaults, verify dynamic field is not stored + StoreDynamic = false + defer func() { + StoreDynamic = true + }() + + mapping = NewIndexMapping() + doc = document.NewDocument("y") + err = mapping.MapDocument(doc, data) + if err != nil { + t.Fatal(err) + } + for _, field := range doc.Fields { + if field.Name() == "name" && field.Options().IsStored() { + t.Errorf("expected field 'name' to be not stored, is") + } + } + + // now override the system level defaults inside the index mapping + mapping = NewIndexMapping() + mapping.StoreDynamic = true + doc = document.NewDocument("y") + err = mapping.MapDocument(doc, data) + if err != nil { + t.Fatal(err) + } + for _, field := range doc.Fields { + if field.Name() == "name" && !field.Options().IsStored() { + t.Errorf("expected field 'name' to be stored, isn't") + } + } +} + +func TestMappingBool(t *testing.T) { + boolMapping := NewBooleanFieldMapping() + docMapping := NewDocumentMapping() + docMapping.AddFieldMappingsAt("prop", boolMapping) + mapping := NewIndexMapping() + mapping.AddDocumentMapping("doc", docMapping) + + pprop := false + x := struct { + Prop bool `json:"prop"` + PProp *bool `json:"pprop"` + }{ + Prop: true, + PProp: &pprop, + } + + doc := document.NewDocument("1") + err := mapping.MapDocument(doc, x) + if err != nil { + t.Fatal(err) + } + foundProp := false + foundPProp := false + count := 0 + for _, f := range doc.Fields { + if f.Name() == "prop" { + foundProp = true + } + if f.Name() == "pprop" { + foundPProp = true + } + count++ + } + if !foundProp { + t.Errorf("expected to find bool field named 'prop'") + } + if !foundPProp { + t.Errorf("expected to find pointer to bool field named 'pprop'") + } + if count != 2 { + t.Errorf("expected to find 2 fields, found %d", count) + } +} + +func TestDisableDefaultMapping(t *testing.T) { + indexMapping := NewIndexMapping() + indexMapping.DefaultMapping.Enabled = false + + data := map[string]string{ + "name": "bleve", + } + + doc := document.NewDocument("x") + err := indexMapping.MapDocument(doc, data) + if err != nil { + t.Error(err) + } + + if len(doc.Fields) > 0 { + t.Errorf("expected no fields, got %d", len(doc.Fields)) + } +} + +func TestInvalidFieldMappingStrict(t *testing.T) { + mappingBytes := []byte(`{"includeInAll":true,"name":"a parsed name"}`) + + // first unmarhsal it without strict + var fm FieldMapping + err := json.Unmarshal(mappingBytes, &fm) + if err != nil { + t.Fatal(err) + } + + if fm.Name != "a parsed name" { + t.Fatalf("expect to find field mapping name 'a parsed name', got '%s'", fm.Name) + } + + // reset + fm.Name = "" + + // now enable strict + MappingJSONStrict = true + defer func() { + MappingJSONStrict = false + }() + + expectedInvalidKeys := []string{"includeInAll"} + expectedErr := fmt.Errorf("field mapping contains invalid keys: %v", expectedInvalidKeys) + err = json.Unmarshal(mappingBytes, &fm) + if err.Error() != expectedErr.Error() { + t.Fatalf("expected err: %v, got err: %v", expectedErr, err) + } + + if fm.Name != "a parsed name" { + t.Fatalf("expect to find field mapping name 'a parsed name', got '%s'", fm.Name) + } + +} + +func TestInvalidDocumentMappingStrict(t *testing.T) { + mappingBytes := []byte(`{"defaultAnalyzer":true,"enabled":false}`) + + // first unmarhsal it without strict + var dm DocumentMapping + err := json.Unmarshal(mappingBytes, &dm) + if err != nil { + t.Fatal(err) + } + + if dm.Enabled != false { + t.Fatalf("expect to find document mapping enabled false, got '%t'", dm.Enabled) + } + + // reset + dm.Enabled = true + + // now enable strict + MappingJSONStrict = true + defer func() { + MappingJSONStrict = false + }() + + expectedInvalidKeys := []string{"defaultAnalyzer"} + expectedErr := fmt.Errorf("document mapping contains invalid keys: %v", expectedInvalidKeys) + err = json.Unmarshal(mappingBytes, &dm) + if err.Error() != expectedErr.Error() { + t.Fatalf("expected err: %v, got err: %v", expectedErr, err) + } + + if dm.Enabled != false { + t.Fatalf("expect to find document mapping enabled false, got '%t'", dm.Enabled) + } +} + +func TestInvalidIndexMappingStrict(t *testing.T) { + mappingBytes := []byte(`{"typeField":"type","default_field":"all"}`) + + // first unmarhsal it without strict + var im IndexMappingImpl + err := json.Unmarshal(mappingBytes, &im) + if err != nil { + t.Fatal(err) + } + + if im.DefaultField != "all" { + t.Fatalf("expect to find index mapping default field 'all', got '%s'", im.DefaultField) + } + + // reset + im.DefaultField = "_all" + + // now enable strict + MappingJSONStrict = true + defer func() { + MappingJSONStrict = false + }() + + expectedInvalidKeys := []string{"typeField"} + expectedErr := fmt.Errorf("index mapping contains invalid keys: %v", expectedInvalidKeys) + err = json.Unmarshal(mappingBytes, &im) + if err.Error() != expectedErr.Error() { + t.Fatalf("expected err: %v, got err: %v", expectedErr, err) + } + + if im.DefaultField != "all" { + t.Fatalf("expect to find index mapping default field 'all', got '%s'", im.DefaultField) + } +} + +func TestMappingBug353(t *testing.T) { + dataBytes := `{ + "Reviews": [ + { + "ReviewID": "RX16692001", + "Content": "Usually stay near the airport..." + } + ], + "Other": { + "Inside": "text" + }, + "Name": "The Inn at Baltimore White Marsh" +}` + + var data map[string]interface{} + err := json.Unmarshal([]byte(dataBytes), &data) + if err != nil { + t.Fatal(err) + } + + reviewContentFieldMapping := NewTextFieldMapping() + reviewContentFieldMapping.Analyzer = "crazy" + + reviewsMapping := NewDocumentMapping() + reviewsMapping.Dynamic = false + reviewsMapping.AddFieldMappingsAt("Content", reviewContentFieldMapping) + otherMapping := NewDocumentMapping() + otherMapping.Dynamic = false + mapping := NewIndexMapping() + mapping.DefaultMapping.AddSubDocumentMapping("Reviews", reviewsMapping) + mapping.DefaultMapping.AddSubDocumentMapping("Other", otherMapping) + + doc := document.NewDocument("x") + err = mapping.MapDocument(doc, data) + if err != nil { + t.Fatal(err) + } + + // expect doc has only 2 fields + if len(doc.Fields) != 2 { + t.Errorf("expected doc with 2 fields, got: %d", len(doc.Fields)) + for _, f := range doc.Fields { + t.Logf("field named: %s", f.Name()) + } + } +} + +func TestAnonymousStructFields(t *testing.T) { + + type Contact0 string + + type Contact1 struct { + Name string + } + + type Contact2 interface{} + + type Contact3 interface{} + + type Thing struct { + Contact0 + Contact1 + Contact2 + Contact3 + } + + x := Thing{ + Contact0: "hello", + Contact1: Contact1{ + Name: "marty", + }, + Contact2: Contact1{ + Name: "will", + }, + Contact3: "steve", + } + + doc := document.NewDocument("1") + m := NewIndexMapping() + err := m.MapDocument(doc, x) + if err != nil { + t.Fatal(err) + } + + if len(doc.Fields) != 4 { + t.Fatalf("expected 4 fields, got %d", len(doc.Fields)) + } + if doc.Fields[0].Name() != "Contact0" { + t.Errorf("expected field named 'Contact0', got '%s'", doc.Fields[0].Name()) + } + if doc.Fields[1].Name() != "Name" { + t.Errorf("expected field named 'Name', got '%s'", doc.Fields[1].Name()) + } + if doc.Fields[2].Name() != "Contact2.Name" { + t.Errorf("expected field named 'Contact2.Name', got '%s'", doc.Fields[2].Name()) + } + if doc.Fields[3].Name() != "Contact3" { + t.Errorf("expected field named 'Contact3', got '%s'", doc.Fields[3].Name()) + } + + type AnotherThing struct { + Contact0 `json:"Alternate0"` + Contact1 `json:"Alternate1"` + Contact2 `json:"Alternate2"` + Contact3 `json:"Alternate3"` + } + + y := AnotherThing{ + Contact0: "hello", + Contact1: Contact1{ + Name: "marty", + }, + Contact2: Contact1{ + Name: "will", + }, + Contact3: "steve", + } + + doc2 := document.NewDocument("2") + err = m.MapDocument(doc2, y) + if err != nil { + t.Fatal(err) + } + + if len(doc2.Fields) != 4 { + t.Fatalf("expected 4 fields, got %d", len(doc2.Fields)) + } + if doc2.Fields[0].Name() != "Alternate0" { + t.Errorf("expected field named 'Alternate0', got '%s'", doc2.Fields[0].Name()) + } + if doc2.Fields[1].Name() != "Alternate1.Name" { + t.Errorf("expected field named 'Name', got '%s'", doc2.Fields[1].Name()) + } + if doc2.Fields[2].Name() != "Alternate2.Name" { + t.Errorf("expected field named 'Alternate2.Name', got '%s'", doc2.Fields[2].Name()) + } + if doc2.Fields[3].Name() != "Alternate3" { + t.Errorf("expected field named 'Alternate3', got '%s'", doc2.Fields[3].Name()) + } +} + +func TestAnonymousStructFieldWithJSONStructTagEmptString(t *testing.T) { + type InterfaceThing interface{} + type Thing struct { + InterfaceThing `json:""` + } + + x := Thing{ + InterfaceThing: map[string]interface{}{ + "key": "value", + }, + } + + doc := document.NewDocument("1") + m := NewIndexMapping() + err := m.MapDocument(doc, x) + if err != nil { + t.Fatal(err) + } + + if len(doc.Fields) != 1 { + t.Fatalf("expected 1 field, got %d", len(doc.Fields)) + } + if doc.Fields[0].Name() != "key" { + t.Errorf("expected field named 'key', got '%s'", doc.Fields[0].Name()) + } +} + +func TestMappingPrimitives(t *testing.T) { + + tests := []struct { + data interface{} + }{ + {data: "marty"}, + {data: int(1)}, + {data: int8(2)}, + {data: int16(3)}, + {data: int32(4)}, + {data: int64(5)}, + {data: uint(6)}, + {data: uint8(7)}, + {data: uint16(8)}, + {data: uint32(9)}, + {data: uint64(10)}, + {data: float32(11.0)}, + {data: float64(12.0)}, + {data: false}, + } + + m := NewIndexMapping() + for _, test := range tests { + doc := document.NewDocument("x") + err := m.MapDocument(doc, test.data) + if err != nil { + t.Fatal(err) + } + if len(doc.Fields) != 1 { + t.Errorf("expected 1 field, got %d for %v", len(doc.Fields), test.data) + } + } +} + +func TestMappingForGeo(t *testing.T) { + + type Location struct { + Lat float64 + Lon float64 + } + + nameFieldMapping := NewTextFieldMapping() + nameFieldMapping.Name = "name" + nameFieldMapping.Analyzer = "standard" + + locFieldMapping := NewGeoPointFieldMapping() + + thingMapping := NewDocumentMapping() + thingMapping.AddFieldMappingsAt("name", nameFieldMapping) + thingMapping.AddFieldMappingsAt("location", locFieldMapping) + + mapping := NewIndexMapping() + mapping.DefaultMapping = thingMapping + + geopoints := []interface{}{} + expect := [][]float64{} // to contain expected [lon,lat] for geopoints + + // geopoint as a struct + geopoints = append(geopoints, struct { + Name string `json:"name"` + Location *Location `json:"location"` + }{ + Name: "struct", + Location: &Location{ + Lon: -180, + Lat: -90, + }, + }) + expect = append(expect, []float64{-180, -90}) + + // geopoint as a map + geopoints = append(geopoints, struct { + Name string `json:"name"` + Location map[string]interface{} `json:"location"` + }{ + Name: "map", + Location: map[string]interface{}{ + "lon": -180, + "lat": -90, + }, + }) + expect = append(expect, []float64{-180, -90}) + + // geopoint as a slice, format: {lon, lat} + geopoints = append(geopoints, struct { + Name string `json:"name"` + Location []interface{} `json:"location"` + }{ + Name: "slice", + Location: []interface{}{ + -180, -90, + }, + }) + expect = append(expect, []float64{-180, -90}) + + // geopoint as a string, format: "lat,lon" + geopoints = append(geopoints, struct { + Name string `json:"name"` + Location []interface{} `json:"location"` + }{ + Name: "string", + Location: []interface{}{ + "-90,-180", + }, + }) + expect = append(expect, []float64{-180, -90}) + + // geopoint as a string, format: "lat , lon" with leading/trailing whitespaces + geopoints = append(geopoints, struct { + Name string `json:"name"` + Location []interface{} `json:"location"` + }{ + Name: "string", + Location: []interface{}{ + "-90 , -180", + }, + }) + expect = append(expect, []float64{-180, -90}) + + // geopoint as a string - geohash + geopoints = append(geopoints, struct { + Name string `json:"name"` + Location []interface{} `json:"location"` + }{ + Name: "string", + Location: []interface{}{ + "000000000000", + }, + }) + expect = append(expect, []float64{-180, -90}) + + // geopoint as a string - geohash + geopoints = append(geopoints, struct { + Name string `json:"name"` + Location []interface{} `json:"location"` + }{ + Name: "string", + Location: []interface{}{ + "drm3btev3e86", + }, + }) + expect = append(expect, []float64{-71.34, 41.12}) + + for i, geopoint := range geopoints { + doc := document.NewDocument(fmt.Sprint(i)) + err := mapping.MapDocument(doc, geopoint) + if err != nil { + t.Fatal(err) + } + + var foundGeo bool + for _, f := range doc.Fields { + if f.Name() == "location" { + foundGeo = true + geoF, ok := f.(index.GeoPointField) + if !ok { + t.Errorf("expected a geopoint field!") + } + lon, err := geoF.Lon() + if err != nil { + t.Errorf("error in fetching lon, err: %v", err) + } + lat, err := geoF.Lat() + if err != nil { + t.Errorf("error in fetching lat, err: %v", err) + } + // round obtained lon, lat to 2 decimal places + roundLon, _ := strconv.ParseFloat(fmt.Sprintf("%.2f", lon), 64) + roundLat, _ := strconv.ParseFloat(fmt.Sprintf("%.2f", lat), 64) + if roundLon != expect[i][0] || roundLat != expect[i][1] { + t.Errorf("expected geo point: {%v, %v}, got {%v, %v}", + expect[i][0], expect[i][1], lon, lat) + } + } + } + + if !foundGeo { + t.Errorf("expected to find geo point, did not") + } + } +} + +type textMarshalable struct { + body string + Extra string +} + +func (t *textMarshalable) MarshalText() ([]byte, error) { + return []byte(t.body), nil +} + +func TestMappingForTextMarshaler(t *testing.T) { + tm := struct { + Marshalable *textMarshalable + }{ + Marshalable: &textMarshalable{ + body: "text", + Extra: "stuff", + }, + } + + // first verify that when using a mapping that doesn't explicitly + // map the struct field as text, then we traverse inside the struct + // and do our best + m := NewIndexMapping() + doc := document.NewDocument("x") + err := m.MapDocument(doc, tm) + if err != nil { + t.Fatal(err) + } + + if len(doc.Fields) != 1 { + t.Fatalf("expected 1 field, got: %d", len(doc.Fields)) + } + if doc.Fields[0].Name() != "Marshalable.Extra" { + t.Errorf("expected field to be named 'Marshalable.Extra', got: '%s'", doc.Fields[0].Name()) + } + if string(doc.Fields[0].Value()) != tm.Marshalable.Extra { + t.Errorf("expected field value to be '%s', got: '%s'", tm.Marshalable.Extra, string(doc.Fields[0].Value())) + } + + // now verify that when a mapping explicitly + m = NewIndexMapping() + txt := NewTextFieldMapping() + m.DefaultMapping.AddFieldMappingsAt("Marshalable", txt) + doc = document.NewDocument("x") + err = m.MapDocument(doc, tm) + if err != nil { + t.Fatal(err) + } + + if len(doc.Fields) != 1 { + t.Fatalf("expected 1 field, got: %d", len(doc.Fields)) + + } + if doc.Fields[0].Name() != "Marshalable" { + t.Errorf("expected field to be named 'Marshalable', got: '%s'", doc.Fields[0].Name()) + } + want, err := tm.Marshalable.MarshalText() + if err != nil { + t.Fatal(err) + } + if string(doc.Fields[0].Value()) != string(want) { + t.Errorf("expected field value to be '%s', got: '%s'", string(want), string(doc.Fields[0].Value())) + } + +} + +func TestMappingForNilTextMarshaler(t *testing.T) { + tm := struct { + Marshalable *time.Time + }{ + Marshalable: nil, + } + + // now verify that when a mapping explicitly + m := NewIndexMapping() + txt := NewTextFieldMapping() + m.DefaultMapping.AddFieldMappingsAt("Marshalable", txt) + doc := document.NewDocument("x") + err := m.MapDocument(doc, tm) + if err != nil { + t.Fatal(err) + } + + if len(doc.Fields) != 0 { + t.Fatalf("expected 1 field, got: %d", len(doc.Fields)) + + } + +} + +func TestClosestDocDynamicMapping(t *testing.T) { + mapping := NewIndexMapping() + mapping.IndexDynamic = false + mapping.DefaultMapping = NewDocumentStaticMapping() + mapping.DefaultMapping.AddFieldMappingsAt("foo", NewTextFieldMapping()) + + doc := document.NewDocument("x") + err := mapping.MapDocument(doc, map[string]interface{}{ + "foo": "value", + "bar": map[string]string{ + "foo": "value2", + "baz": "value3", + }, + }) + if err != nil { + t.Fatal(err) + } + + if len(doc.Fields) != 1 { + t.Fatalf("expected 1 field, got: %d", len(doc.Fields)) + } +} + +func TestMappingPointerToTimeBug1152(t *testing.T) { + when, err := time.Parse(time.RFC3339, "2019-03-06T15:04:05Z") + if err != nil { + t.Fatal(err) + } + + thing := struct { + When *time.Time + }{ + When: &when, + } + + // this case tests when there WAS an explicit mapping, but it was NOT type text + // as this was the specific case that was problematic + m := NewIndexMapping() + dtf := NewDateTimeFieldMapping() + m.DefaultMapping.AddFieldMappingsAt("When", dtf) + doc := document.NewDocument("x") + err = m.MapDocument(doc, thing) + if err != nil { + t.Fatal(err) + } + + if len(doc.Fields) != 1 { + t.Fatalf("expected 1 field, got: %d", len(doc.Fields)) + } + if _, ok := doc.Fields[0].(index.DateTimeField); !ok { + t.Fatalf("expected field to be type *document.DateTimeField, got %T", doc.Fields[0]) + } +} + +func TestDefaultAnalyzerInheritance(t *testing.T) { + docMapping := NewDocumentMapping() + docMapping.DefaultAnalyzer = "xyz" + childMapping := NewTextFieldMapping() + docMapping.AddFieldMappingsAt("field", childMapping) + + if analyzer := docMapping.defaultAnalyzerName([]string{"field"}); analyzer != "xyz" { + t.Fatalf("Expected analyzer: xyz to be inherited by field, but got: '%v'", analyzer) + } +} + +func TestWrongAnalyzerSearchableAs(t *testing.T) { + fieldMapping := NewTextFieldMapping() + fieldMapping.Name = "geo.accuracy" + fieldMapping.Analyzer = "xyz" + + nestedMapping := NewDocumentMapping() + nestedMapping.AddFieldMappingsAt("accuracy", fieldMapping) + + docMapping := NewDocumentMapping() + docMapping.AddSubDocumentMapping("geo", nestedMapping) + + indexMapping := NewIndexMapping() + indexMapping.AddDocumentMapping("brewery", docMapping) + + analyzerName := indexMapping.AnalyzerNameForPath("geo.geo.accuracy") + if analyzerName != "xyz" { + t.Errorf("expected analyzer name `xyz`, got `%s`", analyzerName) + } +} + +func TestMappingArrayOfStringGeoPoints(t *testing.T) { + nameFieldMapping := NewTextFieldMapping() + nameFieldMapping.Name = "name" + nameFieldMapping.Analyzer = "standard" + + locFieldMapping := NewGeoPointFieldMapping() + + thingMapping := NewDocumentMapping() + thingMapping.AddFieldMappingsAt("points", locFieldMapping) + + mapping := NewIndexMapping() + mapping.DefaultMapping = thingMapping + + docs := []map[string]interface{}{ + { + // string: "lat,lon" + "points": []string{ + "1.0, 2.0", + "3.0, 4.0", + "5.0, 6.0", + }, + }, + { + // slice: {lon, lat} + "points": [][]float64{ + {2.0, 1.0}, + {4.0, 3.0}, + {6.0, 5.0}, + }, + }, + { + // struct: {"lon/lng": .., "lat": ..} + "points": []map[string]interface{}{ + {"lon": 2.0, "lat": 1.0}, + {"lng": 4.0, "lat": 3.0}, + {"lng": 6.0, "lat": 5.0}, + }, + }, + } + + for _, docSrc := range docs { + doc := document.NewDocument("x") + err := mapping.MapDocument(doc, docSrc) + if err != nil { + t.Fatal(err) + } + + // points here in lon, lat order + expectPoints := map[string][]float64{ + "first": {2.0, 1.0}, + "second": {4.0, 3.0}, + "third": {6.0, 5.0}, + } + + for _, f := range doc.Fields { + if f.Name() == "points" { + geoF, ok := f.(*document.GeoPointField) + if !ok { + t.Errorf("expected a geopoint field!") + } + lon, err := geoF.Lon() + if err != nil { + t.Errorf("error in fetching lon, err: %v", err) + } + lat, err := geoF.Lat() + if err != nil { + t.Errorf("error in fetching lat, err: %v", err) + } + // round obtained lon, lat to 2 decimal places + roundLon, _ := strconv.ParseFloat(fmt.Sprintf("%.2f", lon), 64) + roundLat, _ := strconv.ParseFloat(fmt.Sprintf("%.2f", lat), 64) + + for key, point := range expectPoints { + if roundLon == point[0] && roundLat == point[1] { + delete(expectPoints, key) + } + } + } + } + + if len(expectPoints) > 0 { + t.Errorf("some points not found: %v", expectPoints) + } + } +} diff --git a/mapping/mapping_vectors.go b/mapping/mapping_vectors.go new file mode 100644 index 0000000..81d2cb9 --- /dev/null +++ b/mapping/mapping_vectors.go @@ -0,0 +1,351 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package mapping + +import ( + "fmt" + "reflect" + "slices" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" + faiss "github.com/blevesearch/go-faiss" +) + +// Min and Max allowed dimensions for a vector field; +// p.s must be set/updated at process init() _only_ +var ( + MinVectorDims = 1 + MaxVectorDims = 4096 +) + +func NewVectorFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "vector", + Store: false, + Index: true, + IncludeInAll: false, + DocValues: false, + SkipFreqNorm: true, + } +} + +func NewVectorBase64FieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "vector_base64", + Store: false, + Index: true, + IncludeInAll: false, + DocValues: false, + SkipFreqNorm: true, + } +} + +// validate and process a flat vector +func processFlatVector(vecV reflect.Value, dims int) ([]float32, bool) { + if vecV.Len() != dims { + return nil, false + } + + rv := make([]float32, dims) + for i := 0; i < vecV.Len(); i++ { + item := vecV.Index(i) + if !item.CanInterface() { + return nil, false + } + itemI := item.Interface() + itemFloat, ok := util.ExtractNumericValFloat32(itemI) + if !ok { + return nil, false + } + rv[i] = itemFloat + } + + return rv, true +} + +// validate and process a vector +// max supported depth of nesting is 2 ([][]float32) +func processVector(vecI interface{}, dims int) ([]float32, bool) { + vecV := reflect.ValueOf(vecI) + if !vecV.IsValid() || vecV.Kind() != reflect.Slice || vecV.Len() == 0 { + return nil, false + } + + // Let's examine the first element (head) of the vector. + // If head is a slice, then vector is nested, otherwise flat. + head := vecV.Index(0) + if !head.CanInterface() { + return nil, false + } + headI := head.Interface() + headV := reflect.ValueOf(headI) + if !headV.IsValid() { + return nil, false + } + if headV.Kind() != reflect.Slice { // vector is flat + return processFlatVector(vecV, dims) + } + + // # process nested vector + + // pre-allocate memory for the flattened vector + // so that we can use copy() later + rv := make([]float32, dims*vecV.Len()) + + for i := 0; i < vecV.Len(); i++ { + subVec := vecV.Index(i) + if !subVec.CanInterface() { + return nil, false + } + subVecI := subVec.Interface() + subVecV := reflect.ValueOf(subVecI) + if !subVecV.IsValid() { + return nil, false + } + + if subVecV.Kind() != reflect.Slice { + return nil, false + } + + flatVector, ok := processFlatVector(subVecV, dims) + if !ok { + return nil, false + } + + copy(rv[i*dims:(i+1)*dims], flatVector) + } + + return rv, true +} + +func (fm *FieldMapping) processVector(propertyMightBeVector interface{}, + pathString string, path []string, indexes []uint64, context *walkContext) bool { + vector, ok := processVector(propertyMightBeVector, fm.Dims) + // Don't add field to document if vector is invalid + if !ok { + return false + } + // Apply defaults for similarity and optimization if not set + similarity := fm.Similarity + if similarity == "" { + similarity = index.DefaultVectorSimilarityMetric + } + vectorIndexOptimizedFor := fm.VectorIndexOptimizedFor + if vectorIndexOptimizedFor == "" { + vectorIndexOptimizedFor = index.DefaultIndexOptimization + } + // bivf indexes only supports hamming distance for the primary + // binary index. Similarity here is used for the backing flat index, + // which is set to cosine similarity for recall reasons + if index.OptimizationRequiresBinaryIndex(vectorIndexOptimizedFor) { + similarity = index.CosineSimilarity + } + // normalize raw vector if similarity is cosine + // Since the vector can be multi-vector (flattened array of multiple vectors), + // we use NormalizeMultiVector to normalize each sub-vector independently. + if similarity == index.CosineSimilarity { + vector = NormalizeMultiVector(vector, fm.Dims) + } + + fieldName := getFieldName(pathString, path, fm) + + options := fm.Options() + field := document.NewVectorFieldWithIndexingOptions(fieldName, indexes, vector, + fm.Dims, similarity, vectorIndexOptimizedFor, options) + context.doc.AddField(field) + + // "_all" composite field is not applicable for vector field + context.excludedFromAll = append(context.excludedFromAll, fieldName) + return true +} + +func (fm *FieldMapping) processVectorBase64(propertyMightBeVectorBase64 interface{}, + pathString string, path []string, indexes []uint64, context *walkContext) { + encodedString, ok := propertyMightBeVectorBase64.(string) + if !ok { + return + } + // Apply defaults for similarity and optimization if not set + similarity := fm.Similarity + if similarity == "" { + similarity = index.DefaultVectorSimilarityMetric + } + vectorIndexOptimizedFor := fm.VectorIndexOptimizedFor + if vectorIndexOptimizedFor == "" { + vectorIndexOptimizedFor = index.DefaultIndexOptimization + } + // bivf indexes only supports hamming distance for the primary + // binary index. Similarity here is used for the backing flat index, + // which is set to cosine similarity for recall reasons + if index.OptimizationRequiresBinaryIndex(vectorIndexOptimizedFor) { + similarity = index.CosineSimilarity + } + decodedVector, err := document.DecodeVector(encodedString) + if err != nil || len(decodedVector) != fm.Dims { + return + } + // normalize raw vector if similarity is cosine, multi-vector is not supported + // for base64 encoded vectors, so we use NormalizeVector directly. + if similarity == index.CosineSimilarity { + decodedVector = NormalizeVector(decodedVector) + } + + fieldName := getFieldName(pathString, path, fm) + options := fm.Options() + + field := document.NewVectorFieldWithIndexingOptions(fieldName, indexes, decodedVector, + fm.Dims, similarity, vectorIndexOptimizedFor, options) + context.doc.AddField(field) + + // "_all" composite field is not applicable for vector_base64 field + context.excludedFromAll = append(context.excludedFromAll, fieldName) +} + +// ----------------------------------------------------------------------------- +// document validation functions + +func validateFieldMapping(field *FieldMapping, path []string, + fieldAliasCtx map[string]*FieldMapping) error { + switch field.Type { + case "vector", "vector_base64": + return validateVectorFieldAlias(field, path, fieldAliasCtx) + default: // non-vector field + return validateFieldType(field) + } +} + +func validateVectorFieldAlias(field *FieldMapping, path []string, + fieldAliasCtx map[string]*FieldMapping) error { + // fully qualified field name + pathString := encodePath(path) + // check if field has a name set, else use path to compute effective name + effectiveFieldName := getFieldName(pathString, path, field) + // Compute effective values for validation + effectiveSimilarity := field.Similarity + if effectiveSimilarity == "" { + effectiveSimilarity = index.DefaultVectorSimilarityMetric + } + effectiveOptimizedFor := field.VectorIndexOptimizedFor + if effectiveOptimizedFor == "" { + effectiveOptimizedFor = index.DefaultIndexOptimization + } + + // # If alias is present, validate the field options as per the alias. + // note: reading from a nil map is safe + if fieldAlias, ok := fieldAliasCtx[effectiveFieldName]; ok { + if field.Dims != fieldAlias.Dims { + return fmt.Errorf("field: '%s', invalid alias "+ + "(different dimensions %d and %d)", effectiveFieldName, field.Dims, + fieldAlias.Dims) + } + + // Compare effective similarity values + aliasSimilarity := fieldAlias.Similarity + if aliasSimilarity == "" { + aliasSimilarity = index.DefaultVectorSimilarityMetric + } + if effectiveSimilarity != aliasSimilarity { + return fmt.Errorf("field: '%s', invalid alias "+ + "(different similarity values %s and %s)", effectiveFieldName, + effectiveSimilarity, aliasSimilarity) + } + + // Compare effective vector index optimization values + aliasOptimizedFor := fieldAlias.VectorIndexOptimizedFor + if aliasOptimizedFor == "" { + aliasOptimizedFor = index.DefaultIndexOptimization + } + if effectiveOptimizedFor != aliasOptimizedFor { + return fmt.Errorf("field: '%s', invalid alias "+ + "(different vector index optimization values %s and %s)", effectiveFieldName, + effectiveOptimizedFor, aliasOptimizedFor) + } + if field.GPU != fieldAlias.GPU { + return fmt.Errorf("field: '%s', invalid alias "+ + "(different gpu values %v and %v)", effectiveFieldName, + field.GPU, fieldAlias.GPU) + } + + return nil + } + + // # Validate field options + // Vector dimensions must be within allowed range + if field.Dims < MinVectorDims || field.Dims > MaxVectorDims { + return fmt.Errorf("field: '%s', invalid vector dimension: %d,"+ + " value should be in range [%d, %d]", effectiveFieldName, field.Dims, + MinVectorDims, MaxVectorDims) + } + // Similarity metric must be supported + if _, ok := index.SupportedVectorSimilarityMetrics[effectiveSimilarity]; !ok { + return fmt.Errorf("field: '%s', invalid similarity "+ + "metric: '%s', valid metrics are: %+v", effectiveFieldName, effectiveSimilarity, + reflect.ValueOf(index.SupportedVectorSimilarityMetrics).MapKeys()) + } + // Vector index optimization must be supported + if _, ok := index.SupportedVectorIndexOptimizations[effectiveOptimizedFor]; !ok { + return fmt.Errorf("field: '%s', invalid vector index "+ + "optimization: '%s', valid optimizations are: %+v", effectiveFieldName, + effectiveOptimizedFor, + reflect.ValueOf(index.SupportedVectorIndexOptimizations).MapKeys()) + } + // bivf indexes requires vector dimensionality to be a multiple of 8 + if index.OptimizationRequiresBinaryIndex(effectiveOptimizedFor) && field.Dims%8 != 0 { + return fmt.Errorf("field: '%s', incompatible vector dimensionality for BIVF: %d,"+ + " dimension should be a multiple of 8", effectiveFieldName, field.Dims) + } + + if fieldAliasCtx != nil { // writing to a nil map is unsafe + fieldAliasCtx[effectiveFieldName] = field + } + + return nil +} + +// NormalizeVector normalizes a single vector to unit length. +// It makes a copy of the input vector to avoid modifying it in-place. +func NormalizeVector(vec []float32) []float32 { + // make a copy of the vector to avoid modifying the original + // vector in-place + vecCopy := slices.Clone(vec) + // normalize the vector copy using in-place normalization provided by faiss + return faiss.NormalizeVector(vecCopy) +} + +// NormalizeMultiVector normalizes each sub-vector of size `dims` independently. +// For a flattened array containing multiple vectors, each sub-vector is +// normalized separately to unit length. +// It makes a copy of the input vector to avoid modifying it in-place. +func NormalizeMultiVector(vec []float32, dims int) []float32 { + if len(vec) == 0 || dims <= 0 || len(vec)%dims != 0 { + return vec + } + // Single vector - delegate to NormalizeVector + if len(vec) == dims { + return NormalizeVector(vec) + } + // Multi-vector - make a copy to avoid modifying the original + result := slices.Clone(vec) + // Normalize each sub-vector in-place + for i := 0; i < len(result); i += dims { + faiss.NormalizeVector(result[i : i+dims]) + } + return result +} diff --git a/mapping/mapping_vectors_test.go b/mapping/mapping_vectors_test.go new file mode 100644 index 0000000..f575722 --- /dev/null +++ b/mapping/mapping_vectors_test.go @@ -0,0 +1,1245 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package mapping + +import ( + "math" + "reflect" + "strings" + "testing" +) + +func TestVectorFieldAliasValidation(t *testing.T) { + tests := []struct { + // input + name string // name of the test + mappingStr string // index mapping json string + + // expected output + expValidity bool // validity of the mapping + errMsgs []string // error message, given expValidity is false + }{ + { + name: "test1", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "type": "vector", + "dims": 3 + }, + { + "name": "cityVec", + "type": "vector", + "dims": 4 + } + ] + } + } + } + }`, + expValidity: false, + errMsgs: []string{`field: 'cityVec', invalid alias (different dimensions 4 and 3)`}, + }, + { + name: "test2", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "type": "vector", + "dims": 3, + "similarity": "l2_norm" + }, + { + "name": "cityVec", + "type": "vector", + "dims": 3, + "similarity": "dot_product" + } + ] + } + } + } + }`, + expValidity: false, + errMsgs: []string{`field: 'cityVec', invalid alias (different similarity values dot_product and l2_norm)`}, + }, + { + name: "test3", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "type": "vector", + "dims": 3 + }, + { + "name": "cityVec", + "type": "vector", + "dims": 3 + } + ] + } + } + } + }`, + expValidity: true, + errMsgs: []string{}, + }, + { + name: "test4", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "name": "vecData", + "type": "vector", + "dims": 4 + } + ] + }, + "countryVec": { + "fields": [ + { + "name": "vecData", + "type": "vector", + "dims": 3 + } + ] + } + } + } + }`, + expValidity: false, + errMsgs: []string{`field: 'vecData', invalid alias (different dimensions 3 and 4)`, `field: 'vecData', invalid alias (different dimensions 4 and 3)`}, + }, + { + name: "test5", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "name": "vecData", + "type": "vector", + "dims": 3 + } + ] + } + } + }, + "types": { + "type1": { + "properties": { + "cityVec": { + "fields": [ + { + "name": "vecData", + "type": "vector", + "dims": 4 + } + ] + } + } + } + } + }`, + expValidity: false, + errMsgs: []string{`field: 'vecData', invalid alias (different dimensions 4 and 3)`}, + }, + // Test 6: Different vector index optimization values (alias case) + { + name: "different_optimization_alias", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "type": "vector", + "dims": 3, + "vector_index_optimized_for": "recall" + }, + { + "name": "cityVec", + "type": "vector", + "dims": 3, + "vector_index_optimized_for": "latency" + } + ] + } + } + } + }`, + expValidity: false, + errMsgs: []string{`field: 'cityVec', invalid alias (different vector index optimization values latency and recall)`}, + }, + // Test 7: Invalid dimensions - below minimum + { + name: "dims_below_minimum", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "type": "vector", + "dims": 0 + } + ] + } + } + } + }`, + expValidity: false, + errMsgs: []string{`field: 'cityVec', invalid vector dimension: 0, value should be in range [1, 4096]`}, + }, + // Test 8: Invalid dimensions - above maximum + { + name: "dims_above_maximum", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "type": "vector", + "dims": 5000 + } + ] + } + } + } + }`, + expValidity: false, + errMsgs: []string{`field: 'cityVec', invalid vector dimension: 5000, value should be in range [1, 4096]`}, + }, + // Test 8b: Different GPU usage in alias should be invalid + { + name: "different_gpu_usage_alias", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "type": "vector", + "dims": 3, + "gpu": true + }, + { + "name": "cityVec", + "type": "vector", + "dims": 3, + "gpu": false + } + ] + } + } + } + }`, + expValidity: false, + errMsgs: []string{`field: 'cityVec', invalid alias (different gpu values false and true)`}, + }, + // Test 8c: Matching GPU usage in alias should be valid + { + name: "same_gpu_usage_alias", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "type": "vector", + "dims": 3, + "gpu": true + }, + { + "name": "cityVec", + "type": "vector", + "dims": 3, + "gpu": true + } + ] + } + } + } + }`, + expValidity: true, + errMsgs: []string{}, + }, + // Test 9: Invalid similarity metric + { + name: "invalid_similarity_metric", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "type": "vector", + "dims": 3, + "similarity": "invalid_metric" + } + ] + } + } + } + }`, + expValidity: false, + // Note: error message contains map keys which have non-deterministic order + errMsgs: []string{`invalid similarity metric: 'invalid_metric'`}, + }, + // Test 10: Invalid vector index optimization + { + name: "invalid_optimization", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "type": "vector", + "dims": 3, + "vector_index_optimized_for": "invalid_opt" + } + ] + } + } + } + }`, + expValidity: false, + // Note: error message contains map keys which have non-deterministic order + errMsgs: []string{`invalid vector index optimization: 'invalid_opt'`}, + }, + // Test 11: vector_base64 type with valid dimensions + { + name: "vector_base64_valid", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "type": "vector_base64", + "dims": 128 + } + ] + } + } + } + }`, + expValidity: true, + errMsgs: []string{}, + }, + // Test 12: vector_base64 alias with different dimensions + { + name: "vector_base64_different_dims_alias", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "type": "vector_base64", + "dims": 128 + }, + { + "name": "cityVec", + "type": "vector_base64", + "dims": 256 + } + ] + } + } + } + }`, + expValidity: false, + errMsgs: []string{`field: 'cityVec', invalid alias (different dimensions 256 and 128)`}, + }, + // Test 13: Default similarity matching explicit similarity in alias + { + name: "default_similarity_matches_explicit", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "type": "vector", + "dims": 3 + }, + { + "name": "cityVec", + "type": "vector", + "dims": 3, + "similarity": "l2_norm" + } + ] + } + } + } + }`, + expValidity: true, + errMsgs: []string{}, + }, + // Test 14: Default optimization matching explicit optimization in alias + { + name: "default_optimization_matches_explicit", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "type": "vector", + "dims": 3 + }, + { + "name": "cityVec", + "type": "vector", + "dims": 3, + "vector_index_optimized_for": "recall" + } + ] + } + } + } + }`, + expValidity: true, + errMsgs: []string{}, + }, + // Test 15: Valid alias with all explicit matching values + { + name: "valid_alias_all_explicit_matching", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "type": "vector", + "dims": 64, + "similarity": "dot_product", + "vector_index_optimized_for": "latency" + }, + { + "name": "cityVec", + "type": "vector", + "dims": 64, + "similarity": "dot_product", + "vector_index_optimized_for": "latency" + } + ] + } + } + } + }`, + expValidity: true, + errMsgs: []string{}, + }, + // Test 16: Cross-property alias with different similarity + { + name: "cross_property_different_similarity", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "name": "vecData", + "type": "vector", + "dims": 3, + "similarity": "cosine" + } + ] + }, + "countryVec": { + "fields": [ + { + "name": "vecData", + "type": "vector", + "dims": 3, + "similarity": "l2_norm" + } + ] + } + } + } + }`, + expValidity: false, + errMsgs: []string{ + `field: 'vecData', invalid alias (different similarity values l2_norm and cosine)`, + `field: 'vecData', invalid alias (different similarity values cosine and l2_norm)`, + }, + }, + // Test 17: Cross-property alias with different optimization + { + name: "cross_property_different_optimization", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "name": "vecData", + "type": "vector", + "dims": 3, + "vector_index_optimized_for": "recall" + } + ] + }, + "countryVec": { + "fields": [ + { + "name": "vecData", + "type": "vector", + "dims": 3, + "vector_index_optimized_for": "memory-efficient" + } + ] + } + } + } + }`, + expValidity: false, + errMsgs: []string{ + `field: 'vecData', invalid alias (different vector index optimization values memory-efficient and recall)`, + `field: 'vecData', invalid alias (different vector index optimization values recall and memory-efficient)`, + }, + }, + // Test 18: Valid cross-property alias with matching values + { + name: "valid_cross_property_alias", + mappingStr: ` + { + "default_mapping": { + "properties": { + "cityVec": { + "fields": [ + { + "name": "vecData", + "type": "vector", + "dims": 64, + "similarity": "dot_product", + "vector_index_optimized_for": "latency" + } + ] + }, + "countryVec": { + "fields": [ + { + "name": "vecData", + "type": "vector", + "dims": 64, + "similarity": "dot_product", + "vector_index_optimized_for": "latency" + } + ] + } + } + } + }`, + expValidity: true, + errMsgs: []string{}, + }, + // Test 20: Different fully qualified paths - a.b.c.f vs f (different effective names, no conflict) + { + name: "different_fq_paths_no_conflict", + mappingStr: ` + { + "default_mapping": { + "properties": { + "a": { + "properties": { + "b": { + "properties": { + "c": { + "fields": [ + { + "name": "f", + "type": "vector", + "dims": 64 + } + ] + } + } + } + } + }, + "x": { + "fields": [ + { + "name": "f", + "type": "vector", + "dims": 128 + } + ] + } + } + } + }`, + expValidity: true, + errMsgs: []string{}, + }, + // Test 21: Same leaf property name at different paths (a.b.vec vs x.y.vec) - no conflict + { + name: "same_leaf_different_paths_no_conflict", + mappingStr: ` + { + "default_mapping": { + "properties": { + "a": { + "properties": { + "b": { + "properties": { + "vec": { + "fields": [ + { + "type": "vector", + "dims": 64 + } + ] + } + } + } + } + }, + "x": { + "properties": { + "y": { + "properties": { + "vec": { + "fields": [ + { + "type": "vector", + "dims": 128 + } + ] + } + } + } + } + } + } + } + }`, + expValidity: true, + errMsgs: []string{}, + }, + // Test 22: Field name override creates same effective name - alias conflict + // a.b with name "data" → effective "a.data" + // a with name "data" → effective "data" + // These are different, so no conflict + { + name: "field_name_override_different_parents_no_conflict", + mappingStr: ` + { + "default_mapping": { + "properties": { + "a": { + "properties": { + "b": { + "fields": [ + { + "name": "data", + "type": "vector", + "dims": 64 + } + ] + } + } + }, + "a2": { + "fields": [ + { + "name": "data", + "type": "vector", + "dims": 128 + } + ] + } + } + } + }`, + expValidity: true, + errMsgs: []string{}, + }, + // Test 23: Same effective field name via name override - should conflict + // a.b with name "sharedVec" → effective "a.sharedVec" + // a.c with name "sharedVec" → effective "a.sharedVec" + // Both resolve to same effective name with different dims → conflict + { + name: "same_effective_name_via_override_conflict", + mappingStr: ` + { + "default_mapping": { + "properties": { + "a": { + "properties": { + "b": { + "fields": [ + { + "name": "sharedVec", + "type": "vector", + "dims": 64 + } + ] + }, + "c": { + "fields": [ + { + "name": "sharedVec", + "type": "vector", + "dims": 128 + } + ] + } + } + } + } + } + }`, + expValidity: false, + errMsgs: []string{ + `field: 'a.sharedVec', invalid alias (different dimensions 128 and 64)`, + `field: 'a.sharedVec', invalid alias (different dimensions 64 and 128)`, + }, + }, + // Test 24: Deep nesting with same effective name via name override - should conflict + // level1.level2.propA with name "vec" → effective "level1.level2.vec" + // level1.level2.propB with name "vec" → effective "level1.level2.vec" + { + name: "deep_nesting_same_effective_name_conflict", + mappingStr: ` + { + "default_mapping": { + "properties": { + "level1": { + "properties": { + "level2": { + "properties": { + "propA": { + "fields": [ + { + "name": "vec", + "type": "vector", + "dims": 64 + } + ] + }, + "propB": { + "fields": [ + { + "name": "vec", + "type": "vector", + "dims": 128 + } + ] + } + } + } + } + } + } + } + }`, + expValidity: false, + errMsgs: []string{ + `field: 'level1.level2.vec', invalid alias (different dimensions 128 and 64)`, + `field: 'level1.level2.vec', invalid alias (different dimensions 64 and 128)`, + }, + }, + // Test 25: Root level field vs nested field with same name - no conflict + // Root: "embedding" → effective "embedding" + // Nested: a.b.embedding → effective "a.b.embedding" + { + name: "root_vs_nested_same_name_no_conflict", + mappingStr: ` + { + "default_mapping": { + "properties": { + "embedding": { + "fields": [ + { + "type": "vector", + "dims": 64 + } + ] + }, + "nested": { + "properties": { + "deep": { + "properties": { + "embedding": { + "fields": [ + { + "type": "vector", + "dims": 256 + } + ] + } + } + } + } + } + } + } + }`, + expValidity: true, + errMsgs: []string{}, + }, + // Test 26: Multiple levels with name override targeting same effective path + // a.b.x with name "target" → effective "a.b.target" + // a.b.target (no override) → effective "a.b.target" + // Same effective name, different dims → conflict + { + name: "name_override_matches_sibling_path_conflict", + mappingStr: ` + { + "default_mapping": { + "properties": { + "a": { + "properties": { + "b": { + "properties": { + "x": { + "fields": [ + { + "name": "target", + "type": "vector", + "dims": 64 + } + ] + }, + "target": { + "fields": [ + { + "type": "vector", + "dims": 128 + } + ] + } + } + } + } + } + } + } + }`, + expValidity: false, + errMsgs: []string{ + `field: 'a.b.target', invalid alias (different dimensions 128 and 64)`, + `field: 'a.b.target', invalid alias (different dimensions 64 and 128)`, + }, + }, + // Test 27: Valid alias at deep nesting level + { + name: "valid_alias_deep_nesting", + mappingStr: ` + { + "default_mapping": { + "properties": { + "a": { + "properties": { + "b": { + "properties": { + "c": { + "properties": { + "vec": { + "fields": [ + { + "type": "vector", + "dims": 128, + "similarity": "dot_product" + }, + { + "name": "vec", + "type": "vector", + "dims": 128, + "similarity": "dot_product" + } + ] + } + } + } + } + } + } + } + } + } + }`, + expValidity: true, + errMsgs: []string{}, + }, + // Test 28: Valid alias with different paths but same effective field name + // vectors.vec with name "vec" → effective "vectors.vec" + // vec with name "vec" → effective "vec" + // Different effective names, so no conflict + { + name: "valid_alias_different_paths_same_field_name", + mappingStr: ` + { + "default_mapping": { + "dynamic": false, + "enabled": true, + "properties": { + "vectors": { + "dynamic": true, + "enabled": true, + "properties": { + "vec": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "dims": 3, + "index": true, + "name": "vec", + "type": "vector" + } + ] + } + } + }, + "vec": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "dims": 3, + "index": true, + "name": "vec", + "similarity": "l2_norm", + "type": "vector", + "vector_index_optimized_for": "recall" + } + ] + } + } + } + }`, + expValidity: true, + errMsgs: []string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + im := NewIndexMapping() + err := im.UnmarshalJSON([]byte(test.mappingStr)) + if err != nil { + t.Fatalf("failed to unmarshal index mapping: %v", err) + } + + err = im.Validate() + isValid := err == nil + if test.expValidity != isValid { + t.Fatalf("validity mismatch, expected: %v, got: %v", + test.expValidity, isValid) + } + + if !isValid { + errStringMatched := false + for _, possibleErrMsg := range test.errMsgs { + // Use Contains for matching since some error messages include + // map keys which have non-deterministic ordering + if err.Error() == possibleErrMsg || strings.Contains(err.Error(), possibleErrMsg) { + errStringMatched = true + break + } + } + if !errStringMatched { + t.Fatalf("invalid error message, expected one of: %v, got: %v", + test.errMsgs, err.Error()) + } + } + }) + } +} + +// A test case for processVector function +type vectorTest struct { + // Input + + ipVec interface{} // input vector + dims int // dimensionality of input vector + + // Expected Output + + expValidity bool // expected validity of the input + expOpVec []float32 // expected output vector, given the input is valid +} + +func TestProcessVector(t *testing.T) { + // Note: while creating vectors, we are using []any instead of []float32, + // this is done to enhance our test coverage. + // When we unmarshal a vector from a JSON, we get []any, not []float32. + tests := []vectorTest{ + // # Flat vectors + + // ## numeric cases + // (all numeric elements) + {[]any{1, 2.2, 3}, 3, true, []float32{1, 2.2, 3}}, // len==dims + {[]any{1, 2.2, 3}, 2, false, nil}, // len>dims + {[]any{1, 2.2, 3}, 4, false, nil}, // lendims + {[]any{[]any{1, 2, 3}}, 2, false, nil}, // len 0 && len(tt.input) > 0 && len(tt.input)%tt.dims == 0 { + numVecs := len(result) / tt.dims + for i := 0; i < numVecs; i++ { + subVec := result[i*tt.dims : (i+1)*tt.dims] + mag := magnitude(subVec) + // Allow for zero vectors (magnitude 0) or unit vectors (magnitude 1) + if mag > 1e-6 && !floatApproxEqual(mag, 1.0, 1e-5) { + t.Errorf("sub-vector %d has magnitude %v, expected 1.0", i, mag) + } + } + } + }) + } +} + +// Helper to compute magnitude of a vector +func magnitude(v []float32) float32 { + var sum float32 + for _, x := range v { + sum += x * x + } + return float32(math.Sqrt(float64(sum))) +} + +// Helper for approximate float comparison +func floatApproxEqual(a, b, epsilon float32) bool { + diff := a - b + if diff < 0 { + diff = -diff + } + return diff < epsilon +} diff --git a/mapping/reflect.go b/mapping/reflect.go new file mode 100644 index 0000000..6500a70 --- /dev/null +++ b/mapping/reflect.go @@ -0,0 +1,92 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapping + +import ( + "reflect" + "strings" +) + +func lookupPropertyPath(data interface{}, path string) interface{} { + pathParts := decodePath(path) + + current := data + for _, part := range pathParts { + current = lookupPropertyPathPart(current, part) + if current == nil { + break + } + } + + return current +} + +func lookupPropertyPathPart(data interface{}, part string) interface{} { + val := reflect.ValueOf(data) + if !val.IsValid() { + return nil + } + typ := val.Type() + switch typ.Kind() { + case reflect.Map: + // FIXME can add support for other map keys in the future + if typ.Key().Kind() == reflect.String { + key := reflect.ValueOf(part) + entry := val.MapIndex(key) + if entry.IsValid() { + return entry.Interface() + } + } + case reflect.Struct: + field := val.FieldByName(part) + if field.IsValid() && field.CanInterface() { + return field.Interface() + } + case reflect.Ptr: + ptrElem := val.Elem() + if ptrElem.IsValid() && ptrElem.CanInterface() { + return lookupPropertyPathPart(ptrElem.Interface(), part) + } + } + return nil +} + +const pathSeparator = "." + +func decodePath(path string) []string { + return strings.Split(path, pathSeparator) +} + +func encodePath(pathElements []string) string { + return strings.Join(pathElements, pathSeparator) +} + +func mustString(data interface{}) (string, bool) { + if data != nil { + str, ok := data.(string) + if ok { + return str, true + } + } + return "", false +} + +// parseTagName extracts the field name from a struct tag +func parseTagName(tag string) string { + if idx := strings.Index(tag, ","); idx != -1 { + return tag[:idx] + } + return tag +} diff --git a/mapping/reflect_test.go b/mapping/reflect_test.go new file mode 100644 index 0000000..fdba06f --- /dev/null +++ b/mapping/reflect_test.go @@ -0,0 +1,47 @@ +package mapping + +import ( + "reflect" + "testing" +) + +func TestLookupPropertyPath(t *testing.T) { + tests := []struct { + input interface{} + path string + output interface{} + }{ + { + input: map[string]interface{}{ + "Type": "a", + }, + path: "Type", + output: "a", + }, + { + input: struct { + Type string + }{ + Type: "b", + }, + path: "Type", + output: "b", + }, + { + input: &struct { + Type string + }{ + Type: "b", + }, + path: "Type", + output: "b", + }, + } + + for _, test := range tests { + actual := lookupPropertyPath(test.input, test.path) + if !reflect.DeepEqual(actual, test.output) { + t.Fatalf("expected '%v', got '%v', for path '%s' in %+v", test.output, actual, test.path, test.input) + } + } +} diff --git a/mapping/synonym.go b/mapping/synonym.go new file mode 100644 index 0000000..1982821 --- /dev/null +++ b/mapping/synonym.go @@ -0,0 +1,71 @@ +// Copyright (c) 2024 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapping + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +type SynonymSource struct { + CollectionName string `json:"collection"` + AnalyzerName string `json:"analyzer"` +} + +func NewSynonymSource(collection, analyzer string) *SynonymSource { + return &SynonymSource{ + CollectionName: collection, + AnalyzerName: analyzer, + } +} + +func (s *SynonymSource) Collection() string { + return s.CollectionName +} + +func (s *SynonymSource) Analyzer() string { + return s.AnalyzerName +} + +func (s *SynonymSource) SetCollection(c string) { + s.CollectionName = c +} + +func (s *SynonymSource) SetAnalyzer(a string) { + s.AnalyzerName = a +} +func SynonymSourceConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.SynonymSource, error) { + collection, ok := config["collection"].(string) + if !ok { + return nil, fmt.Errorf("must specify collection") + } + analyzer, ok := config["analyzer"].(string) + if !ok { + return nil, fmt.Errorf("must specify analyzer") + } + if _, err := cache.AnalyzerNamed(analyzer); err != nil { + return nil, fmt.Errorf("analyzer named '%s' not found", analyzer) + } + return NewSynonymSource(collection, analyzer), nil +} + +func init() { + err := registry.RegisterSynonymSource(analysis.SynonymSourceType, SynonymSourceConstructor) + if err != nil { + panic(err) + } +} diff --git a/mapping_vector.go b/mapping_vector.go new file mode 100644 index 0000000..c73dac9 --- /dev/null +++ b/mapping_vector.go @@ -0,0 +1,28 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package bleve + +import "github.com/blevesearch/bleve/v2/mapping" + +func NewVectorFieldMapping() *mapping.FieldMapping { + return mapping.NewVectorFieldMapping() +} + +func NewVectorBase64FieldMapping() *mapping.FieldMapping { + return mapping.NewVectorBase64FieldMapping() +} diff --git a/numeric/bin.go b/numeric/bin.go new file mode 100644 index 0000000..eee0ff5 --- /dev/null +++ b/numeric/bin.go @@ -0,0 +1,43 @@ +package numeric + +var interleaveMagic = []uint64{ + 0x5555555555555555, + 0x3333333333333333, + 0x0F0F0F0F0F0F0F0F, + 0x00FF00FF00FF00FF, + 0x0000FFFF0000FFFF, + 0x00000000FFFFFFFF, + 0xAAAAAAAAAAAAAAAA, +} + +var interleaveShift = []uint{1, 2, 4, 8, 16} + +// Interleave the first 32 bits of each uint64 +// adapted from org.apache.lucene.util.BitUtil +// which was adapted from: +// http://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN +func Interleave(v1, v2 uint64) uint64 { + v1 = (v1 | (v1 << interleaveShift[4])) & interleaveMagic[4] + v1 = (v1 | (v1 << interleaveShift[3])) & interleaveMagic[3] + v1 = (v1 | (v1 << interleaveShift[2])) & interleaveMagic[2] + v1 = (v1 | (v1 << interleaveShift[1])) & interleaveMagic[1] + v1 = (v1 | (v1 << interleaveShift[0])) & interleaveMagic[0] + v2 = (v2 | (v2 << interleaveShift[4])) & interleaveMagic[4] + v2 = (v2 | (v2 << interleaveShift[3])) & interleaveMagic[3] + v2 = (v2 | (v2 << interleaveShift[2])) & interleaveMagic[2] + v2 = (v2 | (v2 << interleaveShift[1])) & interleaveMagic[1] + v2 = (v2 | (v2 << interleaveShift[0])) & interleaveMagic[0] + return (v2 << 1) | v1 +} + +// Deinterleave the 32-bit value starting at position 0 +// to get the other 32-bit value, shift it by 1 first +func Deinterleave(b uint64) uint64 { + b &= interleaveMagic[0] + b = (b ^ (b >> interleaveShift[0])) & interleaveMagic[1] + b = (b ^ (b >> interleaveShift[1])) & interleaveMagic[2] + b = (b ^ (b >> interleaveShift[2])) & interleaveMagic[3] + b = (b ^ (b >> interleaveShift[3])) & interleaveMagic[4] + b = (b ^ (b >> interleaveShift[4])) & interleaveMagic[5] + return b +} diff --git a/numeric/bin_test.go b/numeric/bin_test.go new file mode 100644 index 0000000..f6dfb47 --- /dev/null +++ b/numeric/bin_test.go @@ -0,0 +1,27 @@ +package numeric + +import "testing" + +func TestInterleaveDeinterleave(t *testing.T) { + tests := []struct { + v1 uint64 + v2 uint64 + }{ + {0, 0}, + {1, 1}, + {27, 39}, + {1<<32 - 1, 1<<32 - 1}, // largest that should still work + } + + for _, test := range tests { + i := Interleave(test.v1, test.v2) + gotv1 := Deinterleave(i) + gotv2 := Deinterleave(i >> 1) + if gotv1 != test.v1 { + t.Errorf("expected v1: %d, got %d, interleaved was %x", test.v1, gotv1, i) + } + if gotv2 != test.v2 { + t.Errorf("expected v2: %d, got %d, interleaved was %x", test.v2, gotv2, i) + } + } +} diff --git a/numeric/float.go b/numeric/float.go new file mode 100644 index 0000000..2bb14d7 --- /dev/null +++ b/numeric/float.go @@ -0,0 +1,34 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package numeric + +import ( + "math" +) + +func Float64ToInt64(f float64) int64 { + fasint := int64(math.Float64bits(f)) + if fasint < 0 { + fasint = fasint ^ 0x7fffffffffffffff + } + return fasint +} + +func Int64ToFloat64(i int64) float64 { + if i < 0 { + i ^= 0x7fffffffffffffff + } + return math.Float64frombits(uint64(i)) +} diff --git a/numeric/float_test.go b/numeric/float_test.go new file mode 100644 index 0000000..b5e99a9 --- /dev/null +++ b/numeric/float_test.go @@ -0,0 +1,64 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package numeric + +import ( + "testing" +) + +// test that the float/sortable int operations work both ways +// and that the corresponding integers sort the same as +// the original floats would have +func TestSortabledFloat64ToInt64(t *testing.T) { + tests := []struct { + input float64 + }{ + { + input: -4640094584139352638, + }, + { + input: -167.42, + }, + { + input: -1.11, + }, + { + input: 0, + }, + { + input: 3.14, + }, + { + input: 167.42, + }, + } + + var lastInt64 *int64 + for _, test := range tests { + actual := Float64ToInt64(test.input) + if lastInt64 != nil { + // check that this float is greater than the last one + if actual <= *lastInt64 { + t.Errorf("expected greater than prev, this: %d, last %d", actual, *lastInt64) + } + } + lastInt64 = &actual + convertedBack := Int64ToFloat64(actual) + // assert that we got back what we started with + if convertedBack != test.input { + t.Errorf("expected %f, got %f", test.input, convertedBack) + } + } +} diff --git a/numeric/prefix_coded.go b/numeric/prefix_coded.go new file mode 100644 index 0000000..03ba043 --- /dev/null +++ b/numeric/prefix_coded.go @@ -0,0 +1,119 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package numeric + +import "fmt" + +const ShiftStartInt64 byte = 0x20 + +// PrefixCoded is a byte array encoding of +// 64-bit numeric values shifted by 0-63 bits +type PrefixCoded []byte + +func NewPrefixCodedInt64(in int64, shift uint) (PrefixCoded, error) { + rv, _, err := NewPrefixCodedInt64Prealloc(in, shift, nil) + return rv, err +} + +func NewPrefixCodedInt64Prealloc(in int64, shift uint, prealloc []byte) ( + rv PrefixCoded, preallocRest []byte, err error) { + if shift > 63 { + return nil, prealloc, fmt.Errorf("cannot shift %d, must be between 0 and 63", shift) + } + + nChars := ((63 - shift) / 7) + 1 + + size := int(nChars + 1) + if len(prealloc) >= size { + rv = PrefixCoded(prealloc[0:size]) + preallocRest = prealloc[size:] + } else { + rv = make(PrefixCoded, size) + } + + rv[0] = ShiftStartInt64 + byte(shift) + + sortableBits := int64(uint64(in) ^ 0x8000000000000000) + sortableBits = int64(uint64(sortableBits) >> shift) + for nChars > 0 { + // Store 7 bits per byte for compatibility + // with UTF-8 encoding of terms + rv[nChars] = byte(sortableBits & 0x7f) + nChars-- + sortableBits = int64(uint64(sortableBits) >> 7) + } + + return rv, preallocRest, nil +} + +func MustNewPrefixCodedInt64(in int64, shift uint) PrefixCoded { + rv, err := NewPrefixCodedInt64(in, shift) + if err != nil { + panic(err) + } + return rv +} + +func MustNewPrefixCodedInt64Prealloc(in int64, shift uint, prealloc []byte) PrefixCoded { + rv, _, err := NewPrefixCodedInt64Prealloc(in, shift, prealloc) + if err != nil { + panic(err) + } + return rv +} + +// Shift returns the number of bits shifted +// returns 0 if in uninitialized state +func (p PrefixCoded) Shift() (uint, error) { + if len(p) > 0 { + shift := p[0] - ShiftStartInt64 + if shift < 0 || shift < 63 { + return uint(shift), nil + } + } + return 0, fmt.Errorf("invalid prefix coded value") +} + +func (p PrefixCoded) Int64() (int64, error) { + shift, err := p.Shift() + if err != nil { + return 0, err + } + var sortableBits int64 + for _, inbyte := range p[1:] { + sortableBits <<= 7 + sortableBits |= int64(inbyte) + } + return int64(uint64((sortableBits << shift)) ^ 0x8000000000000000), nil +} + +func ValidPrefixCodedTerm(p string) (bool, int) { + return ValidPrefixCodedTermBytes([]byte(p)) +} + +func ValidPrefixCodedTermBytes(p []byte) (bool, int) { + if len(p) > 0 { + if p[0] < ShiftStartInt64 || p[0] > ShiftStartInt64+63 { + return false, 0 + } + shift := p[0] - ShiftStartInt64 + nChars := ((63 - int(shift)) / 7) + 1 + if len(p) != nChars+1 { + return false, 0 + } + return true, int(shift) + } + return false, 0 +} diff --git a/numeric/prefix_coded_test.go b/numeric/prefix_coded_test.go new file mode 100644 index 0000000..9256c91 --- /dev/null +++ b/numeric/prefix_coded_test.go @@ -0,0 +1,158 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package numeric + +import ( + "reflect" + "testing" +) + +var tests = []struct { + input int64 + shift uint + output PrefixCoded +}{ + { + input: 1, + shift: 0, + output: PrefixCoded{0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, + }, + { + input: -1, + shift: 0, + output: PrefixCoded{0x20, 0x0, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f}, + }, + { + input: -94582, + shift: 0, + output: PrefixCoded{0x20, 0x0, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7a, 0x1d, 0xa}, + }, + { + input: 314729851, + shift: 0, + output: PrefixCoded{0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x1, 0x16, 0x9, 0x4a, 0x7b}, + }, + { + input: 314729851, + shift: 4, + output: PrefixCoded{0x24, 0x8, 0x0, 0x0, 0x0, 0x0, 0x9, 0x30, 0x4c, 0x57}, + }, + { + input: 314729851, + shift: 8, + output: PrefixCoded{0x28, 0x40, 0x0, 0x0, 0x0, 0x0, 0x4b, 0x4, 0x65}, + }, + { + input: 314729851, + shift: 16, + output: PrefixCoded{0x30, 0x20, 0x0, 0x0, 0x0, 0x0, 0x25, 0x42}, + }, + { + input: 314729851, + shift: 32, + output: PrefixCoded{0x40, 0x8, 0x0, 0x0, 0x0, 0x0}, + }, + { + input: 1234729851, + shift: 32, + output: PrefixCoded{0x40, 0x8, 0x0, 0x0, 0x0, 0x0}, + }, +} + +// these array encoding values have been verified manually +// against the lucene implementation +func TestPrefixCoded(t *testing.T) { + + for _, test := range tests { + actual, err := NewPrefixCodedInt64(test.input, test.shift) + if err != nil { + t.Error(err) + } + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected %#v, got %#v", test.output, actual) + } + checkedShift, err := actual.Shift() + if err != nil { + t.Error(err) + } + if checkedShift != test.shift { + t.Errorf("expected %d, got %d", test.shift, checkedShift) + } + // if the shift was 0, make sure we can go back to the original + if test.shift == 0 { + backToLong, err := actual.Int64() + if err != nil { + t.Error(err) + } + if backToLong != test.input { + t.Errorf("expected %v, got %v", test.input, backToLong) + } + } + } +} + +func TestPrefixCodedValid(t *testing.T) { + // all of the shared tests should be valid + for _, test := range tests { + valid, _ := ValidPrefixCodedTerm(string(test.output)) + if !valid { + t.Errorf("expected %s to be valid prefix coded, is not", string(test.output)) + } + } + + invalidTests := []struct { + data PrefixCoded + }{ + // first byte invalid skip (too low) + { + data: PrefixCoded{0x19, 'c', 'a', 't'}, + }, + // first byte invalid skip (too high) + { + data: PrefixCoded{0x20 + 64, 'c'}, + }, + // length of trailing bytes wrong (too long) + { + data: PrefixCoded{0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, + }, + // length of trailing bytes wrong (too short) + { + data: PrefixCoded{0x20 + 63}, + }, + } + + // all of the shared tests should be valid + for _, test := range invalidTests { + valid, _ := ValidPrefixCodedTerm(string(test.data)) + if valid { + t.Errorf("expected %s to be invalid prefix coded, it is", string(test.data)) + } + } +} + +func BenchmarkTestPrefixCoded(b *testing.B) { + + for i := 0; i < b.N; i++ { + for _, test := range tests { + actual, err := NewPrefixCodedInt64(test.input, test.shift) + if err != nil { + b.Error(err) + } + if !reflect.DeepEqual(actual, test.output) { + b.Errorf("expected %#v, got %#v", test.output, actual) + } + } + } +} diff --git a/pre_search.go b/pre_search.go new file mode 100644 index 0000000..3dd7e0f --- /dev/null +++ b/pre_search.go @@ -0,0 +1,170 @@ +// Copyright (c) 2024 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "github.com/blevesearch/bleve/v2/search" +) + +// A preSearchResultProcessor processes the data in +// the preSearch result from multiple +// indexes in an alias and merges them together to +// create the final preSearch result +type preSearchResultProcessor interface { + // adds the preSearch result to the processor + add(*SearchResult, string) + // updates the final search result with the finalized + // data from the processor + finalize(*SearchResult) +} + +// ----------------------------------------------------------------------------- +// KNN preSearchResultProcessor for handling KNN presearch results +type knnPreSearchResultProcessor struct { + addFn func(sr *SearchResult, indexName string) + finalizeFn func(sr *SearchResult) +} + +func (k *knnPreSearchResultProcessor) add(sr *SearchResult, indexName string) { + if k.addFn != nil { + k.addFn(sr, indexName) + } +} + +func (k *knnPreSearchResultProcessor) finalize(sr *SearchResult) { + if k.finalizeFn != nil { + k.finalizeFn(sr) + } +} + +// ----------------------------------------------------------------------------- +// Synonym preSearchResultProcessor for handling Synonym presearch results +type synonymPreSearchResultProcessor struct { + finalizedFts search.FieldTermSynonymMap +} + +func newSynonymPreSearchResultProcessor() *synonymPreSearchResultProcessor { + return &synonymPreSearchResultProcessor{} +} + +func (s *synonymPreSearchResultProcessor) add(sr *SearchResult, indexName string) { + // Check if SynonymResult or the synonym data key is nil + if sr.SynonymResult == nil { + return + } + + // Attempt to cast PreSearchResults to FieldTermSynonymMap + + // Merge with finalizedFts or initialize it if nil + if s.finalizedFts == nil { + s.finalizedFts = sr.SynonymResult + } else { + s.finalizedFts.MergeWith(sr.SynonymResult) + } +} + +func (s *synonymPreSearchResultProcessor) finalize(sr *SearchResult) { + // Set the finalized synonym data to the PreSearchResults + if s.finalizedFts != nil { + sr.SynonymResult = s.finalizedFts + } +} + +type bm25PreSearchResultProcessor struct { + docCount float64 // bm25 specific stats + fieldCardinality map[string]int +} + +func newBM25PreSearchResultProcessor() *bm25PreSearchResultProcessor { + return &bm25PreSearchResultProcessor{ + fieldCardinality: make(map[string]int), + } +} + +// TODO How will this work for queries other than term queries? +func (b *bm25PreSearchResultProcessor) add(sr *SearchResult, indexName string) { + if sr.BM25Stats != nil { + b.docCount += sr.BM25Stats.DocCount + for field, cardinality := range sr.BM25Stats.FieldCardinality { + b.fieldCardinality[field] += cardinality + } + } +} + +func (b *bm25PreSearchResultProcessor) finalize(sr *SearchResult) { + sr.BM25Stats = &search.BM25Stats{ + DocCount: b.docCount, + FieldCardinality: b.fieldCardinality, + } +} + +// ----------------------------------------------------------------------------- +// Master struct that can hold any number of presearch result processors +type compositePreSearchResultProcessor struct { + presearchResultProcessors []preSearchResultProcessor +} + +// Implements the add method, which forwards to all the internal processors +func (m *compositePreSearchResultProcessor) add(sr *SearchResult, indexName string) { + for _, p := range m.presearchResultProcessors { + p.add(sr, indexName) + } +} + +// Implements the finalize method, which forwards to all the internal processors +func (m *compositePreSearchResultProcessor) finalize(sr *SearchResult) { + for _, p := range m.presearchResultProcessors { + p.finalize(sr) + } +} + +// ----------------------------------------------------------------------------- +// Function to create the appropriate preSearchResultProcessor(s) +func createPreSearchResultProcessor(req *SearchRequest, flags *preSearchFlags) preSearchResultProcessor { + // return nil for invalid input + if flags == nil || req == nil { + return nil + } + var processors []preSearchResultProcessor + // Add KNN processor if the request has KNN + if flags.knn { + if knnProcessor := newKnnPreSearchResultProcessor(req); knnProcessor != nil { + processors = append(processors, knnProcessor) + } + } + // Add Synonym processor if the request has Synonym + if flags.synonyms { + if synonymProcessor := newSynonymPreSearchResultProcessor(); synonymProcessor != nil { + processors = append(processors, synonymProcessor) + } + } + if flags.bm25 { + if bm25Processtor := newBM25PreSearchResultProcessor(); bm25Processtor != nil { + processors = append(processors, bm25Processtor) + } + } + // Return based on the number of processors, optimizing for the common case of 1 processor + // If there are no processors, return nil + switch len(processors) { + case 0: + return nil + case 1: + return processors[0] + default: + return &compositePreSearchResultProcessor{ + presearchResultProcessors: processors, + } + } +} diff --git a/query.go b/query.go new file mode 100644 index 0000000..93e662b --- /dev/null +++ b/query.go @@ -0,0 +1,290 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "time" + + "github.com/blevesearch/bleve/v2/search/query" +) + +// NewBoolFieldQuery creates a new Query for boolean fields +func NewBoolFieldQuery(val bool) *query.BoolFieldQuery { + return query.NewBoolFieldQuery(val) +} + +// NewBooleanQuery creates a compound Query composed +// of several other Query objects. +// These other query objects are added using the +// AddMust() AddShould() and AddMustNot() methods. +// Result documents must satisfy ALL of the +// must Queries. +// Result documents must satisfy NONE of the must not +// Queries. +// Result documents that ALSO satisfy any of the should +// Queries will score higher. +func NewBooleanQuery() *query.BooleanQuery { + return query.NewBooleanQuery(nil, nil, nil) +} + +// NewConjunctionQuery creates a new compound Query. +// Result documents must satisfy all of the queries. +func NewConjunctionQuery(conjuncts ...query.Query) *query.ConjunctionQuery { + return query.NewConjunctionQuery(conjuncts) +} + +// NewDateRangeQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser configured in the +// +// top-level config.QueryDateTimeParser +// +// Either, but not both endpoints can be nil. +func NewDateRangeQuery(start, end time.Time) *query.DateRangeQuery { + return query.NewDateRangeQuery(start, end) +} + +// NewDateRangeInclusiveQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser configured in the +// +// top-level config.QueryDateTimeParser +// +// Either, but not both endpoints can be nil. +// startInclusive and endInclusive control inclusion of the endpoints. +func NewDateRangeInclusiveQuery(start, end time.Time, startInclusive, endInclusive *bool) *query.DateRangeQuery { + return query.NewDateRangeInclusiveQuery(start, end, startInclusive, endInclusive) +} + +// NewDateRangeStringQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser set using +// +// the DateRangeStringQuery.SetDateTimeParser() method. +// +// If no DateTimeParser is set, then the +// +// top-level config.QueryDateTimeParser +// +// is used. +func NewDateRangeStringQuery(start, end string) *query.DateRangeStringQuery { + return query.NewDateRangeStringQuery(start, end) +} + +// NewDateRangeInclusiveStringQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser set using +// +// the DateRangeStringQuery.SetDateTimeParser() method. +// +// this DateTimeParser is a custom date time parser defined in the index mapping, +// using AddCustomDateTimeParser() method. +// If no DateTimeParser is set, then the +// +// top-level config.QueryDateTimeParser +// +// is used. +// Either, but not both endpoints can be nil. +// startInclusive and endInclusive control inclusion of the endpoints. +func NewDateRangeInclusiveStringQuery(start, end string, startInclusive, endInclusive *bool) *query.DateRangeStringQuery { + return query.NewDateRangeStringInclusiveQuery(start, end, startInclusive, endInclusive) +} + +// NewDisjunctionQuery creates a new compound Query. +// Result documents satisfy at least one Query. +func NewDisjunctionQuery(disjuncts ...query.Query) *query.DisjunctionQuery { + return query.NewDisjunctionQuery(disjuncts) +} + +// NewDocIDQuery creates a new Query object returning indexed documents among +// the specified set. Combine it with ConjunctionQuery to restrict the scope of +// other queries output. +func NewDocIDQuery(ids []string) *query.DocIDQuery { + return query.NewDocIDQuery(ids) +} + +// NewFuzzyQuery creates a new Query which finds +// documents containing terms within a specific +// fuzziness of the specified term. +// The default fuzziness is 1. +// +// The current implementation uses Levenshtein edit +// distance as the fuzziness metric. +func NewFuzzyQuery(term string) *query.FuzzyQuery { + return query.NewFuzzyQuery(term) +} + +// NewMatchAllQuery creates a Query which will +// match all documents in the index. +func NewMatchAllQuery() *query.MatchAllQuery { + return query.NewMatchAllQuery() +} + +// NewMatchNoneQuery creates a Query which will not +// match any documents in the index. +func NewMatchNoneQuery() *query.MatchNoneQuery { + return query.NewMatchNoneQuery() +} + +// NewMatchPhraseQuery creates a new Query object +// for matching phrases in the index. +// An Analyzer is chosen based on the field. +// Input text is analyzed using this analyzer. +// Token terms resulting from this analysis are +// used to build a search phrase. Result documents +// must match this phrase. Queried field must have been indexed with +// IncludeTermVectors set to true. +func NewMatchPhraseQuery(matchPhrase string) *query.MatchPhraseQuery { + return query.NewMatchPhraseQuery(matchPhrase) +} + +// NewMatchQuery creates a Query for matching text. +// An Analyzer is chosen based on the field. +// Input text is analyzed using this analyzer. +// Token terms resulting from this analysis are +// used to perform term searches. Result documents +// must satisfy at least one of these term searches. +func NewMatchQuery(match string) *query.MatchQuery { + return query.NewMatchQuery(match) +} + +// NewNumericRangeQuery creates a new Query for ranges +// of numeric values. +// Either, but not both endpoints can be nil. +// The minimum value is inclusive. +// The maximum value is exclusive. +func NewNumericRangeQuery(min, max *float64) *query.NumericRangeQuery { + return query.NewNumericRangeQuery(min, max) +} + +// NewNumericRangeInclusiveQuery creates a new Query for ranges +// of numeric values. +// Either, but not both endpoints can be nil. +// Control endpoint inclusion with inclusiveMin, inclusiveMax. +func NewNumericRangeInclusiveQuery(min, max *float64, minInclusive, maxInclusive *bool) *query.NumericRangeQuery { + return query.NewNumericRangeInclusiveQuery(min, max, minInclusive, maxInclusive) +} + +// NewTermRangeQuery creates a new Query for ranges +// of text terms. +// Either, but not both endpoints can be "". +// The minimum value is inclusive. +// The maximum value is exclusive. +func NewTermRangeQuery(min, max string) *query.TermRangeQuery { + return query.NewTermRangeQuery(min, max) +} + +// NewTermRangeInclusiveQuery creates a new Query for ranges +// of text terms. +// Either, but not both endpoints can be "". +// Control endpoint inclusion with inclusiveMin, inclusiveMax. +func NewTermRangeInclusiveQuery(min, max string, minInclusive, maxInclusive *bool) *query.TermRangeQuery { + return query.NewTermRangeInclusiveQuery(min, max, minInclusive, maxInclusive) +} + +// NewPhraseQuery creates a new Query for finding +// exact term phrases in the index. +// The provided terms must exist in the correct +// order, at the correct index offsets, in the +// specified field. Queried field must have been indexed with +// IncludeTermVectors set to true. +func NewPhraseQuery(terms []string, field string) *query.PhraseQuery { + return query.NewPhraseQuery(terms, field) +} + +// NewPrefixQuery creates a new Query which finds +// documents containing terms that start with the +// specified prefix. +func NewPrefixQuery(prefix string) *query.PrefixQuery { + return query.NewPrefixQuery(prefix) +} + +// NewRegexpQuery creates a new Query which finds +// documents containing terms that match the +// specified regular expression. +func NewRegexpQuery(regexp string) *query.RegexpQuery { + return query.NewRegexpQuery(regexp) +} + +// NewQueryStringQuery creates a new Query used for +// finding documents that satisfy a query string. The +// query string is a small query language for humans. +func NewQueryStringQuery(q string) *query.QueryStringQuery { + return query.NewQueryStringQuery(q) +} + +// NewTermQuery creates a new Query for finding an +// exact term match in the index. +func NewTermQuery(term string) *query.TermQuery { + return query.NewTermQuery(term) +} + +// NewWildcardQuery creates a new Query which finds +// documents containing terms that match the +// specified wildcard. In the wildcard pattern '*' +// will match any sequence of 0 or more characters, +// and '?' will match any single character. +func NewWildcardQuery(wildcard string) *query.WildcardQuery { + return query.NewWildcardQuery(wildcard) +} + +// NewGeoBoundingBoxQuery creates a new Query for performing geo bounding +// box searches. The arguments describe the position of the box and documents +// which have an indexed geo point inside the box will be returned. +func NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightLat float64) *query.GeoBoundingBoxQuery { + return query.NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightLat) +} + +// NewGeoDistanceQuery creates a new Query for performing geo distance +// searches. The arguments describe a position and a distance. Documents +// which have an indexed geo point which is less than or equal to the provided +// distance from the given position will be returned. +func NewGeoDistanceQuery(lon, lat float64, distance string) *query.GeoDistanceQuery { + return query.NewGeoDistanceQuery(lon, lat, distance) +} + +// NewIPRangeQuery creates a new Query for matching IP addresses. +// If the argument is in CIDR format, then the query will match all +// IP addresses in the network specified. If the argument is an IP address, +// then the query will return documents which contain that IP. +// Both ipv4 and ipv6 are supported. +func NewIPRangeQuery(cidr string) *query.IPRangeQuery { + return query.NewIPRangeQuery(cidr) +} + +// NewGeoShapeQuery creates a new Query for matching the given geo shape. +// This method can be used for creating geoshape queries for shape types +// like: point, linestring, polygon, multipoint, multilinestring, +// multipolygon and envelope. +func NewGeoShapeQuery(coordinates [][][][]float64, typ, relation string) (*query.GeoShapeQuery, error) { + return query.NewGeoShapeQuery(coordinates, typ, relation) +} + +// NewGeoShapeCircleQuery creates a new query for a geoshape that is a +// circle given center point and the radius. Radius formats supported: +// "5in" "5inch" "7yd" "7yards" "9ft" "9feet" "11km" "11kilometers" +// "3nm" "3nauticalmiles" "13mm" "13millimeters" "15cm" "15centimeters" +// "17mi" "17miles" "19m" "19meters" If the unit cannot be determined, +// the entire string is parsed and the unit of meters is assumed. +func NewGeoShapeCircleQuery(coordinates []float64, radius, relation string) (*query.GeoShapeQuery, error) { + return query.NewGeoShapeCircleQuery(coordinates, radius, relation) +} + +// NewGeometryCollectionQuery creates a new query for the provided +// geometrycollection coordinates and types, which could contain +// multiple geo shapes. +func NewGeometryCollectionQuery(coordinates [][][][][]float64, types []string, relation string) (*query.GeoShapeQuery, error) { + return query.NewGeometryCollectionQuery(coordinates, types, relation) +} diff --git a/query_bench_test.go b/query_bench_test.go new file mode 100644 index 0000000..436f2ea --- /dev/null +++ b/query_bench_test.go @@ -0,0 +1,393 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "strconv" + "testing" + "time" + + "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + "github.com/blevesearch/bleve/v2/mapping" +) + +func BenchmarkQueryTerm(b *testing.B) { + tmpIndexPath := createTmpIndexPath(b) + defer cleanupTmpIndexPath(b, tmpIndexPath) + + fm := mapping.NewTextFieldMapping() + fm.Analyzer = keyword.Name + dmap := mapping.NewDocumentMapping() + dmap.AddFieldMappingsAt("text", fm) + imap := mapping.NewIndexMapping() + imap.DefaultMapping = dmap + + idx, err := New(tmpIndexPath, imap) + if err != nil { + b.Fatal(err) + } + + defer func() { + err = idx.Close() + if err != nil { + b.Fatal(err) + } + }() + + members := []string{"abc", "abcdef", "ghi", "jkl", "jklmno"} + for i := 0; i < 100; i++ { + if err = idx.Index(strconv.Itoa(i), + map[string]interface{}{"text": members[i%len(members)]}); err != nil { + b.Fatal(err) + } + } + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + q := NewTermQuery(members[i%len(members)]) + q.SetField("text") + req := NewSearchRequest(q) + if _, err = idx.Search(req); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkQueryTermRange(b *testing.B) { + tmpIndexPath := createTmpIndexPath(b) + defer cleanupTmpIndexPath(b, tmpIndexPath) + + fm := mapping.NewTextFieldMapping() + fm.Analyzer = keyword.Name + dmap := mapping.NewDocumentMapping() + dmap.AddFieldMappingsAt("text", fm) + imap := mapping.NewIndexMapping() + imap.DefaultMapping = dmap + + idx, err := New(tmpIndexPath, imap) + if err != nil { + b.Fatal(err) + } + + defer func() { + err = idx.Close() + if err != nil { + b.Fatal(err) + } + }() + + members := []string{"abc", "abcdef", "ghi", "jkl", "jklmno"} + for i := 0; i < 100; i++ { + if err = idx.Index(strconv.Itoa(i), + map[string]interface{}{"text": members[i%len(members)]}); err != nil { + b.Fatal(err) + } + } + + b.ReportAllocs() + b.ResetTimer() + + inclusive := true + for i := 0; i < b.N; i++ { + q := NewTermRangeInclusiveQuery( + members[i%(len(members)-2)], + members[(i+2)%(len(members)-2)], + &inclusive, + &inclusive, + ) + q.SetField("text") + req := NewSearchRequest(q) + if _, err = idx.Search(req); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkQueryWildcard(b *testing.B) { + tmpIndexPath := createTmpIndexPath(b) + defer cleanupTmpIndexPath(b, tmpIndexPath) + + fm := mapping.NewTextFieldMapping() + fm.Analyzer = keyword.Name + dmap := mapping.NewDocumentMapping() + dmap.AddFieldMappingsAt("text", fm) + imap := mapping.NewIndexMapping() + imap.DefaultMapping = dmap + + idx, err := New(tmpIndexPath, imap) + if err != nil { + b.Fatal(err) + } + + defer func() { + err = idx.Close() + if err != nil { + b.Fatal(err) + } + }() + + members := []string{"abc", "abcdef", "ghi", "jkl", "jklmno"} + for i := 0; i < 100; i++ { + if err = idx.Index(strconv.Itoa(i), + map[string]interface{}{"text": members[i%len(members)]}); err != nil { + b.Fatal(err) + } + } + + wildcards := []string{"ab*", "jk*"} + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + q := NewWildcardQuery(wildcards[i%len(wildcards)]) + q.SetField("text") + req := NewSearchRequest(q) + if _, err = idx.Search(req); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkQueryNumericRange(b *testing.B) { + tmpIndexPath := createTmpIndexPath(b) + defer cleanupTmpIndexPath(b, tmpIndexPath) + + fm := mapping.NewNumericFieldMapping() + dmap := mapping.NewDocumentMapping() + dmap.AddFieldMappingsAt("number", fm) + imap := mapping.NewIndexMapping() + imap.DefaultMapping = dmap + + idx, err := New(tmpIndexPath, imap) + if err != nil { + b.Fatal(err) + } + + defer func() { + err = idx.Close() + if err != nil { + b.Fatal(err) + } + }() + + for i := 0; i < 100; i++ { + if err = idx.Index(strconv.Itoa(i), + map[string]interface{}{"number": i}); err != nil { + b.Fatal(err) + } + } + + b.ReportAllocs() + b.ResetTimer() + + inclusive := true + for i := 0; i < b.N; i++ { + start := float64(i % 90) + end := float64((i + 10) % 90) + q := NewNumericRangeInclusiveQuery(&start, &end, &inclusive, &inclusive) + q.SetField("number") + req := NewSearchRequest(q) + if _, err = idx.Search(req); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkQueryDateRange(b *testing.B) { + tmpIndexPath := createTmpIndexPath(b) + defer cleanupTmpIndexPath(b, tmpIndexPath) + + fm := mapping.NewDateTimeFieldMapping() + dmap := mapping.NewDocumentMapping() + dmap.AddFieldMappingsAt("date", fm) + imap := mapping.NewIndexMapping() + imap.DefaultMapping = dmap + + idx, err := New(tmpIndexPath, imap) + if err != nil { + b.Fatal(err) + } + + defer func() { + err = idx.Close() + if err != nil { + b.Fatal(err) + } + }() + + members := []string{ + "2022-11-16T18:45:45Z", + "2022-11-17T18:45:45Z", + "2022-11-18T18:45:45Z", + "2022-11-19T18:45:45Z", + "2022-11-20T18:45:45Z", + } + for i := 0; i < 100; i++ { + if err = idx.Index(strconv.Itoa(i), + map[string]interface{}{"date": members[i%len(members)]}); err != nil { + b.Fatal(err) + } + } + + b.ReportAllocs() + b.ResetTimer() + + inclusive := true + for i := 0; i < b.N; i++ { + start, _ := time.Parse("2006-01-02T15:04:05Z", members[i%(len(members)-2)]) + end, _ := time.Parse("2006-01-02T15:04:05Z", members[(i+2)%(len(members)-2)]) + q := NewDateRangeInclusiveQuery(start, end, &inclusive, &inclusive) + q.SetField("date") + req := NewSearchRequest(q) + if _, err = idx.Search(req); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkQueryGeoDistance(b *testing.B) { + tmpIndexPath := createTmpIndexPath(b) + defer cleanupTmpIndexPath(b, tmpIndexPath) + + fm := mapping.NewGeoPointFieldMapping() + dmap := mapping.NewDocumentMapping() + dmap.AddFieldMappingsAt("geo", fm) + imap := mapping.NewIndexMapping() + imap.DefaultMapping = dmap + + idx, err := New(tmpIndexPath, imap) + if err != nil { + b.Fatal(err) + } + + defer func() { + err = idx.Close() + if err != nil { + b.Fatal(err) + } + }() + + members := [][]float64{ + {-121.96713072883645, 37.380331474621045}, + {-97.75518866579938, 30.38974491308761}, + {-0.08653451918110022, 51.51063984942306}, + {-2.230759791360498, 53.481514330841236}, + {77.59542326042589, 12.97215865921956}, + } + for i := 0; i < 100; i++ { + if err = idx.Index(strconv.Itoa(i), + map[string]interface{}{"geo": members[i%len(members)]}); err != nil { + b.Fatal(err) + } + } + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + coordinates := members[i%len(members)] + q := NewGeoDistanceQuery(coordinates[0], coordinates[1], "1mi") + q.SetField("geo") + req := NewSearchRequest(q) + if _, err = idx.Search(req); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkQueryGeoBoundingBox(b *testing.B) { + tmpIndexPath := createTmpIndexPath(b) + defer cleanupTmpIndexPath(b, tmpIndexPath) + + fm := mapping.NewGeoPointFieldMapping() + dmap := mapping.NewDocumentMapping() + dmap.AddFieldMappingsAt("geo", fm) + imap := mapping.NewIndexMapping() + imap.DefaultMapping = dmap + + idx, err := New(tmpIndexPath, imap) + if err != nil { + b.Fatal(err) + } + + defer func() { + err = idx.Close() + if err != nil { + b.Fatal(err) + } + }() + + members := [][]float64{ + {-121.96713072883645, 37.380331474621045}, + {-97.75518866579938, 30.38974491308761}, + {-0.08653451918110022, 51.51063984942306}, + {-2.230759791360498, 53.481514330841236}, + {77.59542326042589, 12.97215865921956}, + } + for i := 0; i < 100; i++ { + if err = idx.Index(strconv.Itoa(i), + map[string]interface{}{"geo": members[i%len(members)]}); err != nil { + b.Fatal(err) + } + } + + boundingBoxes := []struct { + topLeft []float64 + bottomRight []float64 + }{ + { + topLeft: []float64{-122.14424992609722, 37.49751487670511}, + bottomRight: []float64{-121.78076546622579, 37.26963069737202}, + }, + { + topLeft: []float64{-97.85362236226437, 30.473743975245725}, + bottomRight: []float64{-97.58691085968482, 30.285211697102895}, + }, + { + topLeft: []float64{-0.28538822102223094, 51.61106497119687}, + bottomRight: []float64{0.16776748108466677, 51.395702237541286}, + }, + { + topLeft: []float64{-2.373683904907921, 53.54371945714075}, + bottomRight: []float64{-2.134365533113197, 53.41788831720595}, + }, + { + topLeft: []float64{77.52617635172015, 13.037587208986437}, + bottomRight: []float64{77.66508989028102, 12.924426170584738}, + }, + } + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + topLeftCoordinates := boundingBoxes[i%len(boundingBoxes)].topLeft + bottomRightCoordinates := boundingBoxes[i%len(boundingBoxes)].bottomRight + q := NewGeoBoundingBoxQuery( + topLeftCoordinates[0], + topLeftCoordinates[1], + bottomRightCoordinates[0], + bottomRightCoordinates[1], + ) + q.SetField("geo") + req := NewSearchRequest(q) + if _, err = idx.Search(req); err != nil { + b.Fatal(err) + } + } +} diff --git a/registry/analyzer.go b/registry/analyzer.go new file mode 100644 index 0000000..af95b88 --- /dev/null +++ b/registry/analyzer.go @@ -0,0 +1,90 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func RegisterAnalyzer(name string, constructor AnalyzerConstructor) error { + _, exists := analyzers[name] + if exists { + return fmt.Errorf("attempted to register duplicate analyzer named '%s'", name) + } + analyzers[name] = constructor + return nil +} + +type AnalyzerConstructor func(config map[string]interface{}, cache *Cache) (analysis.Analyzer, error) +type AnalyzerRegistry map[string]AnalyzerConstructor + +type AnalyzerCache struct { + *ConcurrentCache +} + +func NewAnalyzerCache() *AnalyzerCache { + return &AnalyzerCache{ + NewConcurrentCache(), + } +} + +func AnalyzerBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := analyzers[name] + if !registered { + return nil, fmt.Errorf("no analyzer with name or type '%s' registered", name) + } + analyzer, err := cons(config, cache) + if err != nil { + return nil, fmt.Errorf("error building analyzer: %v", err) + } + return analyzer, nil +} + +func (c *AnalyzerCache) AnalyzerNamed(name string, cache *Cache) (analysis.Analyzer, error) { + item, err := c.ItemNamed(name, cache, AnalyzerBuild) + if err != nil { + return nil, err + } + return item.(analysis.Analyzer), nil +} + +func (c *AnalyzerCache) DefineAnalyzer(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.Analyzer, error) { + item, err := c.DefineItem(name, typ, config, cache, AnalyzerBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("analyzer named '%s' already defined", name) + } + return nil, err + } + return item.(analysis.Analyzer), nil +} + +func AnalyzerTypesAndInstances() ([]string, []string) { + emptyConfig := map[string]interface{}{} + emptyCache := NewCache() + var types []string + var instances []string + for name, cons := range analyzers { + _, err := cons(emptyConfig, emptyCache) + if err == nil { + instances = append(instances, name) + } else { + types = append(types, name) + } + } + return types, instances +} diff --git a/registry/cache.go b/registry/cache.go new file mode 100644 index 0000000..b0ce852 --- /dev/null +++ b/registry/cache.go @@ -0,0 +1,87 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "fmt" + "sync" +) + +var ErrAlreadyDefined = fmt.Errorf("item already defined") + +type CacheBuild func(name string, config map[string]interface{}, cache *Cache) (interface{}, error) + +type ConcurrentCache struct { + mutex sync.RWMutex + data map[string]interface{} +} + +func NewConcurrentCache() *ConcurrentCache { + return &ConcurrentCache{ + data: make(map[string]interface{}), + } +} + +func (c *ConcurrentCache) ItemNamed(name string, cache *Cache, build CacheBuild) (interface{}, error) { + c.mutex.RLock() + item, cached := c.data[name] + if cached { + c.mutex.RUnlock() + return item, nil + } + // give up read lock + c.mutex.RUnlock() + // try to build it + newItem, err := build(name, nil, cache) + if err != nil { + return nil, err + } + // acquire write lock + c.mutex.Lock() + defer c.mutex.Unlock() + // check again because it could have been created while trading locks + item, cached = c.data[name] + if cached { + return item, nil + } + c.data[name] = newItem + return newItem, nil +} + +func (c *ConcurrentCache) DefineItem(name string, typ string, config map[string]interface{}, cache *Cache, build CacheBuild) (interface{}, error) { + c.mutex.RLock() + _, cached := c.data[name] + if cached { + c.mutex.RUnlock() + return nil, ErrAlreadyDefined + } + // give up read lock so others lookups can proceed + c.mutex.RUnlock() + // really not there, try to build it + newItem, err := build(typ, config, cache) + if err != nil { + return nil, err + } + // now we've built it, acquire lock + c.mutex.Lock() + defer c.mutex.Unlock() + // check again because it could have been created while trading locks + _, cached = c.data[name] + if cached { + return nil, ErrAlreadyDefined + } + c.data[name] = newItem + return newItem, nil +} diff --git a/registry/char_filter.go b/registry/char_filter.go new file mode 100644 index 0000000..e888dac --- /dev/null +++ b/registry/char_filter.go @@ -0,0 +1,90 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func RegisterCharFilter(name string, constructor CharFilterConstructor) error { + _, exists := charFilters[name] + if exists { + return fmt.Errorf("attempted to register duplicate char filter named '%s'", name) + } + charFilters[name] = constructor + return nil +} + +type CharFilterConstructor func(config map[string]interface{}, cache *Cache) (analysis.CharFilter, error) +type CharFilterRegistry map[string]CharFilterConstructor + +type CharFilterCache struct { + *ConcurrentCache +} + +func NewCharFilterCache() *CharFilterCache { + return &CharFilterCache{ + NewConcurrentCache(), + } +} + +func CharFilterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := charFilters[name] + if !registered { + return nil, fmt.Errorf("no char filter with name or type '%s' registered", name) + } + charFilter, err := cons(config, cache) + if err != nil { + return nil, fmt.Errorf("error building char filter: %v", err) + } + return charFilter, nil +} + +func (c *CharFilterCache) CharFilterNamed(name string, cache *Cache) (analysis.CharFilter, error) { + item, err := c.ItemNamed(name, cache, CharFilterBuild) + if err != nil { + return nil, err + } + return item.(analysis.CharFilter), nil +} + +func (c *CharFilterCache) DefineCharFilter(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.CharFilter, error) { + item, err := c.DefineItem(name, typ, config, cache, CharFilterBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("char filter named '%s' already defined", name) + } + return nil, err + } + return item.(analysis.CharFilter), nil +} + +func CharFilterTypesAndInstances() ([]string, []string) { + emptyConfig := map[string]interface{}{} + emptyCache := NewCache() + var types []string + var instances []string + for name, cons := range charFilters { + _, err := cons(emptyConfig, emptyCache) + if err == nil { + instances = append(instances, name) + } else { + types = append(types, name) + } + } + return types, instances +} diff --git a/registry/datetime_parser.go b/registry/datetime_parser.go new file mode 100644 index 0000000..ff9a80c --- /dev/null +++ b/registry/datetime_parser.go @@ -0,0 +1,90 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func RegisterDateTimeParser(name string, constructor DateTimeParserConstructor) error { + _, exists := dateTimeParsers[name] + if exists { + return fmt.Errorf("attempted to register duplicate date time parser named '%s'", name) + } + dateTimeParsers[name] = constructor + return nil +} + +type DateTimeParserConstructor func(config map[string]interface{}, cache *Cache) (analysis.DateTimeParser, error) +type DateTimeParserRegistry map[string]DateTimeParserConstructor + +type DateTimeParserCache struct { + *ConcurrentCache +} + +func NewDateTimeParserCache() *DateTimeParserCache { + return &DateTimeParserCache{ + NewConcurrentCache(), + } +} + +func DateTimeParserBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := dateTimeParsers[name] + if !registered { + return nil, fmt.Errorf("no date time parser with name or type '%s' registered", name) + } + dateTimeParser, err := cons(config, cache) + if err != nil { + return nil, fmt.Errorf("error building date time parser: %v", err) + } + return dateTimeParser, nil +} + +func (c *DateTimeParserCache) DateTimeParserNamed(name string, cache *Cache) (analysis.DateTimeParser, error) { + item, err := c.ItemNamed(name, cache, DateTimeParserBuild) + if err != nil { + return nil, err + } + return item.(analysis.DateTimeParser), nil +} + +func (c *DateTimeParserCache) DefineDateTimeParser(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.DateTimeParser, error) { + item, err := c.DefineItem(name, typ, config, cache, DateTimeParserBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("date time parser named '%s' already defined", name) + } + return nil, err + } + return item.(analysis.DateTimeParser), nil +} + +func DateTimeParserTypesAndInstances() ([]string, []string) { + emptyConfig := map[string]interface{}{} + emptyCache := NewCache() + var types []string + var instances []string + for name, cons := range dateTimeParsers { + _, err := cons(emptyConfig, emptyCache) + if err == nil { + instances = append(instances, name) + } else { + types = append(types, name) + } + } + return types, instances +} diff --git a/registry/fragment_formatter.go b/registry/fragment_formatter.go new file mode 100644 index 0000000..f32c557 --- /dev/null +++ b/registry/fragment_formatter.go @@ -0,0 +1,90 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/search/highlight" +) + +func RegisterFragmentFormatter(name string, constructor FragmentFormatterConstructor) error { + _, exists := fragmentFormatters[name] + if exists { + return fmt.Errorf("attempted to register duplicate fragment formatter named '%s'", name) + } + fragmentFormatters[name] = constructor + return nil +} + +type FragmentFormatterConstructor func(config map[string]interface{}, cache *Cache) (highlight.FragmentFormatter, error) +type FragmentFormatterRegistry map[string]FragmentFormatterConstructor + +type FragmentFormatterCache struct { + *ConcurrentCache +} + +func NewFragmentFormatterCache() *FragmentFormatterCache { + return &FragmentFormatterCache{ + NewConcurrentCache(), + } +} + +func FragmentFormatterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := fragmentFormatters[name] + if !registered { + return nil, fmt.Errorf("no fragment formatter with name or type '%s' registered", name) + } + fragmentFormatter, err := cons(config, cache) + if err != nil { + return nil, fmt.Errorf("error building fragment formatter: %v", err) + } + return fragmentFormatter, nil +} + +func (c *FragmentFormatterCache) FragmentFormatterNamed(name string, cache *Cache) (highlight.FragmentFormatter, error) { + item, err := c.ItemNamed(name, cache, FragmentFormatterBuild) + if err != nil { + return nil, err + } + return item.(highlight.FragmentFormatter), nil +} + +func (c *FragmentFormatterCache) DefineFragmentFormatter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.FragmentFormatter, error) { + item, err := c.DefineItem(name, typ, config, cache, FragmentFormatterBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("fragment formatter named '%s' already defined", name) + } + return nil, err + } + return item.(highlight.FragmentFormatter), nil +} + +func FragmentFormatterTypesAndInstances() ([]string, []string) { + emptyConfig := map[string]interface{}{} + emptyCache := NewCache() + var types []string + var instances []string + for name, cons := range fragmentFormatters { + _, err := cons(emptyConfig, emptyCache) + if err == nil { + instances = append(instances, name) + } else { + types = append(types, name) + } + } + return types, instances +} diff --git a/registry/fragmenter.go b/registry/fragmenter.go new file mode 100644 index 0000000..da2a7b5 --- /dev/null +++ b/registry/fragmenter.go @@ -0,0 +1,90 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/search/highlight" +) + +func RegisterFragmenter(name string, constructor FragmenterConstructor) error { + _, exists := fragmenters[name] + if exists { + return fmt.Errorf("attempted to register duplicate fragmenter named '%s'", name) + } + fragmenters[name] = constructor + return nil +} + +type FragmenterConstructor func(config map[string]interface{}, cache *Cache) (highlight.Fragmenter, error) +type FragmenterRegistry map[string]FragmenterConstructor + +type FragmenterCache struct { + *ConcurrentCache +} + +func NewFragmenterCache() *FragmenterCache { + return &FragmenterCache{ + NewConcurrentCache(), + } +} + +func FragmenterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := fragmenters[name] + if !registered { + return nil, fmt.Errorf("no fragmenter with name or type '%s' registered", name) + } + fragmenter, err := cons(config, cache) + if err != nil { + return nil, fmt.Errorf("error building fragmenter: %v", err) + } + return fragmenter, nil +} + +func (c *FragmenterCache) FragmenterNamed(name string, cache *Cache) (highlight.Fragmenter, error) { + item, err := c.ItemNamed(name, cache, FragmenterBuild) + if err != nil { + return nil, err + } + return item.(highlight.Fragmenter), nil +} + +func (c *FragmenterCache) DefineFragmenter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.Fragmenter, error) { + item, err := c.DefineItem(name, typ, config, cache, FragmenterBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("fragmenter named '%s' already defined", name) + } + return nil, err + } + return item.(highlight.Fragmenter), nil +} + +func FragmenterTypesAndInstances() ([]string, []string) { + emptyConfig := map[string]interface{}{} + emptyCache := NewCache() + var types []string + var instances []string + for name, cons := range fragmenters { + _, err := cons(emptyConfig, emptyCache) + if err == nil { + instances = append(instances, name) + } else { + types = append(types, name) + } + } + return types, instances +} diff --git a/registry/highlighter.go b/registry/highlighter.go new file mode 100644 index 0000000..75de254 --- /dev/null +++ b/registry/highlighter.go @@ -0,0 +1,90 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/search/highlight" +) + +func RegisterHighlighter(name string, constructor HighlighterConstructor) error { + _, exists := highlighters[name] + if exists { + return fmt.Errorf("attempted to register duplicate highlighter named '%s'", name) + } + highlighters[name] = constructor + return nil +} + +type HighlighterConstructor func(config map[string]interface{}, cache *Cache) (highlight.Highlighter, error) +type HighlighterRegistry map[string]HighlighterConstructor + +type HighlighterCache struct { + *ConcurrentCache +} + +func NewHighlighterCache() *HighlighterCache { + return &HighlighterCache{ + NewConcurrentCache(), + } +} + +func HighlighterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := highlighters[name] + if !registered { + return nil, fmt.Errorf("no highlighter with name or type '%s' registered", name) + } + highlighter, err := cons(config, cache) + if err != nil { + return nil, fmt.Errorf("error building highlighter: %v", err) + } + return highlighter, nil +} + +func (c *HighlighterCache) HighlighterNamed(name string, cache *Cache) (highlight.Highlighter, error) { + item, err := c.ItemNamed(name, cache, HighlighterBuild) + if err != nil { + return nil, err + } + return item.(highlight.Highlighter), nil +} + +func (c *HighlighterCache) DefineHighlighter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.Highlighter, error) { + item, err := c.DefineItem(name, typ, config, cache, HighlighterBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("highlighter named '%s' already defined", name) + } + return nil, err + } + return item.(highlight.Highlighter), nil +} + +func HighlighterTypesAndInstances() ([]string, []string) { + emptyConfig := map[string]interface{}{} + emptyCache := NewCache() + var types []string + var instances []string + for name, cons := range highlighters { + _, err := cons(emptyConfig, emptyCache) + if err == nil { + instances = append(instances, name) + } else { + types = append(types, name) + } + } + return types, instances +} diff --git a/registry/index_type.go b/registry/index_type.go new file mode 100644 index 0000000..0c2c87f --- /dev/null +++ b/registry/index_type.go @@ -0,0 +1,46 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "fmt" + + index "github.com/blevesearch/bleve_index_api" +) + +func RegisterIndexType(name string, constructor IndexTypeConstructor) error { + _, exists := indexTypes[name] + if exists { + return fmt.Errorf("attempted to register duplicate index encoding named '%s'", name) + } + indexTypes[name] = constructor + return nil +} + +type IndexTypeConstructor func(storeName string, storeConfig map[string]interface{}, analysisQueue *index.AnalysisQueue) (index.Index, error) +type IndexTypeRegistry map[string]IndexTypeConstructor + +func IndexTypeConstructorByName(name string) IndexTypeConstructor { + return indexTypes[name] +} + +func IndexTypesAndInstances() ([]string, []string) { + var types []string + var instances []string + for name := range indexTypes { + types = append(types, name) + } + return types, instances +} diff --git a/registry/nested.go b/registry/nested.go new file mode 100644 index 0000000..fee7fda --- /dev/null +++ b/registry/nested.go @@ -0,0 +1,136 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "strings" + "sync" + + "github.com/blevesearch/bleve/v2/search" +) + +// NestedFieldCache caches nested field prefixes and their corresponding nesting levels. +// A nested field prefix is a field path prefix that indicates the start of a nested document. +// The nesting level indicates how deep the nested document is in the overall document structure. +type NestedFieldCache struct { + // nested prefix -> nested level + prefixDepth map[string]int + once sync.Once + m sync.RWMutex +} + +func NewNestedFieldCache() *NestedFieldCache { + return &NestedFieldCache{} +} + +func (nfc *NestedFieldCache) InitOnce(buildFunc func() map[string]int) { + nfc.once.Do(func() { + nfc.m.Lock() + defer nfc.m.Unlock() + nfc.prefixDepth = buildFunc() + }) +} + +// NestedDepth returns two values: +// - common: The nesting level of the longest prefix that applies to every field path +// in the provided FieldSet. A value of 0 means no nested prefix is shared +// across all field paths. +// - max: The nesting level of the longest prefix that applies to at least one +// field path in the provided FieldSet. A value of 0 means none of the +// field paths match any nested prefix. +func (nfc *NestedFieldCache) NestedDepth(fieldPaths search.FieldSet) (common int, max int) { + // if no field paths, no nested depth + if len(fieldPaths) == 0 { + return + } + nfc.m.RLock() + defer nfc.m.RUnlock() + // if no cached prefixes, no nested depth + if len(nfc.prefixDepth) == 0 { + return + } + // for each prefix, check if its a common prefix or matches any path + // update common and max accordingly with the highest nesting level + // possible for each respective case + for prefix, level := range nfc.prefixDepth { + // only check prefixes that could increase one of the results + if level <= common && level <= max { + continue + } + // check prefix against field paths, getting whether it matches all paths (common) + // and whether it matches at least one path (any) + matchAll, matchAny := nfc.prefixMatch(prefix, fieldPaths) + // if it matches all paths, update common + if matchAll && level > common { + common = level + } + // if it matches any path, update max + if matchAny && level > max { + max = level + } + } + return common, max +} + +// CountNested returns the number of nested prefixes +func (nfc *NestedFieldCache) CountNested() int { + nfc.m.RLock() + defer nfc.m.RUnlock() + + return len(nfc.prefixDepth) +} + +// IntersectsPrefix returns true if any of the given +// field paths have a nested prefix +func (nfc *NestedFieldCache) IntersectsPrefix(fieldPaths search.FieldSet) bool { + // if no field paths, no intersection + if len(fieldPaths) == 0 { + return false + } + nfc.m.RLock() + defer nfc.m.RUnlock() + // if no cached prefixes, no intersection + if len(nfc.prefixDepth) == 0 { + return false + } + // Check each cached nested prefix to see if it intersects with any path + for prefix := range nfc.prefixDepth { + _, matchAny := nfc.prefixMatch(prefix, fieldPaths) + if matchAny { + return true + } + } + return false +} + +// prefixMatch checks whether the prefix matches all paths (common) and whether it matches at least one path (any) +// Caller must hold the read lock. +func (nfc *NestedFieldCache) prefixMatch(prefix string, fieldPaths search.FieldSet) (common bool, any bool) { + common = true + any = false + for path := range fieldPaths { + has := strings.HasPrefix(path, prefix) + if has { + any = true + } else { + common = false + } + // early exit if we have determined both values + if any && !common { + break + } + } + return common, any +} diff --git a/registry/registry.go b/registry/registry.go new file mode 100644 index 0000000..36f209d --- /dev/null +++ b/registry/registry.go @@ -0,0 +1,197 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/search/highlight" +) + +var stores = make(KVStoreRegistry, 0) +var indexTypes = make(IndexTypeRegistry, 0) + +// highlight +var fragmentFormatters = make(FragmentFormatterRegistry, 0) +var fragmenters = make(FragmenterRegistry, 0) +var highlighters = make(HighlighterRegistry, 0) + +// analysis +var charFilters = make(CharFilterRegistry, 0) +var tokenizers = make(TokenizerRegistry, 0) +var tokenMaps = make(TokenMapRegistry, 0) +var tokenFilters = make(TokenFilterRegistry, 0) +var analyzers = make(AnalyzerRegistry, 0) +var dateTimeParsers = make(DateTimeParserRegistry, 0) +var synonymSources = make(SynonymSourceRegistry, 0) + +type Cache struct { + CharFilters *CharFilterCache + Tokenizers *TokenizerCache + TokenMaps *TokenMapCache + TokenFilters *TokenFilterCache + Analyzers *AnalyzerCache + DateTimeParsers *DateTimeParserCache + FragmentFormatters *FragmentFormatterCache + Fragmenters *FragmenterCache + Highlighters *HighlighterCache + SynonymSources *SynonymSourceCache + NestedPrefixes *NestedFieldCache +} + +func NewCache() *Cache { + return &Cache{ + CharFilters: NewCharFilterCache(), + Tokenizers: NewTokenizerCache(), + TokenMaps: NewTokenMapCache(), + TokenFilters: NewTokenFilterCache(), + Analyzers: NewAnalyzerCache(), + DateTimeParsers: NewDateTimeParserCache(), + FragmentFormatters: NewFragmentFormatterCache(), + Fragmenters: NewFragmenterCache(), + Highlighters: NewHighlighterCache(), + SynonymSources: NewSynonymSourceCache(), + NestedPrefixes: NewNestedFieldCache(), + } +} + +func typeFromConfig(config map[string]interface{}) (string, error) { + prop, ok := config["type"] + if !ok { + return "", fmt.Errorf("'type' property is not defined") + } + typ, ok := prop.(string) + if !ok { + return "", fmt.Errorf("'type' property must be a string, not %T", prop) + } + return typ, nil +} + +func (c *Cache) CharFilterNamed(name string) (analysis.CharFilter, error) { + return c.CharFilters.CharFilterNamed(name, c) +} + +func (c *Cache) DefineCharFilter(name string, config map[string]interface{}) (analysis.CharFilter, error) { + typ, err := typeFromConfig(config) + if err != nil { + return nil, err + } + return c.CharFilters.DefineCharFilter(name, typ, config, c) +} + +func (c *Cache) TokenizerNamed(name string) (analysis.Tokenizer, error) { + return c.Tokenizers.TokenizerNamed(name, c) +} + +func (c *Cache) DefineTokenizer(name string, config map[string]interface{}) (analysis.Tokenizer, error) { + typ, err := typeFromConfig(config) + if err != nil { + return nil, fmt.Errorf("cannot resolve '%s' tokenizer type: %s", name, err) + } + return c.Tokenizers.DefineTokenizer(name, typ, config, c) +} + +func (c *Cache) TokenMapNamed(name string) (analysis.TokenMap, error) { + return c.TokenMaps.TokenMapNamed(name, c) +} + +func (c *Cache) DefineTokenMap(name string, config map[string]interface{}) (analysis.TokenMap, error) { + typ, err := typeFromConfig(config) + if err != nil { + return nil, err + } + return c.TokenMaps.DefineTokenMap(name, typ, config, c) +} + +func (c *Cache) TokenFilterNamed(name string) (analysis.TokenFilter, error) { + return c.TokenFilters.TokenFilterNamed(name, c) +} + +func (c *Cache) DefineTokenFilter(name string, config map[string]interface{}) (analysis.TokenFilter, error) { + typ, err := typeFromConfig(config) + if err != nil { + return nil, err + } + return c.TokenFilters.DefineTokenFilter(name, typ, config, c) +} + +func (c *Cache) AnalyzerNamed(name string) (analysis.Analyzer, error) { + return c.Analyzers.AnalyzerNamed(name, c) +} + +func (c *Cache) DefineAnalyzer(name string, config map[string]interface{}) (analysis.Analyzer, error) { + typ, err := typeFromConfig(config) + if err != nil { + return nil, err + } + return c.Analyzers.DefineAnalyzer(name, typ, config, c) +} + +func (c *Cache) DateTimeParserNamed(name string) (analysis.DateTimeParser, error) { + return c.DateTimeParsers.DateTimeParserNamed(name, c) +} + +func (c *Cache) DefineDateTimeParser(name string, config map[string]interface{}) (analysis.DateTimeParser, error) { + typ, err := typeFromConfig(config) + if err != nil { + return nil, err + } + return c.DateTimeParsers.DefineDateTimeParser(name, typ, config, c) +} + +func (c *Cache) SynonymSourceNamed(name string) (analysis.SynonymSource, error) { + return c.SynonymSources.SynonymSourceNamed(name, c) +} + +func (c *Cache) DefineSynonymSource(name string, config map[string]interface{}) (analysis.SynonymSource, error) { + return c.SynonymSources.DefineSynonymSource(name, analysis.SynonymSourceType, config, c) +} + +func (c *Cache) FragmentFormatterNamed(name string) (highlight.FragmentFormatter, error) { + return c.FragmentFormatters.FragmentFormatterNamed(name, c) +} + +func (c *Cache) DefineFragmentFormatter(name string, config map[string]interface{}) (highlight.FragmentFormatter, error) { + typ, err := typeFromConfig(config) + if err != nil { + return nil, err + } + return c.FragmentFormatters.DefineFragmentFormatter(name, typ, config, c) +} + +func (c *Cache) FragmenterNamed(name string) (highlight.Fragmenter, error) { + return c.Fragmenters.FragmenterNamed(name, c) +} + +func (c *Cache) DefineFragmenter(name string, config map[string]interface{}) (highlight.Fragmenter, error) { + typ, err := typeFromConfig(config) + if err != nil { + return nil, err + } + return c.Fragmenters.DefineFragmenter(name, typ, config, c) +} + +func (c *Cache) HighlighterNamed(name string) (highlight.Highlighter, error) { + return c.Highlighters.HighlighterNamed(name, c) +} + +func (c *Cache) DefineHighlighter(name string, config map[string]interface{}) (highlight.Highlighter, error) { + typ, err := typeFromConfig(config) + if err != nil { + return nil, err + } + return c.Highlighters.DefineHighlighter(name, typ, config, c) +} diff --git a/registry/store.go b/registry/store.go new file mode 100644 index 0000000..e3a9828 --- /dev/null +++ b/registry/store.go @@ -0,0 +1,52 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "fmt" + + store "github.com/blevesearch/upsidedown_store_api" +) + +func RegisterKVStore(name string, constructor KVStoreConstructor) error { + _, exists := stores[name] + if exists { + return fmt.Errorf("attempted to register duplicate store named '%s'", name) + } + stores[name] = constructor + return nil +} + +// KVStoreConstructor is used to build a KVStore of a specific type when +// specified by the index configuration. In addition to meeting the +// store.KVStore interface, KVStores must also support this constructor. +// Note that currently the values of config must +// be able to be marshaled and unmarshaled using the encoding/json library (used +// when reading/writing the index metadata file). +type KVStoreConstructor func(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) +type KVStoreRegistry map[string]KVStoreConstructor + +func KVStoreConstructorByName(name string) KVStoreConstructor { + return stores[name] +} + +func KVStoreTypesAndInstances() ([]string, []string) { + var types []string + var instances []string + for name := range stores { + types = append(types, name) + } + return types, instances +} diff --git a/registry/synonym_source.go b/registry/synonym_source.go new file mode 100644 index 0000000..f1836f8 --- /dev/null +++ b/registry/synonym_source.go @@ -0,0 +1,86 @@ +// Copyright (c) 2024 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func RegisterSynonymSource(typ string, constructor SynonymSourceConstructor) error { + _, exists := synonymSources[typ] + if exists { + return fmt.Errorf("attempted to register duplicate synonym source with type '%s'", typ) + } + synonymSources[typ] = constructor + return nil +} + +type SynonymSourceCache struct { + *ConcurrentCache +} + +func NewSynonymSourceCache() *SynonymSourceCache { + return &SynonymSourceCache{ + NewConcurrentCache(), + } +} + +type SynonymSourceConstructor func(config map[string]interface{}, cache *Cache) (analysis.SynonymSource, error) +type SynonymSourceRegistry map[string]SynonymSourceConstructor + +func SynonymSourceBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := synonymSources[name] + if !registered { + return nil, fmt.Errorf("no synonym source with name '%s' registered", name) + } + synonymSource, err := cons(config, cache) + if err != nil { + return nil, fmt.Errorf("error building synonym source: %v", err) + } + return synonymSource, nil +} + +func (c *SynonymSourceCache) SynonymSourceNamed(name string, cache *Cache) (analysis.SynonymSource, error) { + item, err := c.ItemNamed(name, cache, SynonymSourceBuild) + if err != nil { + return nil, err + } + return item.(analysis.SynonymSource), nil +} + +func (c *SynonymSourceCache) DefineSynonymSource(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.SynonymSource, error) { + item, err := c.DefineItem(name, typ, config, cache, SynonymSourceBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("synonym source named '%s' already defined", name) + } + return nil, err + } + return item.(analysis.SynonymSource), nil +} + +func (c *SynonymSourceCache) VisitSynonymSources(visitor analysis.SynonymSourceVisitor) error { + c.mutex.RLock() + defer c.mutex.RUnlock() + for k, v := range c.data { + err := visitor(k, v.(analysis.SynonymSource)) + if err != nil { + return err + } + } + return nil +} diff --git a/registry/token_filter.go b/registry/token_filter.go new file mode 100644 index 0000000..533a103 --- /dev/null +++ b/registry/token_filter.go @@ -0,0 +1,90 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func RegisterTokenFilter(name string, constructor TokenFilterConstructor) error { + _, exists := tokenFilters[name] + if exists { + return fmt.Errorf("attempted to register duplicate token filter named '%s'", name) + } + tokenFilters[name] = constructor + return nil +} + +type TokenFilterConstructor func(config map[string]interface{}, cache *Cache) (analysis.TokenFilter, error) +type TokenFilterRegistry map[string]TokenFilterConstructor + +type TokenFilterCache struct { + *ConcurrentCache +} + +func NewTokenFilterCache() *TokenFilterCache { + return &TokenFilterCache{ + NewConcurrentCache(), + } +} + +func TokenFilterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := tokenFilters[name] + if !registered { + return nil, fmt.Errorf("no token filter with name or type '%s' registered", name) + } + tokenFilter, err := cons(config, cache) + if err != nil { + return nil, fmt.Errorf("error building token filter: %v", err) + } + return tokenFilter, nil +} + +func (c *TokenFilterCache) TokenFilterNamed(name string, cache *Cache) (analysis.TokenFilter, error) { + item, err := c.ItemNamed(name, cache, TokenFilterBuild) + if err != nil { + return nil, err + } + return item.(analysis.TokenFilter), nil +} + +func (c *TokenFilterCache) DefineTokenFilter(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.TokenFilter, error) { + item, err := c.DefineItem(name, typ, config, cache, TokenFilterBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("token filter named '%s' already defined", name) + } + return nil, err + } + return item.(analysis.TokenFilter), nil +} + +func TokenFilterTypesAndInstances() ([]string, []string) { + emptyConfig := map[string]interface{}{} + emptyCache := NewCache() + var types []string + var instances []string + for name, cons := range tokenFilters { + _, err := cons(emptyConfig, emptyCache) + if err == nil { + instances = append(instances, name) + } else { + types = append(types, name) + } + } + return types, instances +} diff --git a/registry/token_maps.go b/registry/token_maps.go new file mode 100644 index 0000000..7fd7886 --- /dev/null +++ b/registry/token_maps.go @@ -0,0 +1,90 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func RegisterTokenMap(name string, constructor TokenMapConstructor) error { + _, exists := tokenMaps[name] + if exists { + return fmt.Errorf("attempted to register duplicate token map named '%s'", name) + } + tokenMaps[name] = constructor + return nil +} + +type TokenMapConstructor func(config map[string]interface{}, cache *Cache) (analysis.TokenMap, error) +type TokenMapRegistry map[string]TokenMapConstructor + +type TokenMapCache struct { + *ConcurrentCache +} + +func NewTokenMapCache() *TokenMapCache { + return &TokenMapCache{ + NewConcurrentCache(), + } +} + +func TokenMapBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := tokenMaps[name] + if !registered { + return nil, fmt.Errorf("no token map with name or type '%s' registered", name) + } + tokenMap, err := cons(config, cache) + if err != nil { + return nil, fmt.Errorf("error building token map: %v", err) + } + return tokenMap, nil +} + +func (c *TokenMapCache) TokenMapNamed(name string, cache *Cache) (analysis.TokenMap, error) { + item, err := c.ItemNamed(name, cache, TokenMapBuild) + if err != nil { + return nil, err + } + return item.(analysis.TokenMap), nil +} + +func (c *TokenMapCache) DefineTokenMap(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.TokenMap, error) { + item, err := c.DefineItem(name, typ, config, cache, TokenMapBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("token map named '%s' already defined", name) + } + return nil, err + } + return item.(analysis.TokenMap), nil +} + +func TokenMapTypesAndInstances() ([]string, []string) { + emptyConfig := map[string]interface{}{} + emptyCache := NewCache() + var types []string + var instances []string + for name, cons := range tokenMaps { + _, err := cons(emptyConfig, emptyCache) + if err == nil { + instances = append(instances, name) + } else { + types = append(types, name) + } + } + return types, instances +} diff --git a/registry/tokenizer.go b/registry/tokenizer.go new file mode 100644 index 0000000..81222b8 --- /dev/null +++ b/registry/tokenizer.go @@ -0,0 +1,90 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func RegisterTokenizer(name string, constructor TokenizerConstructor) error { + _, exists := tokenizers[name] + if exists { + return fmt.Errorf("attempted to register duplicate tokenizer named '%s'", name) + } + tokenizers[name] = constructor + return nil +} + +type TokenizerConstructor func(config map[string]interface{}, cache *Cache) (analysis.Tokenizer, error) +type TokenizerRegistry map[string]TokenizerConstructor + +type TokenizerCache struct { + *ConcurrentCache +} + +func NewTokenizerCache() *TokenizerCache { + return &TokenizerCache{ + NewConcurrentCache(), + } +} + +func TokenizerBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := tokenizers[name] + if !registered { + return nil, fmt.Errorf("no tokenizer with name or type '%s' registered", name) + } + tokenizer, err := cons(config, cache) + if err != nil { + return nil, fmt.Errorf("error building tokenizer: %v", err) + } + return tokenizer, nil +} + +func (c *TokenizerCache) TokenizerNamed(name string, cache *Cache) (analysis.Tokenizer, error) { + item, err := c.ItemNamed(name, cache, TokenizerBuild) + if err != nil { + return nil, err + } + return item.(analysis.Tokenizer), nil +} + +func (c *TokenizerCache) DefineTokenizer(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.Tokenizer, error) { + item, err := c.DefineItem(name, typ, config, cache, TokenizerBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("tokenizer named '%s' already defined", name) + } + return nil, err + } + return item.(analysis.Tokenizer), nil +} + +func TokenizerTypesAndInstances() ([]string, []string) { + emptyConfig := map[string]interface{}{} + emptyCache := NewCache() + var types []string + var instances []string + for name, cons := range tokenizers { + _, err := cons(emptyConfig, emptyCache) + if err == nil { + instances = append(instances, name) + } else { + types = append(types, name) + } + } + return types, instances +} diff --git a/rescorer.go b/rescorer.go new file mode 100644 index 0000000..be3e798 --- /dev/null +++ b/rescorer.go @@ -0,0 +1,160 @@ +// Copyright (c) 2025 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "github.com/blevesearch/bleve/v2/fusion" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/query" +) + +const ( + DefaultScoreRankConstant = 60 +) + +// Rescorer is applied after all the query and knn results are obtained. +// The main use of Rescorer is in hybrid search; all the individual scores +// for query and knn are combined using Rescorer. Makes use of algorithms +// defined in `fusion` +type rescorer struct { + req *SearchRequest + + // Stores the original From, Size and Boost parameters from the request + origFrom int + origSize int + origBoosts []float64 + + // Flag variable to make sure that restoreSearchRequest is only run once + // when it is deferred + restored bool +} + +// Stores information about the hybrid search into FusionRescorer. +// Also mutates the SearchRequest by: +// - Setting boosts to 1: top level boosts only used for rescoring +// - Setting From and Size to 0 and ScoreWindowSize +func (r *rescorer) prepareSearchRequest() error { + if r.req.Params == nil { + r.req.Params = NewDefaultParams(r.req.From, r.req.Size) + } + + r.origFrom = r.req.From + r.origSize = r.req.Size + + r.req.From = 0 + r.req.Size = r.req.Params.ScoreWindowSize + + // req.Query's top level boost comes first, followed by the KNN queries + numQueries := numKNNQueries(r.req) + 1 + r.origBoosts = make([]float64, numQueries) + + // only modify queries if it is boostable. If not, ignore + if bQuery, ok := r.req.Query.(query.BoostableQuery); ok { + r.origBoosts[0] = bQuery.Boost() + bQuery.SetBoost(1.0) + } else { + r.origBoosts[0] = 1.0 + } + + // for all the knn queries, replace boost values + r.prepareKnnRequest() + + return nil +} + +func (r *rescorer) restoreSearchRequest() { + // Skip if already restored + if r.restored { + return + } + r.restored = true + + r.req.From = r.origFrom + r.req.Size = r.origSize + + if bQuery, ok := r.req.Query.(query.BoostableQuery); ok { + bQuery.SetBoost(r.origBoosts[0]) + } + + // for all the knn queries, restore boost values + r.restoreKnnRequest() +} + +func (r *rescorer) rescore(ftsHits, knnHits search.DocumentMatchCollection) (search.DocumentMatchCollection, uint64, float64) { + mergedHits := r.mergeDocs(ftsHits, knnHits) + + var fusionResult *fusion.FusionResult + + switch r.req.Score { + case ScoreRRF: + fusionResult = fusion.ReciprocalRankFusion( + mergedHits, + r.origBoosts, + r.req.Params.ScoreRankConstant, + r.req.Params.ScoreWindowSize, + numKNNQueries(r.req), + r.req.Explain, + ) + case ScoreRSF: + fusionResult = fusion.RelativeScoreFusion( + mergedHits, + r.origBoosts, + r.req.Params.ScoreWindowSize, + numKNNQueries(r.req), + r.req.Explain, + ) + } + + return fusionResult.Hits, fusionResult.Total, fusionResult.MaxScore +} + +// Merge all the FTS and KNN docs along with explanations +func (r *rescorer) mergeDocs(ftsHits, knnHits search.DocumentMatchCollection) search.DocumentMatchCollection { + if len(knnHits) == 0 { + return ftsHits + } + + knnHitMap := make(map[string]*search.DocumentMatch, len(knnHits)) + + for _, hit := range knnHits { + knnHitMap[hit.ID] = hit + } + + for _, hit := range ftsHits { + if knnHit, ok := knnHitMap[hit.ID]; ok { + hit.ScoreBreakdown = knnHit.ScoreBreakdown + if r.req.Explain { + hit.Expl = &search.Explanation{Value: 0.0, Message: "", Children: append([]*search.Explanation{hit.Expl}, knnHit.Expl.Children...)} + } + delete(knnHitMap, hit.ID) + } + } + + for _, hit := range knnHitMap { + hit.Score = 0 + ftsHits = append(ftsHits, hit) + if r.req.Explain { + hit.Expl = &search.Explanation{Value: 0.0, Message: "", Children: append([]*search.Explanation{nil}, hit.Expl.Children...)} + } + } + + return ftsHits +} + +func newRescorer(req *SearchRequest) *rescorer { + return &rescorer{ + req: req, + } +} diff --git a/rescorer_knn_test.go b/rescorer_knn_test.go new file mode 100644 index 0000000..f84ee9d --- /dev/null +++ b/rescorer_knn_test.go @@ -0,0 +1,1123 @@ +// Copyright (c) 2025 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package bleve + +import ( + "context" + "math" + "testing" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/query" + index "github.com/blevesearch/bleve_index_api" +) + +func createHybridSearchIndex(path string) (Index, error) { + // Index mapping + indexMapping := NewIndexMapping() + + // Disable default mapping to match expected configuration + indexMapping.DefaultMapping.Enabled = false + indexMapping.DefaultMapping.Dynamic = false + + // Create a specific document mapping type + docMapping := NewDocumentMapping() + docMapping.Enabled = true + docMapping.Dynamic = false + + // Text field for color with specific properties + colorFieldMapping := NewTextFieldMapping() + colorFieldMapping.Analyzer = "en" // Use "en" analyzer as specified + colorFieldMapping.DocValues = true + colorFieldMapping.IncludeInAll = true + colorFieldMapping.Store = true + colorFieldMapping.Index = true + docMapping.AddFieldMappingsAt("color", colorFieldMapping) + + // Vector field for color vector with L2 similarity + vecFieldMapping := mapping.NewVectorFieldMapping() + vecFieldMapping.Dims = 3 + vecFieldMapping.Similarity = index.EuclideanDistance // l2_norm equivalent + vecFieldMapping.VectorIndexOptimizedFor = "recall" + docMapping.AddFieldMappingsAt("colorvect_l2", vecFieldMapping) + + // Add the document mapping to the index + indexMapping.AddDocumentMapping("_default", docMapping) + + // Create index + return New(path, indexMapping) +} + +func getHybridSearchDocuments() []map[string]interface{} { + documents := []map[string]interface{}{ + { + "color": "dark slate blue", + "colorvect_l2": []float32{72, 61, 139}, + }, + { + "color": "blue", + "colorvect_l2": []float32{0, 0, 255}, + }, + { + "color": "navy", + "colorvect_l2": []float32{0, 0, 128}, + }, + { + "color": "steel blue", + "colorvect_l2": []float32{70, 130, 180}, + }, + { + "color": "light blue", + "colorvect_l2": []float32{173, 216, 230}, + }, + { + "color": "deep sky blue", + "colorvect_l2": []float32{0, 191, 255}, + }, + { + "color": "royal blue", + "colorvect_l2": []float32{65, 105, 225}, + }, + { + "color": "powder blue", + "colorvect_l2": []float32{176, 224, 230}, + }, + { + "color": "corn flower blue", + "colorvect_l2": []float32{100, 149, 237}, + }, + { + "color": "alice blue", + "colorvect_l2": []float32{240, 248, 255}, + }, + { + "color": "blue violet", + "colorvect_l2": []float32{138, 43, 226}, + }, + { + "color": "sky blue", + "colorvect_l2": []float32{135, 206, 235}, + }, + { + "color": "indigo", + "colorvect_l2": []float32{75, 0, 130}, + }, + { + "color": "midnight blue", + "colorvect_l2": []float32{25, 25, 112}, + }, + { + "color": "dark blue", + "colorvect_l2": []float32{0, 0, 139}, + }, + { + "color": "medium slate blue", + "colorvect_l2": []float32{123, 104, 238}, + }, + { + "color": "cadet blue", + "colorvect_l2": []float32{95, 158, 160}, + }, + { + "color": "light steel blue", + "colorvect_l2": []float32{176, 196, 222}, + }, + { + "color": "dodger blue", + "colorvect_l2": []float32{30, 144, 255}, + }, + { + "color": "medium blue", + "colorvect_l2": []float32{0, 0, 205}, + }, + { + "color": "slate blue", + "colorvect_l2": []float32{106, 90, 205}, + }, + { + "color": "light sky blue", + "colorvect_l2": []float32{135, 206, 250}, + }, + } + + return documents +} + +func createScoreFusionRequest(scoreMethod string, knn bool) *SearchRequest { + // Create hybrid search request (FTS + KNN) + textQuery := query.NewMatchPhraseQuery("dark") + searchRequest := NewSearchRequest(textQuery) + + if knn { + queryVector_1 := []float32{0, 0, 129} // Similar to blue colors + searchRequest.AddKNN("colorvect_l2", queryVector_1, 5, 1.0) + + queryVector_2 := []float32{0, 0, 250} // lighter blue + searchRequest.AddKNN("colorvect_l2", queryVector_2, 5, 1.0) + } + + params := RequestParams{1, 10} + searchRequest.AddParams(params) + + searchRequest.Size = 10 + + searchRequest.Score = scoreMethod + searchRequest.Explain = false + return searchRequest +} + +// verifyRRFResults verifies that the search hits match the expected RRF ranking and scores +func verifyRRFResults(t *testing.T, hits search.DocumentMatchCollection) { + // Manual RRF calculation for verification + // With k=1 (ScoreRankConstant), RRF formula: 1/(1+rank) + // + // FTS "dark" ranks: + // 1. dark blue, 2. dark slate blue + // + // kNN1 [0,0,129] ranks: + // 1. navy, 2. dark blue, 3. midnight blue, 4. indigo, 5. medium blue + // + // kNN2 [0,0,250] ranks: + // 1. blue, 2. medium blue, 3. dark blue, 4. navy, 5. royal blue + + expectedRRFScores := map[string]float64{ + "dark blue": 1.083333, // FTS(1): 1/2 + kNN1(2): 1/3 + kNN2(3): 1/4 = 1.083333 + "navy": 0.7, // kNN1(1): 1/2 + kNN2(4): 1/5 = 0.7 + "blue": 0.5, // kNN2(1): 1/2 = 0.5 + "medium blue": 0.5, // kNN1(5): 1/6 + kNN2(2): 1/3 = 0.5 + "dark slate blue": 0.333333, // FTS(2): 1/3 = 0.333333 + "midnight blue": 0.25, // kNN1(3): 1/4 = 0.25 + "indigo": 0.2, // kNN1(4): 1/5 = 0.2 + "royal blue": 0.166667, // kNN2(5): 1/6 = 0.166667 + } + + // Verify top results match expected RRF ranking + expectedOrder := []string{"dark blue", "navy", "blue", "medium blue", "dark slate blue", "midnight blue", "indigo", "royal blue"} + + if len(hits) < len(expectedOrder) { + t.Fatalf("Expected at least %d results, got %d", len(expectedOrder), len(hits)) + } + + for i, expectedID := range expectedOrder { + if hits[i].ID != expectedID { + id := hits[i].ID + if !(id == "blue" || id == "medium blue") { // Don't throw an error, since these scores are the same + t.Errorf("Position %d: expected %s, got %s", i+1, expectedID, hits[i].ID) + } + } + + expectedScore := expectedRRFScores[expectedID] + actualScore := hits[i].Score + tolerance := 0.001 + + if math.Abs(actualScore-expectedScore) > tolerance { + t.Errorf("Score for %s: expected %.6f, got %.6f (diff: %.6f)", + expectedID, expectedScore, actualScore, math.Abs(actualScore-expectedScore)) + } + } +} + +// setupSingleIndex creates a single index with all documents +func setupSingleIndex(t *testing.T) (Index, func()) { + tmpIndexPath := createTmpIndexPath(t) + + index, err := createHybridSearchIndex(tmpIndexPath) + if err != nil { + t.Fatal(err) + } + + documents := getHybridSearchDocuments() + + // Index documents + batch := index.NewBatch() + for _, doc := range documents { + colorName := doc["color"].(string) + err = batch.Index(colorName, doc) + if err != nil { + t.Fatal(err) + } + } + err = index.Batch(batch) + if err != nil { + t.Fatal(err) + } + + cleanup := func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + cleanupTmpIndexPath(t, tmpIndexPath) + } + + return index, cleanup +} + +// setupAliasWithSingleIndex creates an alias containing one index with all documents +func setupAliasWithSingleIndex(t *testing.T) (Index, func()) { + tmpIndexPath := createTmpIndexPath(t) + + index, err := createHybridSearchIndex(tmpIndexPath) + if err != nil { + t.Fatal(err) + } + + documents := getHybridSearchDocuments() + + // Create alias and add the single index + alias := NewIndexAlias() + alias.Add(index) + + // Index all documents + batch := alias.NewBatch() + for _, doc := range documents { + colorName := doc["color"].(string) + err = batch.Index(colorName, doc) + if err != nil { + t.Fatal(err) + } + } + err = alias.Batch(batch) + if err != nil { + t.Fatal(err) + } + + cleanup := func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + cleanupTmpIndexPath(t, tmpIndexPath) + } + + return alias, cleanup +} + +// setupAliasWithTwoIndexes creates an alias containing two indexes with documents split between them +func setupAliasWithTwoIndexes(t *testing.T) (Index, func()) { + documents := getHybridSearchDocuments() + + // Split documents into two groups + midpoint := len(documents) / 2 + docs1 := documents[:midpoint] + docs2 := documents[midpoint:] + + // Create first index + tmpIndexPath1 := createTmpIndexPath(t) + index1, err := createHybridSearchIndex(tmpIndexPath1) + if err != nil { + t.Fatal(err) + } + + // Index first half of documents + batch1 := index1.NewBatch() + for _, doc := range docs1 { + colorName := doc["color"].(string) + err = batch1.Index(colorName, doc) + if err != nil { + t.Fatal(err) + } + } + err = index1.Batch(batch1) + if err != nil { + t.Fatal(err) + } + + // Create second index + tmpIndexPath2 := createTmpIndexPath(t) + index2, err := createHybridSearchIndex(tmpIndexPath2) + if err != nil { + t.Fatal(err) + } + + // Index second half of documents + batch2 := index2.NewBatch() + for _, doc := range docs2 { + colorName := doc["color"].(string) + err = batch2.Index(colorName, doc) + if err != nil { + t.Fatal(err) + } + } + err = index2.Batch(batch2) + if err != nil { + t.Fatal(err) + } + + // Create alias and add both indexes + alias := NewIndexAlias() + alias.Add(index1, index2) + + cleanup := func() { + err := index1.Close() + if err != nil { + t.Fatal(err) + } + err = index2.Close() + if err != nil { + t.Fatal(err) + } + cleanupTmpIndexPath(t, tmpIndexPath1) + cleanupTmpIndexPath(t, tmpIndexPath2) + } + + return alias, cleanup +} + +// setupNestedAliases creates nested aliases with three indexes spread across sub-aliases +func setupNestedAliases(t *testing.T) (Index, func()) { + documents := getHybridSearchDocuments() + + // Split documents into three groups + thirdPoint1 := len(documents) / 3 + thirdPoint2 := 2 * len(documents) / 3 + docs1 := documents[:thirdPoint1] + docs2 := documents[thirdPoint1:thirdPoint2] + docs3 := documents[thirdPoint2:] + + // Create first index + tmpIndexPath1 := createTmpIndexPath(t) + index1, err := createHybridSearchIndex(tmpIndexPath1) + if err != nil { + t.Fatal(err) + } + + // Index first third of documents + batch1 := index1.NewBatch() + for _, doc := range docs1 { + colorName := doc["color"].(string) + err = batch1.Index(colorName, doc) + if err != nil { + t.Fatal(err) + } + } + err = index1.Batch(batch1) + if err != nil { + t.Fatal(err) + } + + // Create second index + tmpIndexPath2 := createTmpIndexPath(t) + index2, err := createHybridSearchIndex(tmpIndexPath2) + if err != nil { + t.Fatal(err) + } + + // Index second third of documents + batch2 := index2.NewBatch() + for _, doc := range docs2 { + colorName := doc["color"].(string) + err = batch2.Index(colorName, doc) + if err != nil { + t.Fatal(err) + } + } + err = index2.Batch(batch2) + if err != nil { + t.Fatal(err) + } + + // Create third index + tmpIndexPath3 := createTmpIndexPath(t) + index3, err := createHybridSearchIndex(tmpIndexPath3) + if err != nil { + t.Fatal(err) + } + + // Index third third of documents + batch3 := index3.NewBatch() + for _, doc := range docs3 { + colorName := doc["color"].(string) + err = batch3.Index(colorName, doc) + if err != nil { + t.Fatal(err) + } + } + err = index3.Batch(batch3) + if err != nil { + t.Fatal(err) + } + + // Create first sub-alias (contains 1 index) + subAlias1 := NewIndexAlias() + subAlias1.SetName("subAlias1") + subAlias1.Add(index1) + + // Create second sub-alias (contains 2 indexes) + subAlias2 := NewIndexAlias() + subAlias2.SetName("subAlias2") + subAlias2.Add(index2, index3) + + // Create master alias containing the two sub-aliases + masterAlias := NewIndexAlias() + masterAlias.SetName("masterAlias") + masterAlias.Add(subAlias1, subAlias2) + + cleanup := func() { + err := index1.Close() + if err != nil { + t.Fatal(err) + } + err = index2.Close() + if err != nil { + t.Fatal(err) + } + err = index3.Close() + if err != nil { + t.Fatal(err) + } + cleanupTmpIndexPath(t, tmpIndexPath1) + cleanupTmpIndexPath(t, tmpIndexPath2) + cleanupTmpIndexPath(t, tmpIndexPath3) + } + + return masterAlias, cleanup +} + +func TestRRFEndToEnd(t *testing.T) { + // Setup the index configuration + index, cleanup := setupSingleIndex(t) + defer cleanup() + + // Create the search request + searchRequest := createScoreFusionRequest(ScoreRRF, true) + + // Execute search + result, err := index.Search(searchRequest) + if err != nil { + t.Fatal(err) + } + + // Verify RRF results + verifyRRFResults(t, result.Hits) +} + +// TestRRFAliasWithSingleIndex tests RRF with an alias containing one index +func TestRRFAliasWithSingleIndex(t *testing.T) { + // Setup the alias configuration + alias, cleanup := setupAliasWithSingleIndex(t) + defer cleanup() + + // Create the search request + searchRequest := createScoreFusionRequest(ScoreRRF, true) + + // Execute search through alias + result, err := alias.Search(searchRequest) + if err != nil { + t.Fatal(err) + } + + // Verify RRF results - should be identical to direct index search + verifyRRFResults(t, result.Hits) +} + +// TestRRFAliasWithTwoIndexes tests RRF with an alias containing two indexes +func TestRRFAliasWithTwoIndexes(t *testing.T) { + // Setup the alias configuration + alias, cleanup := setupAliasWithTwoIndexes(t) + defer cleanup() + + // Create the search request + searchRequest := createScoreFusionRequest(ScoreRRF, true) + + // Execute search through alias + result, err := alias.Search(searchRequest) + if err != nil { + t.Fatal(err) + } + + // Verify RRF results - should be identical to single index results + verifyRRFResults(t, result.Hits) +} + +// TestRRFNestedAliases tests RRF with an alias containing two index aliases +func TestRRFNestedAliases(t *testing.T) { + // Setup the nested aliases configuration + masterAlias, cleanup := setupNestedAliases(t) + defer cleanup() + + // Create the search request + searchRequest := createScoreFusionRequest(ScoreRRF, true) + + // Execute search through master alias + result, err := masterAlias.Search(searchRequest) + if err != nil { + t.Fatal(err) + } + + // Verify RRF results - should be identical to single index results + verifyRRFResults(t, result.Hits) +} + +// TestRRFPagination tests RRF with pagination across different index/alias configurations +func TestRRFPagination(t *testing.T) { + scenarios := []struct { + name string + setup func(t *testing.T) (Index, func()) + }{ + { + name: "SingleIndex", + setup: setupSingleIndex, + }, + { + name: "AliasWithSingleIndex", + setup: setupAliasWithSingleIndex, + }, + { + name: "AliasWithTwoIndexes", + setup: setupAliasWithTwoIndexes, + }, + { + name: "NestedAliases", + setup: setupNestedAliases, + }, + } + + for _, scenario := range scenarios { + t.Run(scenario.name, func(t *testing.T) { + // Setup the index/alias configuration + index, cleanup := scenario.setup(t) + defer cleanup() + + // Create first page request (first 5 results) + firstPageRequest := createScoreFusionRequest(ScoreRRF, true) + firstPageRequest.From = 0 + firstPageRequest.Size = 5 + + // Execute first page search + firstPageResult, err := index.Search(firstPageRequest) + if err != nil { + t.Fatal(err) + } + + // Create second page request (next 5 results, starting from index 5) + secondPageRequest := createScoreFusionRequest(ScoreRRF, true) + secondPageRequest.From = 5 + secondPageRequest.Size = 5 + + // Execute second page search + secondPageResult, err := index.Search(secondPageRequest) + if err != nil { + t.Fatal(err) + } + + // Combine results from both pages + combinedHits := make(search.DocumentMatchCollection, 0, len(firstPageResult.Hits)+len(secondPageResult.Hits)) + combinedHits = append(combinedHits, firstPageResult.Hits...) + combinedHits = append(combinedHits, secondPageResult.Hits...) + + // Verify we have the expected number of results + if len(firstPageResult.Hits) != 5 { + t.Errorf("Expected 5 results in first page, got %d", len(firstPageResult.Hits)) + } + if len(secondPageResult.Hits) != 3 { + t.Errorf("Expected 3 results in second page, got %d", len(secondPageResult.Hits)) + } + + // Verify combined RRF results match expected ranking + verifyRRFResults(t, combinedHits) + }) + } +} + +// TestHybridRRFFaceting tests that facet results are identical whether using RRF or default scoring in hybrid search +func TestRRFFaceting(t *testing.T) { + scenarios := []struct { + name string + setup func(t *testing.T) (Index, func()) + }{ + { + name: "SingleIndex", + setup: setupSingleIndex, + }, + { + name: "AliasWithSingleIndex", + setup: setupAliasWithSingleIndex, + }, + { + name: "AliasWithTwoIndexes", + setup: setupAliasWithTwoIndexes, + }, + { + name: "NestedAliases", + setup: setupNestedAliases, + }, + } + + for _, scenario := range scenarios { + t.Run(scenario.name, func(t *testing.T) { + // Setup the index/alias configuration + index, cleanup := scenario.setup(t) + defer cleanup() + + // Create search request with default scoring and facets + defaultRequest := createScoreFusionRequest(ScoreDefault, false) + defaultRequest.Score = ScoreDefault // Use default scoring + defaultRequest.Size = 10 + // Add facet for color field with size 10 + colorFacet := NewFacetRequest("color", 10) + defaultRequest.AddFacet("color", colorFacet) + + // Create search request with RRF scoring and identical facets + rrfRequest := createScoreFusionRequest(ScoreRRF, true) + rrfRequest.Score = ScoreRRF // Use RRF scoring + rrfRequest.Size = 10 + // Add identical facet for color field with size 10 + colorFacetRRF := NewFacetRequest("color", 10) + rrfRequest.AddFacet("color", colorFacetRRF) + + // Execute both searches + defaultResult, err := index.Search(defaultRequest) + if err != nil { + t.Fatalf("Default scoring search failed: %v", err) + } + + rrfResult, err := index.Search(rrfRequest) + if err != nil { + t.Fatalf("RRF scoring search failed: %v", err) + } + + // Verify both searches returned results + if len(defaultResult.Hits) == 0 { + t.Fatal("Expected search results with default scoring, got none") + } + if len(rrfResult.Hits) == 0 { + t.Fatal("Expected search results with RRF scoring, got none") + } + + // Verify both searches returned facets + if defaultResult.Facets == nil { + t.Fatal("Expected facets with default scoring, got nil") + } + if rrfResult.Facets == nil { + t.Fatal("Expected facets with RRF scoring, got nil") + } + + // Check that color facet exists in both results + defaultColorFacet, defaultExists := defaultResult.Facets["color"] + rrfColorFacet, rrfExists := rrfResult.Facets["color"] + + if !defaultExists { + t.Fatal("Expected color facet in default scoring results") + } + if !rrfExists { + t.Fatal("Expected color facet in RRF scoring results") + } + + // Compare the facet results - they should be identical + // Since facets are based on the document corpus and not scoring, + // they should not be affected by the scoring method (even with KNN) + if defaultColorFacet.Total != rrfColorFacet.Total { + t.Errorf("Facet totals differ: default=%d, RRF=%d", + defaultColorFacet.Total, rrfColorFacet.Total) + } + + if defaultColorFacet.Missing != rrfColorFacet.Missing { + t.Errorf("Facet missing counts differ: default=%d, RRF=%d", + defaultColorFacet.Missing, rrfColorFacet.Missing) + } + + if defaultColorFacet.Other != rrfColorFacet.Other { + t.Errorf("Facet other counts differ: default=%d, RRF=%d", + defaultColorFacet.Other, rrfColorFacet.Other) + } + + // Compare the facet terms + defaultTerms := defaultColorFacet.Terms.Terms() + rrfTerms := rrfColorFacet.Terms.Terms() + + if len(defaultTerms) != len(rrfTerms) { + t.Errorf("Facet terms count differs: default=%d, RRF=%d", + len(defaultTerms), len(rrfTerms)) + } else { + // Compare each term + for i, defaultTerm := range defaultTerms { + rrfTerm := rrfTerms[i] + if defaultTerm.Term != rrfTerm.Term { + t.Errorf("Facet term differs at position %d: default=%s, RRF=%s", + i, defaultTerm.Term, rrfTerm.Term) + } + if defaultTerm.Count != rrfTerm.Count { + t.Errorf("Facet term count differs for %s: default=%d, RRF=%d", + defaultTerm.Term, defaultTerm.Count, rrfTerm.Count) + } + } + } + }) + } +} + +// verifyRSFResults verifies that the search hits match expected RSF ranking and scores +func verifyRSFResults(t *testing.T, hits search.DocumentMatchCollection) { + // For RSF, we expect similar high-level results to RRF but with different scoring methodology + // RSF uses min-max normalization of scores within the window + // Expected top documents should include those matching "dark" query and similar vectors + + // Verify we have reasonable number of results + if len(hits) == 0 { + t.Fatal("Expected non-empty search results for RSF") + } + + // Verify we have at least 8 results + if len(hits) < 8 { + t.Errorf("Expected at least 6 results for RSF, got %d", len(hits)) + } + + // Documents that should definitely appear in top results (high relevance) + // These all get both text relevance (for "dark blue") or strong vector similarity + topExpectedDocs := []string{"dark blue", "navy", "blue", "medium blue"} + + // Create map of all hits for easier lookup + docMap := make(map[string]int) // doc -> position (0-based) + for i, hit := range hits { + docMap[hit.ID] = i + } + + // Verify that "dark blue" appears in top 5 positions (high text + vector relevance) + if pos, found := docMap["dark blue"]; !found { + t.Error("Expected 'dark blue' to appear in results but not found") + } else if pos >= 5 { + t.Errorf("Expected 'dark blue' in top 3 positions, found at position %d", pos+1) + } + + // Verify that at least 3 of the top expected documents appear in top 5 results + topFoundCount := 0 + for _, expectedDoc := range topExpectedDocs { + if pos, found := docMap[expectedDoc]; found && pos < 5 { + topFoundCount++ + } + } + if topFoundCount < 3 { + t.Errorf("Expected at least 3 of top expected documents in top 5 results, found %d", topFoundCount) + } + + // Verify scores are reasonable and within expected range + // RSF scores should be between 0 and sum of weights (3.0 with default weights) + // but typically should be more constrained than the full range + for i, hit := range hits { + if hit.Score < 0 || hit.Score > 3.0 { + t.Errorf("Hit %d (%s) has unreasonable score: %.6f", i, hit.ID, hit.Score) + } + // First hit should have a substantial score (at least 0.1) + if i == 0 && hit.Score < 0.1 { + t.Errorf("Top hit (%s) has unexpectedly low score: %.6f", hit.ID, hit.Score) + } + } + + // Verify hits are sorted by score descending with strict ordering + for i := 1; i < len(hits); i++ { + if hits[i-1].Score < hits[i].Score { + t.Errorf("Hits not sorted properly: hit %d (%s, score %.6f) < hit %d (%s, score %.6f)", + i, hits[i-1].ID, hits[i-1].Score, i+1, hits[i].ID, hits[i].Score) + } + } + + // Verify score range is reasonable - top score should be significantly higher than bottom + topScore := hits[0].Score + fifthScore := hits[4].Score + if topScore-fifthScore < 0.001 { + t.Errorf("Insufficient score differentiation: top score %.6f, 5rd score %.6f (diff: %.6f)", + topScore, fifthScore, topScore-fifthScore) + } +} + +// TestRSFEndToEnd tests RSF scoring with a single index +func TestRSFEndToEnd(t *testing.T) { + // Setup the index configuration + index, cleanup := setupSingleIndex(t) + defer cleanup() + + // Create the search request + searchRequest := createScoreFusionRequest(ScoreRSF, true) + + ctx := context.Background() + ctx = context.WithValue(ctx, search.SearchTypeKey, search.GlobalScoring) + + // Execute search + result, err := index.SearchInContext(ctx, searchRequest) + if err != nil { + t.Fatal(err) + } + + // Verify RSF results + verifyRSFResults(t, result.Hits) +} + +// TestRSFAliasWithSingleIndex tests RSF with an alias containing one index +func TestRSFAliasWithSingleIndex(t *testing.T) { + // Setup the alias configuration + alias, cleanup := setupAliasWithSingleIndex(t) + defer cleanup() + + // Create the search request + searchRequest := createScoreFusionRequest(ScoreRSF, true) + + ctx := context.Background() + ctx = context.WithValue(ctx, search.SearchTypeKey, search.GlobalScoring) + + // Execute search + result, err := alias.SearchInContext(ctx, searchRequest) + if err != nil { + t.Fatal(err) + } + + // Verify RSF results - should be identical to direct index search + verifyRSFResults(t, result.Hits) +} + +// TestRSFAliasWithTwoIndexes tests RSF with an alias containing two indexes +func TestRSFAliasWithTwoIndexes(t *testing.T) { + // Setup the alias configuration + alias, cleanup := setupAliasWithTwoIndexes(t) + defer cleanup() + + // Create the search request + searchRequest := createScoreFusionRequest(ScoreRSF, true) + + ctx := context.Background() + ctx = context.WithValue(ctx, search.SearchTypeKey, search.GlobalScoring) + + // Execute search + result, err := alias.SearchInContext(ctx, searchRequest) + if err != nil { + t.Fatal(err) + } + + // Verify RSF results - should be identical to single index results + verifyRSFResults(t, result.Hits) +} + +// TestRSFNestedAliases tests RSF with an alias containing two index aliases +func TestRSFNestedAliases(t *testing.T) { + // Setup the nested aliases configuration + masterAlias, cleanup := setupNestedAliases(t) + defer cleanup() + + // Create the search request + searchRequest := createScoreFusionRequest(ScoreRSF, true) + + ctx := context.Background() + ctx = context.WithValue(ctx, search.SearchTypeKey, search.GlobalScoring) + + // Execute search + result, err := masterAlias.SearchInContext(ctx, searchRequest) + if err != nil { + t.Fatal(err) + } + + // Verify RSF results - should be identical to single index results + verifyRSFResults(t, result.Hits) +} + +// TestRSFPagination tests RSF with pagination across different index/alias configurations +func TestRSFPagination(t *testing.T) { + scenarios := []struct { + name string + setup func(t *testing.T) (Index, func()) + }{ + { + name: "SingleIndex", + setup: setupSingleIndex, + }, + { + name: "AliasWithSingleIndex", + setup: setupAliasWithSingleIndex, + }, + { + name: "AliasWithTwoIndexes", + setup: setupAliasWithTwoIndexes, + }, + { + name: "NestedAliases", + setup: setupNestedAliases, + }, + } + + for _, scenario := range scenarios { + t.Run(scenario.name, func(t *testing.T) { + // Setup the index/alias configuration + index, cleanup := scenario.setup(t) + defer cleanup() + + // Create first page request (first 5 results) + firstPageRequest := createScoreFusionRequest(ScoreDefault, true) + firstPageRequest.From = 0 + firstPageRequest.Size = 5 + + // Execute first page search + firstPageResult, err := index.Search(firstPageRequest) + if err != nil { + t.Fatal(err) + } + + // Create second page request (next 5 results, starting from index 5) + secondPageRequest := createScoreFusionRequest(ScoreDefault, true) + secondPageRequest.From = 5 + secondPageRequest.Size = 5 + + // Execute second page search + secondPageResult, err := index.Search(secondPageRequest) + if err != nil { + t.Fatal(err) + } + + // Combine results from both pages + combinedHits := make(search.DocumentMatchCollection, 0, len(firstPageResult.Hits)+len(secondPageResult.Hits)) + combinedHits = append(combinedHits, firstPageResult.Hits...) + combinedHits = append(combinedHits, secondPageResult.Hits...) + + // Verify we have reasonable number of results + if len(firstPageResult.Hits) == 0 { + t.Error("Expected results in first page, got none") + } + if len(combinedHits) == 0 { + t.Error("Expected combined results, got none") + } + + // Verify combined RSF results + verifyRSFResults(t, combinedHits) + }) + } +} + +// TestRSFFaceting tests that facet results are identical whether using RSF or default scoring in hybrid search +func TestRSFFaceting(t *testing.T) { + scenarios := []struct { + name string + setup func(t *testing.T) (Index, func()) + }{ + { + name: "SingleIndex", + setup: setupSingleIndex, + }, + { + name: "AliasWithSingleIndex", + setup: setupAliasWithSingleIndex, + }, + { + name: "AliasWithTwoIndexes", + setup: setupAliasWithTwoIndexes, + }, + { + name: "NestedAliases", + setup: setupNestedAliases, + }, + } + + for _, scenario := range scenarios { + t.Run(scenario.name, func(t *testing.T) { + // Setup the index/alias configuration + index, cleanup := scenario.setup(t) + defer cleanup() + + // Create search request with default scoring and facets + defaultRequest := createScoreFusionRequest(ScoreDefault, false) + defaultRequest.Score = ScoreDefault // Use default scoring + defaultRequest.Size = 10 + // Add facet for color field with size 10 + colorFacet := NewFacetRequest("color", 10) + defaultRequest.AddFacet("color", colorFacet) + + // Create search request with RSF scoring and identical facets + rsfRequest := createScoreFusionRequest(ScoreRSF, true) + rsfRequest.Size = 10 + // Add identical facet for color field with size 10 + colorFacetRSF := NewFacetRequest("color", 10) + rsfRequest.AddFacet("color", colorFacetRSF) + + // Execute both searches + defaultResult, err := index.Search(defaultRequest) + if err != nil { + t.Fatalf("Default scoring search failed: %v", err) + } + + rsfResult, err := index.Search(rsfRequest) + if err != nil { + t.Fatalf("RSF scoring search failed: %v", err) + } + + // Verify both searches returned results + if len(defaultResult.Hits) == 0 { + t.Fatal("Expected search results with default scoring, got none") + } + if len(rsfResult.Hits) == 0 { + t.Fatal("Expected search results with RSF scoring, got none") + } + + // Verify both searches returned facets + if defaultResult.Facets == nil { + t.Fatal("Expected facets with default scoring, got nil") + } + if rsfResult.Facets == nil { + t.Fatal("Expected facets with RSF scoring, got nil") + } + + // Check that color facet exists in both results + defaultColorFacet, defaultExists := defaultResult.Facets["color"] + rsfColorFacet, rsfExists := rsfResult.Facets["color"] + + if !defaultExists { + t.Fatal("Expected color facet in default scoring results") + } + if !rsfExists { + t.Fatal("Expected color facet in RSF scoring results") + } + + // Compare the facet results - they should be identical + // Since facets are based on the document corpus and not scoring, + // they should not be affected by the scoring method (even with KNN) + if defaultColorFacet.Total != rsfColorFacet.Total { + t.Errorf("Facet totals differ: default=%d, RSF=%d", + defaultColorFacet.Total, rsfColorFacet.Total) + } + + if defaultColorFacet.Missing != rsfColorFacet.Missing { + t.Errorf("Facet missing counts differ: default=%d, RSF=%d", + defaultColorFacet.Missing, rsfColorFacet.Missing) + } + + if defaultColorFacet.Other != rsfColorFacet.Other { + t.Errorf("Facet other counts differ: default=%d, RSF=%d", + defaultColorFacet.Other, rsfColorFacet.Other) + } + + // Compare the facet terms + defaultTerms := defaultColorFacet.Terms.Terms() + rsfTerms := rsfColorFacet.Terms.Terms() + + if len(defaultTerms) != len(rsfTerms) { + t.Errorf("Facet terms count differs: default=%d, RSF=%d", + len(defaultTerms), len(rsfTerms)) + } else { + // Compare each term + for i, defaultTerm := range defaultTerms { + rsfTerm := rsfTerms[i] + if defaultTerm.Term != rsfTerm.Term { + t.Errorf("Facet term differs at position %d: default=%s, RSF=%s", + i, defaultTerm.Term, rsfTerm.Term) + } + if defaultTerm.Count != rsfTerm.Count { + t.Errorf("Facet term count differs for %s: default=%d, RSF=%d", + defaultTerm.Term, defaultTerm.Count, rsfTerm.Count) + } + } + } + }) + } +} diff --git a/rescorer_test.go b/rescorer_test.go new file mode 100644 index 0000000..af44d3a --- /dev/null +++ b/rescorer_test.go @@ -0,0 +1,1329 @@ +// Copyright (c) 2025 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "fmt" + "testing" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/query" +) + +func createFTSIndex(path string) (Index, error) { + // Index mapping for FTS-only testing + indexMapping := NewIndexMapping() + + // Disable default mapping to match expected configuration + indexMapping.DefaultMapping.Enabled = false + indexMapping.DefaultMapping.Dynamic = false + + // Create a specific document mapping type + docMapping := NewDocumentMapping() + docMapping.Enabled = true + docMapping.Dynamic = false + + // Text field for color with specific properties + colorFieldMapping := NewTextFieldMapping() + colorFieldMapping.Analyzer = "en" // Use "en" analyzer as specified + colorFieldMapping.DocValues = true + colorFieldMapping.IncludeInAll = true + colorFieldMapping.Store = true + colorFieldMapping.Index = true + docMapping.AddFieldMappingsAt("color", colorFieldMapping) + + // Text field for description with specific properties + descriptionFieldMapping := NewTextFieldMapping() + descriptionFieldMapping.Analyzer = "en" + descriptionFieldMapping.DocValues = true + descriptionFieldMapping.IncludeInAll = true + descriptionFieldMapping.Store = true + descriptionFieldMapping.Index = true + docMapping.AddFieldMappingsAt("description", descriptionFieldMapping) + + // Text field for category with specific properties + categoryFieldMapping := NewTextFieldMapping() + categoryFieldMapping.Analyzer = "en" + categoryFieldMapping.DocValues = true + categoryFieldMapping.IncludeInAll = true + categoryFieldMapping.Store = true + categoryFieldMapping.Index = true + docMapping.AddFieldMappingsAt("category", categoryFieldMapping) + + // Add the document mapping to the index + indexMapping.AddDocumentMapping("_default", docMapping) + + // Create index + return New(path, indexMapping) +} + +var benchmarkResult search.DocumentMatchCollection + +type benchmarkConfig struct { + name string + ftsHits int + knnHits int + knnQueries int +} + +func BenchmarkRescorerRRF(b *testing.B) { + runRescorerBenchmarks(b, ScoreRRF) +} + +func BenchmarkRescorerRSF(b *testing.B) { + runRescorerBenchmarks(b, ScoreRSF) +} + +func runRescorerBenchmarks(b *testing.B, scoreMode string) { + configs := []benchmarkConfig{ + {name: "small", ftsHits: 256, knnHits: 192, knnQueries: 1}, + {name: "medium", ftsHits: 1024, knnHits: 896, knnQueries: 2}, + {name: "large", ftsHits: 4096, knnHits: 3584, knnQueries: 3}, + } + + for _, cfg := range configs { + b.Run(fmt.Sprintf("%s/%s", scoreMode, cfg.name), func(b *testing.B) { + b.ReportAllocs() + + rescorer, baseFTSHits, baseKNNHits := buildBenchmarkInputs(cfg, scoreMode) + + var last search.DocumentMatchCollection + b.StopTimer() + for i := 0; i < b.N; i++ { + ftsHits := cloneDocumentMatches(baseFTSHits) + knnHits := cloneDocumentMatches(baseKNNHits) + + b.StartTimer() + hits, _, _ := rescorer.rescore(ftsHits, knnHits) + b.StopTimer() + + last = hits + } + + if len(last) == 0 { + b.Fatalf("rescorer returned no hits for config %q", cfg.name) + } + + benchmarkResult = last + }) + } +} + +func buildBenchmarkInputs(cfg benchmarkConfig, scoreMode string) (*rescorer, search.DocumentMatchCollection, search.DocumentMatchCollection) { + windowSize := cfg.ftsHits + if cfg.knnHits > windowSize { + windowSize = cfg.knnHits + } + + matchQuery := query.NewMatchQuery("rescorer benchmark payload") + matchQuery.SetBoost(1.0) + + req := &SearchRequest{ + Query: matchQuery, + Size: cfg.ftsHits, + From: 0, + Score: scoreMode, + Params: &RequestParams{ScoreRankConstant: DefaultScoreRankConstant, ScoreWindowSize: windowSize}, + } + + activeKNNQueries := cfg.knnQueries + if knnAdder, ok := interface{}(req).(interface { + AddKNN(field string, vector []float32, k int64, boost float64) + }); ok { + for i := 0; i < cfg.knnQueries; i++ { + knnAdder.AddKNN(fmt.Sprintf("vector_%d", i), []float32{1.0, 0.5, 0.25}, int64(cfg.knnHits), 1.0) + } + } else { + activeKNNQueries = 0 + } + + r := newRescorer(req) + r.origBoosts = make([]float64, activeKNNQueries+1) + for i := range r.origBoosts { + r.origBoosts[i] = 1.0 + } + + ftsHits, knnHits := buildBenchmarkHits(cfg, activeKNNQueries) + return r, ftsHits, knnHits +} + +func buildBenchmarkHits(cfg benchmarkConfig, activeKNNQueries int) (search.DocumentMatchCollection, search.DocumentMatchCollection) { + ftsHits := make(search.DocumentMatchCollection, cfg.ftsHits) + for i := 0; i < cfg.ftsHits; i++ { + ftsHits[i] = &search.DocumentMatch{ + ID: fmt.Sprintf("doc-%06d", i), + Score: float64(cfg.ftsHits - i), + HitNumber: uint64(i + 1), + } + } + + knnHits := make(search.DocumentMatchCollection, cfg.knnHits) + for i := 0; i < cfg.knnHits; i++ { + id := fmt.Sprintf("doc-%06d", i) + if cfg.ftsHits > 0 { + id = fmt.Sprintf("doc-%06d", i%cfg.ftsHits) + } + if cfg.ftsHits == 0 || i%4 == 0 { + id = fmt.Sprintf("knn-only-%06d", i/4) + } + + scoreBreakdown := make(map[int]float64, activeKNNQueries) + for q := 0; q < activeKNNQueries; q++ { + scoreBreakdown[q] = float64(cfg.knnHits - i + q + 1) + } + + knnHits[i] = &search.DocumentMatch{ + ID: id, + Score: float64(cfg.knnHits - i), + ScoreBreakdown: scoreBreakdown, + HitNumber: uint64(i + 1), + } + } + + return ftsHits, knnHits +} + +func cloneDocumentMatches(src search.DocumentMatchCollection) search.DocumentMatchCollection { + dst := make(search.DocumentMatchCollection, len(src)) + for i, hit := range src { + if hit == nil { + continue + } + + cloned := *hit + + if hit.ScoreBreakdown != nil { + cloned.ScoreBreakdown = make(map[int]float64, len(hit.ScoreBreakdown)) + for k, v := range hit.ScoreBreakdown { + cloned.ScoreBreakdown[k] = v + } + } + + if len(hit.Sort) > 0 { + cloned.Sort = append([]string(nil), hit.Sort...) + } + if len(hit.DecodedSort) > 0 { + cloned.DecodedSort = append([]string(nil), hit.DecodedSort...) + } + if len(hit.IndexNames) > 0 { + cloned.IndexNames = append([]string(nil), hit.IndexNames...) + } + + dst[i] = &cloned + } + return dst +} + +func getFTSDocuments() []map[string]interface{} { + documents := []map[string]interface{}{ + { + "color": "dark slate blue", + "description": "deep and rich color with dark undertones", + "category": "blue shades", + }, + { + "color": "blue", + "description": "primary color that is bright and vibrant", + "category": "primary colors", + }, + { + "color": "navy", + "description": "dark blue color often used in uniforms", + "category": "dark colors", + }, + { + "color": "steel blue", + "description": "metallic blue with gray undertones", + "category": "metallic shades", + }, + { + "color": "light blue", + "description": "pale and soft blue color with light appearance", + "category": "light colors", + }, + { + "color": "deep sky blue", + "description": "bright blue reminiscent of clear skies", + "category": "sky colors", + }, + { + "color": "royal blue", + "description": "rich and regal blue color fit for royalty", + "category": "rich colors", + }, + { + "color": "powder blue", + "description": "very light blue with powder-like softness", + "category": "light colors", + }, + { + "color": "corn flower blue", + "description": "medium blue color named after the flower", + "category": "floral colors", + }, + { + "color": "alice blue", + "description": "very pale blue with light and airy quality", + "category": "light colors", + }, + { + "color": "blue violet", + "description": "purple-blue color with violet undertones", + "category": "purple shades", + }, + { + "color": "sky blue", + "description": "bright blue color of a clear day sky", + "category": "sky colors", + }, + { + "color": "indigo", + "description": "deep purple-blue color with dark intensity", + "category": "dark colors", + }, + { + "color": "midnight blue", + "description": "very dark blue like the night sky", + "category": "dark colors", + }, + { + "color": "dark blue", + "description": "deep blue color with dark characteristics", + "category": "dark colors", + }, + { + "color": "medium slate blue", + "description": "medium intensity blue with slate properties", + "category": "blue shades", + }, + { + "color": "cadet blue", + "description": "grayish blue color often used in uniforms", + "category": "metallic shades", + }, + { + "color": "light steel blue", + "description": "light metallic blue with steel-like appearance", + "category": "light colors", + }, + { + "color": "dodger blue", + "description": "bright medium blue with vibrant intensity", + "category": "bright colors", + }, + { + "color": "medium blue", + "description": "standard blue with medium intensity and saturation", + "category": "blue shades", + }, + { + "color": "slate blue", + "description": "blue-gray color with slate-like properties", + "category": "blue shades", + }, + { + "color": "light sky blue", + "description": "light version of sky blue with airy quality", + "category": "light colors", + }, + } + + return documents +} + +func createFTSSearchRequest(scoreMethod string) *SearchRequest { + // Create multi-FTS search request (multiple FTS queries for fusion scoring) + // Query 1: Search for "dark" in color field + query1 := query.NewMatchPhraseQuery("dark") + query1.SetField("color") + + // Query 2: Search for "light" in description field + query2 := query.NewMatchPhraseQuery("light") + query2.SetField("description") + + // // Query 3: Search for "blue" in category field + query3 := query.NewMatchPhraseQuery("blue") + query3.SetField("category") + + // Use the first query as the main query for the search request + searchRequest := NewSearchRequest(query1) + + // Add additional queries for fusion scoring (this simulates multiple query sources) + // Since SearchRequest doesn't have a direct way to add multiple FTS queries, + // we'll use a disjunction query to combine them for fusion scoring simulation + queries := []query.Query{query1, query2, query3} + disjunctionQuery := query.NewDisjunctionQuery(queries) + searchRequest.Query = disjunctionQuery + + params := RequestParams{1, 10} + searchRequest.AddParams(params) + + searchRequest.Size = 10 + searchRequest.Score = scoreMethod + + searchRequest.Explain = false + return searchRequest +} + +// verifyFTSRRFResults verifies that the search hits match the expected RRF ranking and scores +func verifyFTSRRFResults(t *testing.T, hits search.DocumentMatchCollection) { + // Manual RRF calculation for verification + // With k=1 (ScoreRankConstant), RRF formula: 1/(1+rank) + // + // For FTS-only with disjunction query, we need to consider how each document + // matches each of the three query components: + // 1. "dark" in color field + // 2. "light" in description field + // 3. "blue" in category field + // + // Documents that match multiple query components will rank higher + + // Expected matches: + // Query 1 ("dark" in color): dark slate blue, dark blue, midnight blue (has "dark") + // Query 2 ("light" in description): light blue, powder blue, alice blue, light steel blue, light sky blue + // Query 3 ("blue" in category): dark slate blue, medium slate blue, medium blue, slate blue + + expectedTopDocuments := []string{ + "dark slate blue", // matches query 1 and 3 + "light blue", // matches query 2 + "dark blue", // matches query 1 + "light steel blue", // matches query 2 + "medium slate blue", // matches query 3 + } + + if len(hits) == 0 { + t.Fatal("Expected search results, got none") + } + + // Verify we have results and they're ranked by score + for i := 0; i < len(hits)-1; i++ { + if hits[i].Score < hits[i+1].Score { + t.Errorf("Results not properly ranked by score: position %d (%.6f) < position %d (%.6f)", + i, hits[i].Score, i+1, hits[i+1].Score) + } + } + + // Check that some expected top documents are present in results + foundExpected := 0 + for _, hit := range hits { + for _, expected := range expectedTopDocuments { + if hit.ID == expected { + foundExpected++ + break + } + } + } + + if foundExpected < 3 { + t.Errorf("Expected to find at least 3 of the top expected documents, found %d", foundExpected) + t.Logf("Actual results:") + for i, hit := range hits { + t.Logf(" %d: %s (score: %.6f)", i+1, hit.ID, hit.Score) + } + } +} + +// setupFTSSingleIndex creates a single index with all FTS documents +func setupFTSSingleIndex(t *testing.T) (Index, func()) { + tmpIndexPath := createTmpIndexPath(t) + + index, err := createFTSIndex(tmpIndexPath) + if err != nil { + t.Fatal(err) + } + + documents := getFTSDocuments() + + // Index documents + batch := index.NewBatch() + for _, doc := range documents { + colorName := doc["color"].(string) + err = batch.Index(colorName, doc) + if err != nil { + t.Fatal(err) + } + } + err = index.Batch(batch) + if err != nil { + t.Fatal(err) + } + + cleanup := func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + cleanupTmpIndexPath(t, tmpIndexPath) + } + + return index, cleanup +} + +// setupFTSAliasWithSingleIndex creates an alias containing one index with all FTS documents +func setupFTSAliasWithSingleIndex(t *testing.T) (Index, func()) { + tmpIndexPath := createTmpIndexPath(t) + + index, err := createFTSIndex(tmpIndexPath) + if err != nil { + t.Fatal(err) + } + + documents := getFTSDocuments() + + // Create alias and add the single index + alias := NewIndexAlias() + alias.Add(index) + + // Index all documents + batch := alias.NewBatch() + for _, doc := range documents { + colorName := doc["color"].(string) + err = batch.Index(colorName, doc) + if err != nil { + t.Fatal(err) + } + } + err = alias.Batch(batch) + if err != nil { + t.Fatal(err) + } + + cleanup := func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + cleanupTmpIndexPath(t, tmpIndexPath) + } + + return alias, cleanup +} + +// setupFTSAliasWithTwoIndexes creates an alias containing two indexes with FTS documents split between them +func setupFTSAliasWithTwoIndexes(t *testing.T) (Index, func()) { + documents := getFTSDocuments() + + // Split documents into two groups + midpoint := len(documents) / 2 + docs1 := documents[:midpoint] + docs2 := documents[midpoint:] + + // Create first index + tmpIndexPath1 := createTmpIndexPath(t) + index1, err := createFTSIndex(tmpIndexPath1) + if err != nil { + t.Fatal(err) + } + + // Index first half of documents + batch1 := index1.NewBatch() + for _, doc := range docs1 { + colorName := doc["color"].(string) + err = batch1.Index(colorName, doc) + if err != nil { + t.Fatal(err) + } + } + err = index1.Batch(batch1) + if err != nil { + t.Fatal(err) + } + + // Create second index + tmpIndexPath2 := createTmpIndexPath(t) + index2, err := createFTSIndex(tmpIndexPath2) + if err != nil { + t.Fatal(err) + } + + // Index second half of documents + batch2 := index2.NewBatch() + for _, doc := range docs2 { + colorName := doc["color"].(string) + err = batch2.Index(colorName, doc) + if err != nil { + t.Fatal(err) + } + } + err = index2.Batch(batch2) + if err != nil { + t.Fatal(err) + } + + // Create alias and add both indexes + alias := NewIndexAlias() + alias.Add(index1, index2) + + cleanup := func() { + err := index1.Close() + if err != nil { + t.Fatal(err) + } + err = index2.Close() + if err != nil { + t.Fatal(err) + } + cleanupTmpIndexPath(t, tmpIndexPath1) + cleanupTmpIndexPath(t, tmpIndexPath2) + } + + return alias, cleanup +} + +// setupFTSNestedAliases creates nested aliases with three indexes spread across sub-aliases +func setupFTSNestedAliases(t *testing.T) (Index, func()) { + documents := getFTSDocuments() + + // Split documents into three groups + thirdPoint1 := len(documents) / 3 + thirdPoint2 := 2 * len(documents) / 3 + docs1 := documents[:thirdPoint1] + docs2 := documents[thirdPoint1:thirdPoint2] + docs3 := documents[thirdPoint2:] + + // Create first index + tmpIndexPath1 := createTmpIndexPath(t) + index1, err := createFTSIndex(tmpIndexPath1) + if err != nil { + t.Fatal(err) + } + + // Index first third of documents + batch1 := index1.NewBatch() + for _, doc := range docs1 { + colorName := doc["color"].(string) + err = batch1.Index(colorName, doc) + if err != nil { + t.Fatal(err) + } + } + err = index1.Batch(batch1) + if err != nil { + t.Fatal(err) + } + + // Create second index + tmpIndexPath2 := createTmpIndexPath(t) + index2, err := createFTSIndex(tmpIndexPath2) + if err != nil { + t.Fatal(err) + } + + // Index second third of documents + batch2 := index2.NewBatch() + for _, doc := range docs2 { + colorName := doc["color"].(string) + err = batch2.Index(colorName, doc) + if err != nil { + t.Fatal(err) + } + } + err = index2.Batch(batch2) + if err != nil { + t.Fatal(err) + } + + // Create third index + tmpIndexPath3 := createTmpIndexPath(t) + index3, err := createFTSIndex(tmpIndexPath3) + if err != nil { + t.Fatal(err) + } + + // Index third third of documents + batch3 := index3.NewBatch() + for _, doc := range docs3 { + colorName := doc["color"].(string) + err = batch3.Index(colorName, doc) + if err != nil { + t.Fatal(err) + } + } + err = index3.Batch(batch3) + if err != nil { + t.Fatal(err) + } + + // Create first sub-alias (contains 1 index) + subAlias1 := NewIndexAlias() + subAlias1.SetName("subAlias1") + subAlias1.Add(index1) + + // Create second sub-alias (contains 2 indexes) + subAlias2 := NewIndexAlias() + subAlias2.SetName("subAlias2") + subAlias2.Add(index2, index3) + + // Create master alias containing the two sub-aliases + masterAlias := NewIndexAlias() + masterAlias.SetName("masterAlias") + masterAlias.Add(subAlias1, subAlias2) + + cleanup := func() { + err := index1.Close() + if err != nil { + t.Fatal(err) + } + err = index2.Close() + if err != nil { + t.Fatal(err) + } + err = index3.Close() + if err != nil { + t.Fatal(err) + } + cleanupTmpIndexPath(t, tmpIndexPath1) + cleanupTmpIndexPath(t, tmpIndexPath2) + cleanupTmpIndexPath(t, tmpIndexPath3) + } + + return masterAlias, cleanup +} + +func TestFTSRRFEndToEnd(t *testing.T) { + // Setup the index configuration + index, cleanup := setupFTSSingleIndex(t) + defer cleanup() + + // Create the search request + searchRequest := createFTSSearchRequest(ScoreRRF) + + // Execute search + result, err := index.Search(searchRequest) + if err != nil { + t.Fatal(err) + } + + // Verify FTS RRF results + verifyFTSRRFResults(t, result.Hits) +} + +// TestFTSRRFAliasWithSingleIndex tests RRF with an alias containing one index +func TestFTSRRFAliasWithSingleIndex(t *testing.T) { + // Setup the alias configuration + alias, cleanup := setupFTSAliasWithSingleIndex(t) + defer cleanup() + + // Create the search request + searchRequest := createFTSSearchRequest(ScoreRRF) + + // Execute search through alias + result, err := alias.Search(searchRequest) + if err != nil { + t.Fatal(err) + } + + // Verify FTS RRF results - should be identical to direct index search + verifyFTSRRFResults(t, result.Hits) +} + +// TestFTSRRFAliasWithTwoIndexes tests RRF with an alias containing two indexes +func TestFTSRRFAliasWithTwoIndexes(t *testing.T) { + // Setup the alias configuration + alias, cleanup := setupFTSAliasWithTwoIndexes(t) + defer cleanup() + + // Create the search request + searchRequest := createFTSSearchRequest(ScoreRRF) + + // Execute search through alias + result, err := alias.Search(searchRequest) + if err != nil { + t.Fatal(err) + } + + // Verify FTS RRF results - should be consistent across distributed indexes + verifyFTSRRFResults(t, result.Hits) +} + +// TestFTSRRFNestedAliases tests RRF with an alias containing two index aliases +func TestFTSRRFNestedAliases(t *testing.T) { + // Setup the nested aliases configuration + masterAlias, cleanup := setupFTSNestedAliases(t) + defer cleanup() + + // Create the search request + searchRequest := createFTSSearchRequest(ScoreRRF) + + // Execute search through master alias + result, err := masterAlias.Search(searchRequest) + if err != nil { + t.Fatal(err) + } + + // Verify FTS RRF results - should be consistent across nested aliases + verifyFTSRRFResults(t, result.Hits) +} + +// TestFTSRRFPagination tests FTS RRF with pagination across different index/alias configurations +func TestFTSRRFPagination(t *testing.T) { + scenarios := []struct { + name string + setup func(t *testing.T) (Index, func()) + }{ + { + name: "SingleIndex", + setup: setupFTSSingleIndex, + }, + { + name: "AliasWithSingleIndex", + setup: setupFTSAliasWithSingleIndex, + }, + { + name: "AliasWithTwoIndexes", + setup: setupFTSAliasWithTwoIndexes, + }, + { + name: "NestedAliases", + setup: setupFTSNestedAliases, + }, + } + + for _, scenario := range scenarios { + t.Run(scenario.name, func(t *testing.T) { + // Setup the index/alias configuration + index, cleanup := scenario.setup(t) + defer cleanup() + + // Create first page request (first 5 results) + firstPageRequest := createFTSSearchRequest(ScoreRRF) + firstPageRequest.From = 0 + firstPageRequest.Size = 5 + + // Execute first page search + firstPageResult, err := index.Search(firstPageRequest) + if err != nil { + t.Fatal(err) + } + + // Create second page request (next 5 results, starting from index 5) + secondPageRequest := createFTSSearchRequest(ScoreRRF) + secondPageRequest.From = 5 + secondPageRequest.Size = 5 + + // Execute second page search + secondPageResult, err := index.Search(secondPageRequest) + if err != nil { + t.Fatal(err) + } + + // Combine results from both pages + combinedHits := make(search.DocumentMatchCollection, 0, len(firstPageResult.Hits)+len(secondPageResult.Hits)) + combinedHits = append(combinedHits, firstPageResult.Hits...) + combinedHits = append(combinedHits, secondPageResult.Hits...) + + // Verify we have results (FTS may have variable results based on matches) + if len(firstPageResult.Hits) == 0 { + t.Fatal("Expected at least some results in first page, got 0") + } + if len(firstPageResult.Hits) > 5 { + t.Errorf("Expected at most 5 results in first page, got %d", len(firstPageResult.Hits)) + } + + // Total hits should not exceed the number of documents that match our queries + totalHits := len(combinedHits) + if totalHits == 0 { + t.Fatal("Expected at least some combined results, got 0") + } + + // Verify combined FTS RRF results + verifyFTSRRFResults(t, combinedHits) + }) + } +} + +// TestFTSRRFFaceting tests that facet results are identical whether using RRF or default scoring +func TestFTSRRFFaceting(t *testing.T) { + scenarios := []struct { + name string + setup func(t *testing.T) (Index, func()) + }{ + { + name: "SingleIndex", + setup: setupFTSSingleIndex, + }, + { + name: "AliasWithSingleIndex", + setup: setupFTSAliasWithSingleIndex, + }, + { + name: "AliasWithTwoIndexes", + setup: setupFTSAliasWithTwoIndexes, + }, + { + name: "NestedAliases", + setup: setupFTSNestedAliases, + }, + } + + for _, scenario := range scenarios { + t.Run(scenario.name, func(t *testing.T) { + // Setup the index/alias configuration + index, cleanup := scenario.setup(t) + defer cleanup() + + // Create search request with default scoring and facets + defaultRequest := createFTSSearchRequest(ScoreRRF) + defaultRequest.Score = ScoreDefault // Use default scoring + defaultRequest.Size = 10 + // Add facet for color field with size 10 + colorFacet := NewFacetRequest("color", 10) + defaultRequest.AddFacet("color", colorFacet) + + // Create search request with RRF scoring and identical facets + rrfRequest := createFTSSearchRequest(ScoreRRF) + rrfRequest.Size = 10 + rrfRequest.Score = ScoreRRF + // Add identical facet for color field with size 10 + colorFacetRRF := NewFacetRequest("color", 10) + rrfRequest.AddFacet("color", colorFacetRRF) + + // Execute both searches + defaultResult, err := index.Search(defaultRequest) + if err != nil { + t.Fatalf("Default scoring search failed: %v", err) + } + + rrfResult, err := index.Search(rrfRequest) + if err != nil { + t.Fatalf("RRF scoring search failed: %v", err) + } + + // Verify both searches returned results + if len(defaultResult.Hits) == 0 { + t.Fatal("Expected search results with default scoring, got none") + } + if len(rrfResult.Hits) == 0 { + t.Fatal("Expected search results with RRF scoring, got none") + } + + // Verify both searches returned facets + if defaultResult.Facets == nil { + t.Fatal("Expected facets with default scoring, got nil") + } + if rrfResult.Facets == nil { + t.Fatal("Expected facets with RRF scoring, got nil") + } + + // Check that color facet exists in both results + defaultColorFacet, defaultExists := defaultResult.Facets["color"] + rrfColorFacet, rrfExists := rrfResult.Facets["color"] + + if !defaultExists { + t.Fatal("Expected color facet in default scoring results") + } + if !rrfExists { + t.Fatal("Expected color facet in RRF scoring results") + } + + // Compare the facet results - they should be identical + // Since facets are based on the document corpus and not scoring, + // they should not be affected by the scoring method + if defaultColorFacet.Total != rrfColorFacet.Total { + t.Errorf("Facet totals differ: default=%d, RRF=%d", + defaultColorFacet.Total, rrfColorFacet.Total) + } + + if defaultColorFacet.Missing != rrfColorFacet.Missing { + t.Errorf("Facet missing counts differ: default=%d, RRF=%d", + defaultColorFacet.Missing, rrfColorFacet.Missing) + } + + if defaultColorFacet.Other != rrfColorFacet.Other { + t.Errorf("Facet other counts differ: default=%d, RRF=%d", + defaultColorFacet.Other, rrfColorFacet.Other) + } + + // Compare the facet terms + defaultTerms := defaultColorFacet.Terms.Terms() + rrfTerms := rrfColorFacet.Terms.Terms() + + if len(defaultTerms) != len(rrfTerms) { + t.Errorf("Facet terms count differs: default=%d, RRF=%d", + len(defaultTerms), len(rrfTerms)) + } else { + // Compare each term + for i, defaultTerm := range defaultTerms { + rrfTerm := rrfTerms[i] + if defaultTerm.Term != rrfTerm.Term { + t.Errorf("Facet term differs at position %d: default=%s, RRF=%s", + i, defaultTerm.Term, rrfTerm.Term) + } + if defaultTerm.Count != rrfTerm.Count { + t.Errorf("Facet term count differs for %s: default=%d, RRF=%d", + defaultTerm.Term, defaultTerm.Count, rrfTerm.Count) + } + } + } + }) + } +} + +// verifyFTSRSFResults verifies that the search hits match expected RSF ranking and scores for FTS-only search +func verifyFTSRSFResults(t *testing.T, hits search.DocumentMatchCollection) { + // For FTS RSF, we expect documents that match multiple query components to rank higher + // Query components: "dark" in color, "light" in description, "blue" in category + // RSF uses min-max normalization of scores within the window + + // Verify we have reasonable number of results + if len(hits) == 0 { + t.Fatal("Expected non-empty search results for FTS RSF") + } + + // Verify we have at least 5 results for meaningful comparison + if len(hits) < 5 { + t.Errorf("Expected at least 5 results for FTS RSF, got %d", len(hits)) + } + + // Documents that should appear in top results based on multi-query matching: + // - "dark slate blue": matches "dark" in color AND "blue" in category (2 matches) + // - "light blue": matches "light" in description (1 strong match) + // - "dark blue": matches "dark" in color (1 match) + // - Documents with "light" in description should rank well + topExpectedDocs := []string{"dark slate blue", "light blue", "dark blue", "medium slate blue", "light sky blue"} + + // Create map of all hits for easier lookup + docMap := make(map[string]int) // doc -> position (0-based) + for i, hit := range hits { + docMap[hit.ID] = i + } + + // Verify that "dark slate blue" appears in top 3 positions (matches 2 query components) + if pos, found := docMap["dark slate blue"]; !found { + t.Error("Expected 'dark slate blue' to appear in results but not found") + } else if pos >= 3 { + t.Errorf("Expected 'dark slate blue' in top 3 positions, found at position %d", pos+1) + } + + // Verify that at least 3 of the top expected documents appear in top 5 results + topFoundCount := 0 + for _, expectedDoc := range topExpectedDocs { + if pos, found := docMap[expectedDoc]; found && pos < 5 { + topFoundCount++ + } + } + if topFoundCount < 3 { + t.Errorf("Expected at least 3 of top expected documents in top 5 results, found %d", topFoundCount) + } + + // Verify scores are reasonable and within expected range + // RSF scores should be between 0 and sum of weights (3.0 with default weights) + for i, hit := range hits { + if hit.Score < 0 || hit.Score > 3.0 { + t.Errorf("Hit %d (%s) has unreasonable score: %.6f", i, hit.ID, hit.Score) + } + // First hit should have a substantial score (at least 0.1) + if i == 0 && hit.Score < 0.1 { + t.Errorf("Top hit (%s) has unexpectedly low score: %.6f", hit.ID, hit.Score) + } + } + + // Verify hits are sorted by score descending with strict ordering + for i := 1; i < len(hits); i++ { + if hits[i-1].Score < hits[i].Score { + t.Errorf("Hits not sorted properly: hit %d (%s, score %.6f) < hit %d (%s, score %.6f)", + i, hits[i-1].ID, hits[i-1].Score, i+1, hits[i].ID, hits[i].Score) + } + } + + // Verify score range is reasonable - top score should be significantly higher than 5th + if len(hits) >= 5 { + topScore := hits[0].Score + fifthScore := hits[4].Score + if topScore-fifthScore < 0.001 { + t.Errorf("Insufficient score differentiation: top score %.6f, 5th score %.6f (diff: %.6f)", + topScore, fifthScore, topScore-fifthScore) + } + } +} + +// TestFTSRSFEndToEnd tests RSF scoring with a single FTS index +func TestFTSRSFEndToEnd(t *testing.T) { + // Setup the index configuration + index, cleanup := setupFTSSingleIndex(t) + defer cleanup() + + // Create the search request + searchRequest := createFTSSearchRequest(ScoreRSF) + + // Execute search + result, err := index.Search(searchRequest) + if err != nil { + t.Fatal(err) + } + + // Verify FTS RSF results + verifyFTSRSFResults(t, result.Hits) +} + +// TestFTSRSFAliasWithSingleIndex tests RSF with an alias containing one FTS index +func TestFTSRSFAliasWithSingleIndex(t *testing.T) { + // Setup the alias configuration + alias, cleanup := setupFTSAliasWithSingleIndex(t) + defer cleanup() + + // Create the search request + searchRequest := createFTSSearchRequest(ScoreRSF) + + // Execute search through alias + result, err := alias.Search(searchRequest) + if err != nil { + t.Fatal(err) + } + + // Verify FTS RSF results - should be identical to direct index search + verifyFTSRSFResults(t, result.Hits) +} + +// TestFTSRSFAliasWithTwoIndexes tests RSF with an alias containing two FTS indexes +func TestFTSRSFAliasWithTwoIndexes(t *testing.T) { + // Setup the alias configuration + alias, cleanup := setupFTSAliasWithTwoIndexes(t) + defer cleanup() + + // Create the search request + searchRequest := createFTSSearchRequest(ScoreRSF) + + // Execute search through alias + result, err := alias.Search(searchRequest) + if err != nil { + t.Fatal(err) + } + + // Verify FTS RSF results - should be consistent across distributed indexes + verifyFTSRSFResults(t, result.Hits) +} + +// TestFTSRSFNestedAliases tests RSF with an alias containing two index aliases +func TestFTSRSFNestedAliases(t *testing.T) { + // Setup the nested aliases configuration + masterAlias, cleanup := setupFTSNestedAliases(t) + defer cleanup() + + // Create the search request + searchRequest := createFTSSearchRequest(ScoreRSF) + + // Execute search through master alias + result, err := masterAlias.Search(searchRequest) + if err != nil { + t.Fatal(err) + } + + // Verify FTS RSF results - should be consistent across nested aliases + verifyFTSRSFResults(t, result.Hits) +} + +// TestFTSRSFPagination tests FTS RSF with pagination across different index/alias configurations +func TestFTSRSFPagination(t *testing.T) { + scenarios := []struct { + name string + setup func(t *testing.T) (Index, func()) + }{ + { + name: "SingleIndex", + setup: setupFTSSingleIndex, + }, + { + name: "AliasWithSingleIndex", + setup: setupFTSAliasWithSingleIndex, + }, + { + name: "AliasWithTwoIndexes", + setup: setupFTSAliasWithTwoIndexes, + }, + { + name: "NestedAliases", + setup: setupFTSNestedAliases, + }, + } + + for _, scenario := range scenarios { + t.Run(scenario.name, func(t *testing.T) { + // Setup the index/alias configuration + index, cleanup := scenario.setup(t) + defer cleanup() + + // Create first page request (first 5 results) + firstPageRequest := createFTSSearchRequest(ScoreRSF) + firstPageRequest.From = 0 + firstPageRequest.Size = 5 + + // Execute first page search + firstPageResult, err := index.Search(firstPageRequest) + if err != nil { + t.Fatal(err) + } + + // Create second page request (next 5 results, starting from index 5) + secondPageRequest := createFTSSearchRequest(ScoreRSF) + secondPageRequest.From = 5 + secondPageRequest.Size = 5 + + // Execute second page search + secondPageResult, err := index.Search(secondPageRequest) + if err != nil { + t.Fatal(err) + } + + // Combine results from both pages + combinedHits := make(search.DocumentMatchCollection, 0, len(firstPageResult.Hits)+len(secondPageResult.Hits)) + combinedHits = append(combinedHits, firstPageResult.Hits...) + combinedHits = append(combinedHits, secondPageResult.Hits...) + + // Verify we have results (FTS may have variable results based on matches) + if len(firstPageResult.Hits) == 0 { + t.Fatal("Expected at least some results in first page, got 0") + } + if len(firstPageResult.Hits) > 5 { + t.Errorf("Expected at most 5 results in first page, got %d", len(firstPageResult.Hits)) + } + + // Total hits should not exceed the number of documents that match our queries + totalHits := len(combinedHits) + if totalHits == 0 { + t.Fatal("Expected at least some combined results, got 0") + } + + // Verify combined FTS RSF results + verifyFTSRSFResults(t, combinedHits) + }) + } +} + +// TestFTSRSFFaceting tests that facet results are identical whether using RSF or default scoring +func TestFTSRSFFaceting(t *testing.T) { + scenarios := []struct { + name string + setup func(t *testing.T) (Index, func()) + }{ + { + name: "SingleIndex", + setup: setupFTSSingleIndex, + }, + { + name: "AliasWithSingleIndex", + setup: setupFTSAliasWithSingleIndex, + }, + { + name: "AliasWithTwoIndexes", + setup: setupFTSAliasWithTwoIndexes, + }, + { + name: "NestedAliases", + setup: setupFTSNestedAliases, + }, + } + + for _, scenario := range scenarios { + t.Run(scenario.name, func(t *testing.T) { + // Setup the index/alias configuration + index, cleanup := scenario.setup(t) + defer cleanup() + + // Create search request with default scoring and facets + defaultRequest := createFTSSearchRequest(ScoreRRF) + defaultRequest.Score = ScoreDefault // Use default scoring + defaultRequest.Size = 10 + // Add facet for category field with size 10 + categoryFacet := NewFacetRequest("category", 10) + defaultRequest.AddFacet("category", categoryFacet) + + // Create search request with RSF scoring and identical facets + rsfRequest := createFTSSearchRequest(ScoreRSF) + rsfRequest.Size = 10 + // Add identical facet for category field with size 10 + categoryFacetRSF := NewFacetRequest("category", 10) + rsfRequest.AddFacet("category", categoryFacetRSF) + + // Execute both searches + defaultResult, err := index.Search(defaultRequest) + if err != nil { + t.Fatalf("Default scoring search failed: %v", err) + } + + rsfResult, err := index.Search(rsfRequest) + if err != nil { + t.Fatalf("RSF scoring search failed: %v", err) + } + + // Verify both searches returned results + if len(defaultResult.Hits) == 0 { + t.Fatal("Expected search results with default scoring, got none") + } + if len(rsfResult.Hits) == 0 { + t.Fatal("Expected search results with RSF scoring, got none") + } + + // Verify both searches returned facets + if defaultResult.Facets == nil { + t.Fatal("Expected facets with default scoring, got nil") + } + if rsfResult.Facets == nil { + t.Fatal("Expected facets with RSF scoring, got nil") + } + + // Check that category facet exists in both results + defaultCategoryFacet, defaultExists := defaultResult.Facets["category"] + rsfCategoryFacet, rsfExists := rsfResult.Facets["category"] + + if !defaultExists { + t.Fatal("Expected category facet in default scoring results") + } + if !rsfExists { + t.Fatal("Expected category facet in RSF scoring results") + } + + // Compare the facet results - they should be identical + // Since facets are based on the document corpus and not scoring, + // they should not be affected by the scoring method + if defaultCategoryFacet.Total != rsfCategoryFacet.Total { + t.Errorf("Facet totals differ: default=%d, RSF=%d", + defaultCategoryFacet.Total, rsfCategoryFacet.Total) + } + + if defaultCategoryFacet.Missing != rsfCategoryFacet.Missing { + t.Errorf("Facet missing counts differ: default=%d, RSF=%d", + defaultCategoryFacet.Missing, rsfCategoryFacet.Missing) + } + + if defaultCategoryFacet.Other != rsfCategoryFacet.Other { + t.Errorf("Facet other counts differ: default=%d, RSF=%d", + defaultCategoryFacet.Other, rsfCategoryFacet.Other) + } + + // Compare the facet terms + defaultTerms := defaultCategoryFacet.Terms.Terms() + rsfTerms := rsfCategoryFacet.Terms.Terms() + + if len(defaultTerms) != len(rsfTerms) { + t.Errorf("Facet terms count differs: default=%d, RSF=%d", + len(defaultTerms), len(rsfTerms)) + } else { + // Compare each term + for i, defaultTerm := range defaultTerms { + rsfTerm := rsfTerms[i] + if defaultTerm.Term != rsfTerm.Term { + t.Errorf("Facet term differs at position %d: default=%s, RSF=%s", + i, defaultTerm.Term, rsfTerm.Term) + } + if defaultTerm.Count != rsfTerm.Count { + t.Errorf("Facet term count differs for %s: default=%d, RSF=%d", + defaultTerm.Term, defaultTerm.Count, rsfTerm.Count) + } + } + } + }) + } +} diff --git a/search.go b/search.go new file mode 100644 index 0000000..a754a5a --- /dev/null +++ b/search.go @@ -0,0 +1,867 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + "regexp" + "slices" + "sort" + "strconv" + "strings" + "time" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/datetime/optional" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/collector" + "github.com/blevesearch/bleve/v2/search/query" + "github.com/blevesearch/bleve/v2/size" + "github.com/blevesearch/bleve/v2/util" +) + +var ( + reflectStaticSizeSearchResult int + reflectStaticSizeSearchStatus int +) + +func init() { + var sr SearchResult + reflectStaticSizeSearchResult = int(reflect.TypeOf(sr).Size()) + var ss SearchStatus + reflectStaticSizeSearchStatus = int(reflect.TypeOf(ss).Size()) +} + +var cache = registry.NewCache() + +const defaultDateTimeParser = optional.Name + +const ( + ScoreDefault = "" + ScoreNone = "none" + ScoreRRF = "rrf" + ScoreRSF = "rsf" +) + +var AllowedFusionSort = search.SortOrder{&search.SortScore{Desc: true}} + +type dateTimeRange struct { + Name string `json:"name,omitempty"` + Start time.Time `json:"start,omitempty"` + End time.Time `json:"end,omitempty"` + DateTimeParser string `json:"datetime_parser,omitempty"` + startString *string + endString *string +} + +func (dr *dateTimeRange) ParseDates(dateTimeParser analysis.DateTimeParser) (start, end time.Time, err error) { + start = dr.Start + if dr.Start.IsZero() && dr.startString != nil { + s, _, parseError := dateTimeParser.ParseDateTime(*dr.startString) + if parseError != nil { + return start, end, fmt.Errorf("error parsing start date '%s' for date range name '%s': %v", *dr.startString, dr.Name, parseError) + } + start = s + } + end = dr.End + if dr.End.IsZero() && dr.endString != nil { + e, _, parseError := dateTimeParser.ParseDateTime(*dr.endString) + if parseError != nil { + return start, end, fmt.Errorf("error parsing end date '%s' for date range name '%s': %v", *dr.endString, dr.Name, parseError) + } + end = e + } + return start, end, err +} + +func (dr *dateTimeRange) UnmarshalJSON(input []byte) error { + var temp struct { + Name string `json:"name,omitempty"` + Start *string `json:"start,omitempty"` + End *string `json:"end,omitempty"` + DateTimeParser string `json:"datetime_parser,omitempty"` + } + + if err := util.UnmarshalJSON(input, &temp); err != nil { + return err + } + + dr.Name = temp.Name + if temp.Start != nil { + dr.startString = temp.Start + } + if temp.End != nil { + dr.endString = temp.End + } + if temp.DateTimeParser != "" { + dr.DateTimeParser = temp.DateTimeParser + } + + return nil +} + +func (dr *dateTimeRange) MarshalJSON() ([]byte, error) { + rv := map[string]interface{}{ + "name": dr.Name, + } + + if !dr.Start.IsZero() { + rv["start"] = dr.Start + } else if dr.startString != nil { + rv["start"] = dr.startString + } + + if !dr.End.IsZero() { + rv["end"] = dr.End + } else if dr.endString != nil { + rv["end"] = dr.endString + } + + if dr.DateTimeParser != "" { + rv["datetime_parser"] = dr.DateTimeParser + } + return util.MarshalJSON(rv) +} + +type numericRange struct { + Name string `json:"name,omitempty"` + Min *float64 `json:"min,omitempty"` + Max *float64 `json:"max,omitempty"` +} + +// A FacetRequest describes a facet or aggregation +// of the result document set you would like to be +// built. +type FacetRequest struct { + Size int `json:"size"` + Field string `json:"field"` + TermPrefix string `json:"term_prefix,omitempty"` + TermPattern string `json:"term_pattern,omitempty"` + NumericRanges []*numericRange `json:"numeric_ranges,omitempty"` + DateTimeRanges []*dateTimeRange `json:"date_ranges,omitempty"` + + // Compiled regex pattern (cached during validation) + compiledPattern *regexp.Regexp +} + +// NewFacetRequest creates a facet on the specified +// field that limits the number of entries to the +// specified size. +func NewFacetRequest(field string, size int) *FacetRequest { + return &FacetRequest{ + Field: field, + Size: size, + } +} + +// SetPrefixFilter sets the prefix filter for term facets. +func (fr *FacetRequest) SetPrefixFilter(prefix string) { + fr.TermPrefix = prefix +} + +// SetRegexFilter sets the regex pattern filter for term facets. +func (fr *FacetRequest) SetRegexFilter(pattern string) { + fr.TermPattern = pattern +} + +func (fr *FacetRequest) Validate() error { + // Validate regex pattern if provided and cache the compiled regex + if fr.TermPattern != "" { + compiled, err := regexp.Compile(fr.TermPattern) + if err != nil { + return fmt.Errorf("invalid term pattern: %v", err) + } + fr.compiledPattern = compiled + } + + nrCount := len(fr.NumericRanges) + drCount := len(fr.DateTimeRanges) + if nrCount > 0 && drCount > 0 { + return fmt.Errorf("facet can only contain numeric ranges or date ranges, not both") + } + + if nrCount > 0 { + nrNames := map[string]interface{}{} + for _, nr := range fr.NumericRanges { + if _, ok := nrNames[nr.Name]; ok { + return fmt.Errorf("numeric ranges contains duplicate name '%s'", nr.Name) + } + nrNames[nr.Name] = struct{}{} + if nr.Min == nil && nr.Max == nil { + return fmt.Errorf("numeric range query must specify either min, max or both for range name '%s'", nr.Name) + } + } + + } else { + dateTimeParser, err := cache.DateTimeParserNamed(defaultDateTimeParser) + if err != nil { + return err + } + drNames := map[string]interface{}{} + for _, dr := range fr.DateTimeRanges { + if _, ok := drNames[dr.Name]; ok { + return fmt.Errorf("date ranges contains duplicate name '%s'", dr.Name) + } + drNames[dr.Name] = struct{}{} + if dr.DateTimeParser == "" { + // cannot parse the date range dates as the defaultDateTimeParser is overridden + // so perform this validation at query time + start, end, err := dr.ParseDates(dateTimeParser) + if err != nil { + return fmt.Errorf("ParseDates err: %v, using date time parser named %s", err, defaultDateTimeParser) + } + if start.IsZero() && end.IsZero() { + return fmt.Errorf("date range query must specify either start, end or both for range name '%s'", dr.Name) + } + } + } + } + + return nil +} + +// AddDateTimeRange adds a bucket to a field +// containing date values. Documents with a +// date value falling into this range are tabulated +// as part of this bucket/range. +func (fr *FacetRequest) AddDateTimeRange(name string, start, end time.Time) { + if fr.DateTimeRanges == nil { + fr.DateTimeRanges = make([]*dateTimeRange, 0, 1) + } + fr.DateTimeRanges = append(fr.DateTimeRanges, &dateTimeRange{Name: name, Start: start, End: end}) +} + +// AddDateTimeRangeString adds a bucket to a field +// containing date values. Uses defaultDateTimeParser to parse the date strings. +func (fr *FacetRequest) AddDateTimeRangeString(name string, start, end *string) { + if fr.DateTimeRanges == nil { + fr.DateTimeRanges = make([]*dateTimeRange, 0, 1) + } + fr.DateTimeRanges = append(fr.DateTimeRanges, + &dateTimeRange{Name: name, startString: start, endString: end}) +} + +// AddDateTimeRangeString adds a bucket to a field +// containing date values. Uses the specified parser to parse the date strings. +// provided the parser is registered in the index mapping. +func (fr *FacetRequest) AddDateTimeRangeStringWithParser(name string, start, end *string, parser string) { + if fr.DateTimeRanges == nil { + fr.DateTimeRanges = make([]*dateTimeRange, 0, 1) + } + fr.DateTimeRanges = append(fr.DateTimeRanges, + &dateTimeRange{Name: name, startString: start, endString: end, DateTimeParser: parser}) +} + +// AddNumericRange adds a bucket to a field +// containing numeric values. Documents with a +// numeric value falling into this range are +// tabulated as part of this bucket/range. +func (fr *FacetRequest) AddNumericRange(name string, min, max *float64) { + if fr.NumericRanges == nil { + fr.NumericRanges = make([]*numericRange, 0, 1) + } + fr.NumericRanges = append(fr.NumericRanges, &numericRange{Name: name, Min: min, Max: max}) +} + +// FacetsRequest groups together all the +// FacetRequest objects for a single query. +type FacetsRequest map[string]*FacetRequest + +func (fr FacetsRequest) Validate() error { + for _, v := range fr { + if err := v.Validate(); err != nil { + return err + } + } + return nil +} + +// HighlightRequest describes how field matches +// should be highlighted. +type HighlightRequest struct { + Style *string `json:"style"` + Fields []string `json:"fields"` +} + +// NewHighlight creates a default +// HighlightRequest. +func NewHighlight() *HighlightRequest { + return &HighlightRequest{} +} + +// NewHighlightWithStyle creates a HighlightRequest +// with an alternate style. +func NewHighlightWithStyle(style string) *HighlightRequest { + return &HighlightRequest{ + Style: &style, + } +} + +func (h *HighlightRequest) AddField(field string) { + if h.Fields == nil { + h.Fields = make([]string, 0, 1) + } + h.Fields = append(h.Fields, field) +} + +func (r *SearchRequest) Validate() error { + if srq, ok := r.Query.(query.ValidatableQuery); ok { + err := srq.Validate() + if err != nil { + return err + } + } + + if r.SearchAfter != nil && r.SearchBefore != nil { + return fmt.Errorf("cannot use search after and search before together") + } + + if r.SearchAfter != nil { + if r.From != 0 { + return fmt.Errorf("cannot use search after with from !=0") + } + if len(r.SearchAfter) != len(r.Sort) { + return fmt.Errorf("search after must have same size as sort order") + } + } + if r.SearchBefore != nil { + if r.From != 0 { + return fmt.Errorf("cannot use search before with from !=0") + } + if len(r.SearchBefore) != len(r.Sort) { + return fmt.Errorf("search before must have same size as sort order") + } + } + + err := r.validatePagination() + if err != nil { + return err + } + + if IsScoreFusionRequested(r) { + if r.SearchAfter != nil || r.SearchBefore != nil { + return fmt.Errorf("cannot use search after or search before with score fusion") + } + + if r.Sort != nil { + if !reflect.DeepEqual(r.Sort, AllowedFusionSort) { + return fmt.Errorf("sort must be empty or descending order of score for score fusion") + } + } + } + + err = validateKNN(r) + if err != nil { + return err + } + return r.Facets.Validate() +} + +// Validates SearchAfter/SearchBefore +func (r *SearchRequest) validatePagination() error { + var pagination []string + var afterOrBefore string + + if r.SearchAfter != nil { + pagination = r.SearchAfter + afterOrBefore = "search after" + } else if r.SearchBefore != nil { + pagination = r.SearchBefore + afterOrBefore = "search before" + } else { + return nil + } + + for i := range pagination { + switch ss := r.Sort[i].(type) { + case *search.SortGeoDistance: + _, err := strconv.ParseFloat(pagination[i], 64) + if err != nil { + return fmt.Errorf("invalid %s value for sort field '%s': '%s'. %s", afterOrBefore, ss.Field, pagination[i], err) + } + case *search.SortField: + switch ss.Type { + case search.SortFieldAsNumber: + _, err := strconv.ParseFloat(pagination[i], 64) + if err != nil { + return fmt.Errorf("invalid %s value for sort field '%s': '%s'. %s", afterOrBefore, ss.Field, pagination[i], err) + } + case search.SortFieldAsDate: + _, err := time.Parse(time.RFC3339Nano, pagination[i]) + if err != nil { + return fmt.Errorf("invalid %s value for sort field '%s': '%s'. %s", afterOrBefore, ss.Field, pagination[i], err) + } + } + } + } + + return nil +} + +// AddFacet adds a FacetRequest to this SearchRequest +func (r *SearchRequest) AddFacet(facetName string, f *FacetRequest) { + if r.Facets == nil { + r.Facets = make(FacetsRequest, 1) + } + r.Facets[facetName] = f +} + +// SortBy changes the request to use the requested sort order +// this form uses the simplified syntax with an array of strings +// each string can either be a field name +// or the magic value _id and _score which refer to the doc id and search score +// any of these values can optionally be prefixed with - to reverse the order +func (r *SearchRequest) SortBy(order []string) { + so := search.ParseSortOrderStrings(order) + r.Sort = so +} + +// SortByCustom changes the request to use the requested sort order +func (r *SearchRequest) SortByCustom(order search.SortOrder) { + r.Sort = order +} + +// SetSearchAfter sets the request to skip over hits with a sort +// value less than the provided sort after key +func (r *SearchRequest) SetSearchAfter(after []string) { + r.SearchAfter = after +} + +// SetSearchBefore sets the request to skip over hits with a sort +// value greater than the provided sort before key +func (r *SearchRequest) SetSearchBefore(before []string) { + r.SearchBefore = before +} + +// AddParams adds a RequestParams field to the search request +func (r *SearchRequest) AddParams(params RequestParams) { + r.Params = ¶ms +} + +// NewSearchRequest creates a new SearchRequest +// for the Query, using default values for all +// other search parameters. +func NewSearchRequest(q query.Query) *SearchRequest { + return NewSearchRequestOptions(q, 10, 0, false) +} + +// NewSearchRequestOptions creates a new SearchRequest +// for the Query, with the requested size, from +// and explanation search parameters. +// By default results are ordered by score, descending. +func NewSearchRequestOptions(q query.Query, size, from int, explain bool) *SearchRequest { + return &SearchRequest{ + Query: q, + Size: size, + From: from, + Explain: explain, + Sort: search.SortOrder{&search.SortScore{Desc: true}}, + } +} + +// IndexErrMap tracks errors with the name of the index where it occurred +type IndexErrMap map[string]error + +// MarshalJSON serializes the error into a string for JSON consumption +func (iem IndexErrMap) MarshalJSON() ([]byte, error) { + tmp := make(map[string]string, len(iem)) + for k, v := range iem { + tmp[k] = v.Error() + } + return util.MarshalJSON(tmp) +} + +func (iem IndexErrMap) UnmarshalJSON(data []byte) error { + var tmp map[string]string + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return err + } + for k, v := range tmp { + iem[k] = fmt.Errorf("%s", v) + } + return nil +} + +// SearchStatus is a section in the SearchResult reporting how many +// underlying indexes were queried, how many were successful/failed +// and a map of any errors that were encountered +type SearchStatus struct { + Total int `json:"total"` + Failed int `json:"failed"` + Successful int `json:"successful"` + Errors IndexErrMap `json:"errors,omitempty"` +} + +// Merge will merge together multiple SearchStatuses during a MultiSearch +func (ss *SearchStatus) Merge(other *SearchStatus) { + ss.Total += other.Total + ss.Failed += other.Failed + ss.Successful += other.Successful + if len(other.Errors) > 0 { + if ss.Errors == nil { + ss.Errors = make(map[string]error) + } + for otherIndex, otherError := range other.Errors { + ss.Errors[otherIndex] = otherError + } + } +} + +// TotalRelation constants describe the accuracy of SearchResult.Total. +const ( + // TotalRelationEq means Total is an exact count of all matching documents. + TotalRelationEq = "eq" + // TotalRelationGte means Total is a lower bound: collection stopped before + // draining every match, so the true match count is >= Total. + TotalRelationGte = "gte" +) + +// A SearchResult describes the results of executing +// a SearchRequest. +// +// Status - Whether the search was executed on the underlying indexes successfully +// or failed, and the corresponding errors. +// Request - The SearchRequest that was executed. +// Hits - The list of documents that matched the query and their corresponding +// scores, score explanation, location info and so on. +// Total - The total number of documents that matched the query. +// Cost - indicates how expensive was the query with respect to bytes read +// from the mapped index files. +// MaxScore - The maximum score seen across all document hits seen for this query. +// Took - The time taken to execute the search. +// Facets - The facet results for the search. +type SearchResult struct { + Status *SearchStatus `json:"status"` + Request *SearchRequest `json:"request,omitempty"` + Hits search.DocumentMatchCollection `json:"hits"` + Total uint64 `json:"total_hits"` + TotalRelation string `json:"total_relation"` + Cost uint64 `json:"cost"` + MaxScore float64 `json:"max_score"` + Took time.Duration `json:"took"` + Facets search.FacetResults `json:"facets"` + // special fields that are applicable only for search + // results that are obtained from a presearch + SynonymResult search.FieldTermSynonymMap `json:"synonym_result,omitempty"` + + // The following fields are applicable to BM25 preSearch + BM25Stats *search.BM25Stats `json:"bm25_stats,omitempty"` +} + +func (sr *SearchResult) Size() int { + sizeInBytes := reflectStaticSizeSearchResult + size.SizeOfPtr + + reflectStaticSizeSearchStatus + + for _, entry := range sr.Hits { + if entry != nil { + sizeInBytes += entry.Size() + } + } + + for k, v := range sr.Facets { + sizeInBytes += size.SizeOfString + len(k) + + v.Size() + } + + return sizeInBytes +} + +func (sr *SearchResult) String() string { + rv := &strings.Builder{} + if sr.Total > 0 { + switch { + case sr.Request != nil && sr.Request.Size > 0: + start := sr.Request.From + 1 + end := sr.Request.From + len(sr.Hits) + fmt.Fprintf(rv, "%d matches, showing %d through %d, took %s\n", sr.Total, start, end, sr.Took) + for i, hit := range sr.Hits { + rv = formatHit(rv, hit, start+i) + } + case sr.Request == nil: + fmt.Fprintf(rv, "%d matches, took %s\n", sr.Total, sr.Took) + for i, hit := range sr.Hits { + rv = formatHit(rv, hit, i+1) + } + default: + fmt.Fprintf(rv, "%d matches, took %s\n", sr.Total, sr.Took) + } + } else { + fmt.Fprintf(rv, "No matches\n") + } + if len(sr.Facets) > 0 { + fmt.Fprintf(rv, "Facets:\n") + for fn, f := range sr.Facets { + fmt.Fprintf(rv, "%s(%d)\n", fn, f.Total) + for _, t := range f.Terms.Terms() { + fmt.Fprintf(rv, "\t%s(%d)\n", t.Term, t.Count) + } + for _, n := range f.NumericRanges { + fmt.Fprintf(rv, "\t%s(%d)\n", n.Name, n.Count) + } + for _, d := range f.DateRanges { + fmt.Fprintf(rv, "\t%s(%d)\n", d.Name, d.Count) + } + if f.Other != 0 { + fmt.Fprintf(rv, "\tOther(%d)\n", f.Other) + } + } + } + return rv.String() +} + +// formatHit is a helper function to format a single hit in the search result for +// the String() method of SearchResult +func formatHit(rv *strings.Builder, hit *search.DocumentMatch, hitNumber int) *strings.Builder { + fmt.Fprintf(rv, "%5d. %s (%f)\n", hitNumber, hit.ID, hit.Score) + for fragmentField, fragments := range hit.Fragments { + fmt.Fprintf(rv, "\t%s\n", fragmentField) + for _, fragment := range fragments { + fmt.Fprintf(rv, "\t\t%s\n", fragment) + } + } + for otherFieldName, otherFieldValue := range hit.Fields { + if otherFieldName == NestedDocumentKey { + continue + } + if _, ok := hit.Fragments[otherFieldName]; !ok { + fmt.Fprintf(rv, "\t%s\n", otherFieldName) + fmt.Fprintf(rv, "\t\t%v\n", otherFieldValue) + } + } + // nested documents + if nested, ok := hit.Fields[NestedDocumentKey]; ok { + if list, ok := nested.([]*search.NestedDocumentMatch); ok { + fmt.Fprintf(rv, "\t%s (%d nested documents)\n", NestedDocumentKey, len(list)) + for ni, nd := range list { + fmt.Fprintf(rv, "\t\tNested #%d:\n", ni+1) + for f, frags := range nd.Fragments { + fmt.Fprintf(rv, "\t\t\t%s\n", f) + for _, frag := range frags { + fmt.Fprintf(rv, "\t\t\t\t%s\n", frag) + } + } + for f, v := range nd.Fields { + if _, ok := nd.Fragments[f]; !ok { + fmt.Fprintf(rv, "\t\t\t%s\n", f) + fmt.Fprintf(rv, "\t\t\t\t%v\n", v) + } + } + } + } + } + if len(hit.DecodedSort) > 0 { + fmt.Fprintf(rv, "\t_sort: [") + for k, v := range hit.DecodedSort { + if k > 0 { + fmt.Fprintf(rv, ", ") + } + fmt.Fprintf(rv, "%v", v) + } + fmt.Fprintf(rv, "]\n") + } + return rv +} + +// Merge will merge together multiple SearchResults during a MultiSearch +func (sr *SearchResult) Merge(other *SearchResult) { + sr.Status.Merge(other.Status) + sr.Hits = append(sr.Hits, other.Hits...) + sr.Total += other.Total + if other.TotalRelation == TotalRelationGte { + sr.TotalRelation = TotalRelationGte + } + sr.Cost += other.Cost + if other.MaxScore > sr.MaxScore { + sr.MaxScore = other.MaxScore + } + if sr.Facets == nil && len(other.Facets) != 0 { + sr.Facets = other.Facets + return + } + + sr.Facets.Merge(other.Facets) +} + +// MemoryNeededForSearchResult is an exported helper function to determine the RAM +// needed to accommodate the results for a given search request. +func MemoryNeededForSearchResult(req *SearchRequest) uint64 { + if req == nil { + return 0 + } + + numDocMatches := req.Size + req.From + if req.Size+req.From > collector.PreAllocSizeSkipCap { + numDocMatches = collector.PreAllocSizeSkipCap + } + + estimate := 0 + + // overhead from the SearchResult structure + var sr SearchResult + estimate += sr.Size() + + var dm search.DocumentMatch + sizeOfDocumentMatch := dm.Size() + + // overhead from results + estimate += numDocMatches * sizeOfDocumentMatch + + // overhead from facet results + if req.Facets != nil { + var fr search.FacetResult + estimate += len(req.Facets) * fr.Size() + } + + // overhead from fields, highlighting + var d document.Document + if len(req.Fields) > 0 || req.Highlight != nil { + numDocsApplicable := req.Size + if numDocsApplicable > collector.PreAllocSizeSkipCap { + numDocsApplicable = collector.PreAllocSizeSkipCap + } + estimate += numDocsApplicable * d.Size() + } + + return uint64(estimate) +} + +// SetSortFunc sets the sort implementation to use when sorting hits. +// +// SearchRequests can specify a custom sort implementation to meet +// their needs. For instance, by specifying a parallel sort +// that uses all available cores. +func (r *SearchRequest) SetSortFunc(s func(sort.Interface)) { + r.sortFunc = s +} + +// SortFunc returns the sort implementation to use when sorting hits. +// Defaults to sort.Sort. +func (r *SearchRequest) SortFunc() func(data sort.Interface) { + if r.sortFunc != nil { + return r.sortFunc + } + + return sort.Sort +} + +func isMatchNoneQuery(q query.Query) bool { + _, ok := q.(*query.MatchNoneQuery) + return ok +} + +func isMatchAllQuery(q query.Query) bool { + _, ok := q.(*query.MatchAllQuery) + return ok +} + +// Checks if the request is hybrid search. Currently supports: RRF, RSF. +func IsScoreFusionRequested(req *SearchRequest) bool { + switch req.Score { + case ScoreRRF, ScoreRSF: + return true + default: + return false + } +} + +// Additional parameters in the search request. Currently only being +// used for score fusion parameters. +type RequestParams struct { + ScoreRankConstant int `json:"score_rank_constant,omitempty"` + ScoreWindowSize int `json:"score_window_size,omitempty"` +} + +func NewDefaultParams(from, size int) *RequestParams { + return &RequestParams{ + ScoreRankConstant: DefaultScoreRankConstant, + ScoreWindowSize: from + size, + } +} + +func (p *RequestParams) UnmarshalJSON(input []byte) error { + var temp struct { + ScoreRankConstant *int `json:"score_rank_constant,omitempty"` + ScoreWindowSize *int `json:"score_window_size,omitempty"` + } + + if err := util.UnmarshalJSON(input, &temp); err != nil { + return err + } + + if temp.ScoreRankConstant != nil { + p.ScoreRankConstant = *temp.ScoreRankConstant + } + + if temp.ScoreWindowSize != nil { + p.ScoreWindowSize = *temp.ScoreWindowSize + } + + return nil +} + +func (p *RequestParams) Validate(size int) error { + if p.ScoreWindowSize < 1 { + return fmt.Errorf("score window size must be greater than 0") + } else if p.ScoreWindowSize < size { + return fmt.Errorf("score window size must be greater than or equal to Size (%d)", size) + } + + return nil +} + +func ParseParams(r *SearchRequest, input []byte) (*RequestParams, error) { + params := NewDefaultParams(r.From, r.Size) + if len(input) == 0 { + return params, nil + } + + err := util.UnmarshalJSON(input, params) + if err != nil { + return nil, err + } + + // validate params + err = params.Validate(r.Size) + if err != nil { + return nil, err + } + + return params, nil +} + +// OptionalRawMessage is a wrapper around json.RawMessage that treats empty or `null` JSON as nil. +type OptionalRawMessage json.RawMessage + +func (n *OptionalRawMessage) UnmarshalJSON(data []byte) error { + if len(data) == 0 || bytes.Equal(data, []byte("null")) { + *n = nil + return nil + } + *n = slices.Clone(data) + return nil +} + +func (n OptionalRawMessage) MarshalJSON() ([]byte, error) { + if len(n) == 0 { + return []byte("null"), nil + } + return n, nil +} diff --git a/search/collector.go b/search/collector.go new file mode 100644 index 0000000..e81219e --- /dev/null +++ b/search/collector.go @@ -0,0 +1,58 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import ( + "context" + "time" + + index "github.com/blevesearch/bleve_index_api" +) + +type Collector interface { + Collect(ctx context.Context, searcher Searcher, reader index.IndexReader) error + Results() DocumentMatchCollection + Total() uint64 + MaxScore() float64 + Took() time.Duration + SetFacetsBuilder(facetsBuilder *FacetsBuilder) + FacetResults() FacetResults +} + +// DocumentMatchHandler is the type of document match callback +// bleve will invoke during the search. +// Eventually, bleve will indicate the completion of an ongoing search, +// by passing a nil value for the document match callback. +// The application should take a copy of the hit/documentMatch +// if it wish to own it or need prolonged access to it. +type DocumentMatchHandler func(hit *DocumentMatch) error + +type MakeDocumentMatchHandlerKeyType string + +var MakeDocumentMatchHandlerKey = MakeDocumentMatchHandlerKeyType( + "MakeDocumentMatchHandlerKey") + +var MakeKNNDocumentMatchHandlerKey = MakeDocumentMatchHandlerKeyType( + "MakeKNNDocumentMatchHandlerKey") + +// MakeDocumentMatchHandler is an optional DocumentMatchHandler +// builder function which the applications can pass to bleve. +// These builder methods gives a DocumentMatchHandler function +// to bleve, which it will invoke on every document matches. +type MakeDocumentMatchHandler func(ctx *SearchContext) ( + callback DocumentMatchHandler, loadID bool, err error) + +type MakeKNNDocumentMatchHandler func(ctx *SearchContext) ( + callback DocumentMatchHandler, err error) diff --git a/search/collector/bench_test.go b/search/collector/bench_test.go new file mode 100644 index 0000000..3edf369 --- /dev/null +++ b/search/collector/bench_test.go @@ -0,0 +1,50 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + "math/rand" + "strconv" + "testing" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +type createCollector func() search.Collector + +func benchHelper(numOfMatches int, cc createCollector, b *testing.B) { + matches := make([]*search.DocumentMatch, 0, numOfMatches) + for i := 0; i < numOfMatches; i++ { + matches = append(matches, &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID(strconv.Itoa(i)), + Score: rand.Float64(), + }) + } + + b.ResetTimer() + + for run := 0; run < b.N; run++ { + searcher := &stubSearcher{ + matches: matches, + } + collector := cc() + err := collector.Collect(context.Background(), searcher, &stubReader{}) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/search/collector/eligible.go b/search/collector/eligible.go new file mode 100644 index 0000000..49e0448 --- /dev/null +++ b/search/collector/eligible.go @@ -0,0 +1,172 @@ +// Copyright (c) 2024 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package collector + +import ( + "context" + "fmt" + "time" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +type EligibleCollector struct { + size int + total uint64 + took time.Duration + eligibleSelector index.EligibleDocumentSelector +} + +func NewEligibleCollector(size int) *EligibleCollector { + return newEligibleCollector(size) +} + +func newEligibleCollector(size int) *EligibleCollector { + // No sort order & skip always 0 since this is only to filter eligible docs. + ec := &EligibleCollector{ + size: size, + } + return ec +} + +func makeEligibleDocumentMatchHandler(ctx *search.SearchContext, reader index.IndexReader) (search.DocumentMatchHandler, error) { + if ec, ok := ctx.Collector.(*EligibleCollector); ok { + if vr, ok := reader.(index.VectorIndexReader); ok { + // create a new eligible document selector to add eligible document matches + ec.eligibleSelector = vr.NewEligibleDocumentSelector() + // return a document match handler that adds eligible document matches + // to the eligible document selector + return func(d *search.DocumentMatch) error { + if d == nil { + return nil + } + err := ec.eligibleSelector.AddEligibleDocumentMatch(d.IndexInternalID) + if err != nil { + return err + } + // recycle the DocumentMatch + ctx.DocumentMatchPool.Put(d) + return nil + }, nil + } + return nil, fmt.Errorf("reader is not a VectorIndexReader") + } + + return nil, fmt.Errorf("eligiblity collector not available") +} + +func (ec *EligibleCollector) Collect(ctx context.Context, searcher search.Searcher, reader index.IndexReader) error { + startTime := time.Now() + var err error + var next *search.DocumentMatch + + backingSize := ec.size + if backingSize > PreAllocSizeSkipCap { + backingSize = PreAllocSizeSkipCap + 1 + } + searchContext := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(backingSize+searcher.DocumentMatchPoolSize(), 0), + Collector: ec, + IndexReader: reader, + } + + dmHandler, err := makeEligibleDocumentMatchHandler(searchContext, reader) + if err != nil { + return err + } + select { + case <-ctx.Done(): + search.RecordSearchCost(ctx, search.AbortM, 0) + return ctx.Err() + default: + next, err = searcher.Next(searchContext) + } + for err == nil && next != nil { + if ec.total%CheckDoneEvery == 0 { + select { + case <-ctx.Done(): + search.RecordSearchCost(ctx, search.AbortM, 0) + return ctx.Err() + default: + } + } + ec.total++ + + err = dmHandler(next) + if err != nil { + break + } + + next, err = searcher.Next(searchContext) + } + if err != nil { + return err + } + + // help finalize/flush the results in case + // of custom document match handlers. + err = dmHandler(nil) + if err != nil { + return err + } + + // compute search duration + ec.took = time.Since(startTime) + + return nil +} + +// The eligible collector does not return any document matches and hence +// this method is a dummy method returning nil, to conform to the +// search.Collector interface. +func (ec *EligibleCollector) Results() search.DocumentMatchCollection { + return nil +} + +// EligibleSelector returns the eligible document selector, which can be used +// to retrieve the list of eligible documents from this collector. +// If the collector has no results, it returns nil. +func (ec *EligibleCollector) EligibleSelector() index.EligibleDocumentSelector { + if ec.total == 0 { + return nil + } + return ec.eligibleSelector +} + +func (ec *EligibleCollector) Total() uint64 { + return ec.total +} + +// No concept of scoring in the eligible collector. +func (ec *EligibleCollector) MaxScore() float64 { + return 0 +} + +func (ec *EligibleCollector) Took() time.Duration { + return ec.took +} + +func (ec *EligibleCollector) SetFacetsBuilder(facetsBuilder *search.FacetsBuilder) { + // facet unsupported for pre-filtering in KNN search +} + +func (ec *EligibleCollector) FacetResults() search.FacetResults { + // facet unsupported for pre-filtering in KNN search + return nil +} diff --git a/search/collector/heap.go b/search/collector/heap.go new file mode 100644 index 0000000..6a547bb --- /dev/null +++ b/search/collector/heap.go @@ -0,0 +1,140 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "sort" + + "github.com/blevesearch/bleve/v2/search" +) + +// collectStoreHeap is a min-heap of DocumentMatches where the root (heap[0]) +// is always the *worst* result (lowest score / highest sort key). Popping the +// root discards the worst element, so AddNotExceedingSize keeps the best k +// documents at all times. +// +// Implemented as a ternary heap (3 children per node) instead of the standard +// library's binary heap. Height is log₃(n) vs log₂(n), so siftDown is +// shallower and fetches more children per cache line. +type collectStoreHeap struct { + heap search.DocumentMatchCollection + compare collectorCompare +} + +func newStoreHeap(capacity int, compare collectorCompare) *collectStoreHeap { + return &collectStoreHeap{ + heap: make(search.DocumentMatchCollection, 0, capacity), + compare: compare, + } +} + +func (c *collectStoreHeap) AddNotExceedingSize(doc *search.DocumentMatch, + size int) *search.DocumentMatch { + c.add(doc) + if len(c.heap) > size { + return c.removeLast() + } + return nil +} + +func (c *collectStoreHeap) add(doc *search.DocumentMatch) { + c.heap = append(c.heap, doc) + c.siftUp(len(c.heap) - 1) +} + +func (c *collectStoreHeap) removeLast() *search.DocumentMatch { + n := len(c.heap) + c.heap[0], c.heap[n-1] = c.heap[n-1], c.heap[0] + result := c.heap[n-1] + c.heap = c.heap[:n-1] + if len(c.heap) > 0 { + c.siftDown(0) + } + return result +} + +// siftUp restores the heap invariant after appending at index i. +// Moves element at i toward the root while it is less than (worse than) its parent. +func (c *collectStoreHeap) siftUp(i int) { + h := c.heap + for i > 0 { + parent := (i - 1) / 3 + if c.compare(h[i], h[parent]) > 0 { // h[i] is worse → should be closer to root + h[i], h[parent] = h[parent], h[i] + i = parent + } else { + break + } + } +} + +// siftDown restores the heap invariant after replacing the root. +// Moves element at i toward the leaves while any child is worse (less) than it. +func (c *collectStoreHeap) siftDown(i int) { + h := c.heap + n := len(h) + for { + first := 3*i + 1 + if first >= n { + break + } + // Find the worst (least) child among up to three children. + worst := first + if s := first + 1; s < n && c.compare(h[s], h[worst]) > 0 { + worst = s + } + if t := first + 2; t < n && c.compare(h[t], h[worst]) > 0 { + worst = t + } + if c.compare(h[worst], h[i]) > 0 { // worst child is worse than current → swap + h[i], h[worst] = h[worst], h[i] + i = worst + } else { + break + } + } +} + +func (c *collectStoreHeap) Final(skip int, fixup collectorFixup) (search.DocumentMatchCollection, error) { + count := len(c.heap) + size := count - skip + if size <= 0 { + return make(search.DocumentMatchCollection, 0), nil + } + // Sort in-place so heap[0] = best doc (compare < 0 means "better"). + // pdqsort (sort.Slice) has much better cache behavior than repeated + // removeLast (heapsort) because it accesses the array sequentially, + // avoiding the scattered pointer dereferences heapsort pays at each level. + sort.Slice(c.heap, func(i, j int) bool { + return c.compare(c.heap[i], c.heap[j]) < 0 + }) + rv := make(search.DocumentMatchCollection, size) + for i := 0; i < size; i++ { + doc := c.heap[skip+i] + rv[i] = doc + if err := fixup(doc); err != nil { + return nil, err + } + } + return rv, nil +} + +func (c *collectStoreHeap) Len() int { + return len(c.heap) +} + +func (c *collectStoreHeap) Internal() search.DocumentMatchCollection { + return c.heap +} diff --git a/search/collector/heap_test.go b/search/collector/heap_test.go new file mode 100644 index 0000000..31f1c47 --- /dev/null +++ b/search/collector/heap_test.go @@ -0,0 +1,202 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "testing" + + "github.com/blevesearch/bleve/v2/search" +) + +// scoreDesc is a collectorCompare for score-descending order. +// Returns positive when a.Score < b.Score (a is worse — lower score — and +// should sit closer to the root of the min-heap). +func scoreDesc(a, b *search.DocumentMatch) int { + if a.Score < b.Score { + return 1 + } + if a.Score > b.Score { + return -1 + } + return 0 +} + +func makeScoreDoc(score float64) *search.DocumentMatch { + return &search.DocumentMatch{Score: score} +} + +// checkTernaryInvariant verifies the ternary min-heap property: for every node +// i the parent (at (i-1)/3) must be as bad or worse than the child, i.e. +// compare(parent, child) >= 0. +func checkTernaryInvariant(t *testing.T, h *collectStoreHeap) { + t.Helper() + for i := 1; i < len(h.heap); i++ { + parent := (i - 1) / 3 + if h.compare(h.heap[parent], h.heap[i]) < 0 { + t.Errorf("ternary heap invariant violated at index %d (score=%.2f): parent %d (score=%.2f) is better, should be at least as bad", + i, h.heap[i].Score, parent, h.heap[parent].Score) + } + } +} + +// TestTernaryHeapInvariantAfterInserts inserts scores in several orderings and +// verifies the ternary invariant holds after every insertion. +func TestTernaryHeapInvariantAfterInserts(t *testing.T) { + for _, scores := range [][]float64{ + {5, 3, 8, 1, 7, 2, 9, 4, 6, 10}, // unsorted + {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, // ascending + {10, 9, 8, 7, 6, 5, 4, 3, 2, 1}, // descending + {3, 3, 3, 3}, // all equal + } { + h := newStoreHeap(len(scores), scoreDesc) + for _, s := range scores { + h.add(makeScoreDoc(s)) + checkTernaryInvariant(t, h) + } + // Root must be the minimum score in the heap. + min := scores[0] + for _, s := range scores[1:] { + if s < min { + min = s + } + } + if h.heap[0].Score != min { + t.Errorf("root score %.2f, want min %.2f (scores %v)", h.heap[0].Score, min, scores) + } + } +} + +// TestTernaryHeapRemoveLastAscending verifies that successive removeLast calls +// return elements in ascending score order (worst first = heap-sort property). +func TestTernaryHeapRemoveLastAscending(t *testing.T) { + scores := []float64{5, 3, 8, 1, 7, 2, 9, 4, 6, 10} + h := newStoreHeap(len(scores), scoreDesc) + for _, s := range scores { + h.add(makeScoreDoc(s)) + } + + prev := -1.0 + for len(h.heap) > 0 { + doc := h.removeLast() + if doc.Score < prev { + t.Errorf("removeLast out of order: got %.2f after %.2f", doc.Score, prev) + } + prev = doc.Score + checkTernaryInvariant(t, h) + } +} + +// TestTernaryHeapAddNotExceedingSize checks that size enforcement works: +// adding more than k docs keeps the best k and evicts the rest. +func TestTernaryHeapAddNotExceedingSize(t *testing.T) { + const k = 5 + h := newStoreHeap(k, scoreDesc) + for i := 1; i <= 10; i++ { + evicted := h.AddNotExceedingSize(makeScoreDoc(float64(i)), k) + if i <= k { + if evicted != nil { + t.Errorf("insert %d: expected no eviction, got score %.2f", i, evicted.Score) + } + } else { + if evicted == nil { + t.Errorf("insert %d: expected eviction", i) + } + } + checkTernaryInvariant(t, h) + } + if h.Len() != k { + t.Errorf("heap size %d, want %d", h.Len(), k) + } + // The heap should hold the best k scores (6..10); root is their minimum. + if h.heap[0].Score != 6.0 { + t.Errorf("root score %.2f, want 6.00 (worst of top-%d)", h.heap[0].Score, k) + } +} + +// TestTernaryHeapFinalOrder verifies that Final(0, ...) returns results in +// descending score order (best first). +func TestTernaryHeapFinalOrder(t *testing.T) { + scores := []float64{3, 1, 4, 1, 5, 9, 2, 6, 5, 3} + h := newStoreHeap(len(scores), scoreDesc) + for _, s := range scores { + h.add(makeScoreDoc(s)) + } + + fixup := func(*search.DocumentMatch) error { return nil } + result, err := h.Final(0, fixup) + if err != nil { + t.Fatal(err) + } + if len(result) != len(scores) { + t.Fatalf("len(result)=%d, want %d", len(result), len(scores)) + } + for i := 1; i < len(result); i++ { + if result[i].Score > result[i-1].Score { + t.Errorf("result[%d]=%.2f > result[%d]=%.2f (not sorted descending)", + i, result[i].Score, i-1, result[i-1].Score) + } + } +} + +// TestTernaryHeapFinalWithSkip verifies that Final(skip, ...) skips the skip +// worst results and returns the rest best-first. +func TestTernaryHeapFinalWithSkip(t *testing.T) { + // Heap holds 1..5; skip=2 should return scores [3, 2, 1] (skipping 4 and 5). + scores := []float64{1, 2, 3, 4, 5} + h := newStoreHeap(len(scores), scoreDesc) + for _, s := range scores { + h.add(makeScoreDoc(s)) + } + + fixup := func(*search.DocumentMatch) error { return nil } + result, err := h.Final(2, fixup) + if err != nil { + t.Fatal(err) + } + if len(result) != 3 { + t.Fatalf("len(result)=%d, want 3", len(result)) + } + want := []float64{3, 2, 1} + for i, w := range want { + if result[i].Score != w { + t.Errorf("result[%d]=%.2f, want %.2f", i, result[i].Score, w) + } + } +} + +// TestTernaryHeapLargeN exercises multiple levels of siftDown (n > 3^3 = 27). +func TestTernaryHeapLargeN(t *testing.T) { + const n = 200 + h := newStoreHeap(n, scoreDesc) + for i := n; i >= 1; i-- { // insert descending to stress siftUp + h.add(makeScoreDoc(float64(i))) + checkTernaryInvariant(t, h) + } + + prev := -1.0 + count := 0 + for len(h.heap) > 0 { + doc := h.removeLast() + if doc.Score < prev { + t.Errorf("removeLast out of order at position %d: %.2f after %.2f", count, doc.Score, prev) + } + prev = doc.Score + count++ + checkTernaryInvariant(t, h) + } + if count != n { + t.Errorf("extracted %d elements, want %d", count, n) + } +} diff --git a/search/collector/knn.go b/search/collector/knn.go new file mode 100644 index 0000000..465bf69 --- /dev/null +++ b/search/collector/knn.go @@ -0,0 +1,262 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package collector + +import ( + "context" + "time" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +type collectStoreKNN struct { + internalHeaps []collectorStore + kValues []int64 + allHits map[*search.DocumentMatch]struct{} + ejectedDocs map[*search.DocumentMatch]struct{} +} + +func newStoreKNN(internalHeaps []collectorStore, kValues []int64) *collectStoreKNN { + return &collectStoreKNN{ + internalHeaps: internalHeaps, + kValues: kValues, + ejectedDocs: make(map[*search.DocumentMatch]struct{}), + allHits: make(map[*search.DocumentMatch]struct{}), + } +} + +// Adds a document to the collector store and returns the documents that were ejected +// from the store. The documents that were ejected from the store are the ones that +// were not in the top K documents for any of the heaps. +// These document are put back into the pool document match pool in the KNN Collector. +func (c *collectStoreKNN) AddDocument(doc *search.DocumentMatch) []*search.DocumentMatch { + for heapIdx := 0; heapIdx < len(c.internalHeaps); heapIdx++ { + if _, ok := doc.ScoreBreakdown[heapIdx]; !ok { + continue + } + ejectedDoc := c.internalHeaps[heapIdx].AddNotExceedingSize(doc, int(c.kValues[heapIdx])) + if ejectedDoc != nil { + delete(ejectedDoc.ScoreBreakdown, heapIdx) + c.ejectedDocs[ejectedDoc] = struct{}{} + } + } + var rv []*search.DocumentMatch + for doc := range c.ejectedDocs { + if len(doc.ScoreBreakdown) == 0 { + rv = append(rv, doc) + } + // clear out the ejectedDocs map to reuse it in the next AddDocument call + delete(c.ejectedDocs, doc) + } + return rv +} + +func (c *collectStoreKNN) Final(fixup collectorFixup) (search.DocumentMatchCollection, error) { + for _, heap := range c.internalHeaps { + for _, doc := range heap.Internal() { + // duplicates may be present across the internal heaps + // meaning the same document match may be in the top K + // for multiple KNN queries. + c.allHits[doc] = struct{}{} + } + } + size := len(c.allHits) + if size <= 0 { + return make(search.DocumentMatchCollection, 0), nil + } + rv := make(search.DocumentMatchCollection, size) + i := 0 + for doc := range c.allHits { + if fixup != nil { + err := fixup(doc) + if err != nil { + return nil, err + } + } + rv[i] = doc + i++ + } + return rv, nil +} + +func MakeKNNDocMatchHandler(ctx *search.SearchContext) (search.DocumentMatchHandler, error) { + var hc *KNNCollector + var ok bool + if hc, ok = ctx.Collector.(*KNNCollector); ok { + return func(d *search.DocumentMatch) error { + if d == nil { + return nil + } + toRelease := hc.knnStore.AddDocument(d) + for _, doc := range toRelease { + ctx.DocumentMatchPool.Put(doc) + } + return nil + }, nil + } + return nil, nil +} + +func GetNewKNNCollectorStore(kArray []int64) *collectStoreKNN { + internalHeaps := make([]collectorStore, len(kArray)) + for knnIdx, k := range kArray { + // TODO - Check if the datatype of k can be made into an int instead of int64 + idx := knnIdx + internalHeaps[idx] = getOptimalCollectorStore(int(k), 0, func(i, j *search.DocumentMatch) int { + if i.ScoreBreakdown[idx] < j.ScoreBreakdown[idx] { + return 1 + } + return -1 + }) + } + return newStoreKNN(internalHeaps, kArray) +} + +// implements Collector interface +type KNNCollector struct { + knnStore *collectStoreKNN + size int + total uint64 + took time.Duration + results search.DocumentMatchCollection + maxScore float64 +} + +func NewKNNCollector(kArray []int64, size int64) *KNNCollector { + return &KNNCollector{ + knnStore: GetNewKNNCollectorStore(kArray), + size: int(size), + } +} + +func (hc *KNNCollector) Collect(ctx context.Context, searcher search.Searcher, reader index.IndexReader) error { + startTime := time.Now() + var err error + var next *search.DocumentMatch + + // pre-allocate enough space in the DocumentMatchPool + // unless the sum of K is too large, then cap it + // everything should still work, just allocates DocumentMatches on demand + backingSize := hc.size + if backingSize > PreAllocSizeSkipCap { + backingSize = PreAllocSizeSkipCap + 1 + } + searchContext := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(backingSize+searcher.DocumentMatchPoolSize(), 0), + Collector: hc, + IndexReader: reader, + } + + dmHandlerMakerKNN := MakeKNNDocMatchHandler + if cv := ctx.Value(search.MakeKNNDocumentMatchHandlerKey); cv != nil { + dmHandlerMakerKNN = cv.(search.MakeKNNDocumentMatchHandler) + } + // use the application given builder for making the custom document match + // handler and perform callbacks/invocations on the newly made handler. + dmHandler, err := dmHandlerMakerKNN(searchContext) + if err != nil { + return err + } + select { + case <-ctx.Done(): + search.RecordSearchCost(ctx, search.AbortM, 0) + return ctx.Err() + default: + next, err = searcher.Next(searchContext) + } + for err == nil && next != nil { + if hc.total%CheckDoneEvery == 0 { + select { + case <-ctx.Done(): + search.RecordSearchCost(ctx, search.AbortM, 0) + return ctx.Err() + default: + } + } + hc.total++ + + err = dmHandler(next) + if err != nil { + break + } + + next, err = searcher.Next(searchContext) + } + if err != nil { + return err + } + + // help finalize/flush the results in case + // of custom document match handlers. + err = dmHandler(nil) + if err != nil { + return err + } + + // compute search duration + hc.took = time.Since(startTime) + + // finalize actual results + err = hc.finalizeResults(reader) + if err != nil { + return err + } + return nil +} + +func (hc *KNNCollector) finalizeResults(r index.IndexReader) error { + var err error + hc.results, err = hc.knnStore.Final(func(doc *search.DocumentMatch) error { + if doc.ID == "" { + // look up the id since we need it for lookup + var err error + doc.ID, err = r.ExternalID(doc.IndexInternalID) + if err != nil { + return err + } + } + return nil + }) + return err +} + +func (hc *KNNCollector) Results() search.DocumentMatchCollection { + return hc.results +} + +func (hc *KNNCollector) Total() uint64 { + return hc.total +} + +func (hc *KNNCollector) MaxScore() float64 { + return hc.maxScore +} + +func (hc *KNNCollector) Took() time.Duration { + return hc.took +} + +func (hc *KNNCollector) SetFacetsBuilder(facetsBuilder *search.FacetsBuilder) { + // facet unsupported for vector search +} + +func (hc *KNNCollector) FacetResults() search.FacetResults { + // facet unsupported for vector search + return nil +} diff --git a/search/collector/nested.go b/search/collector/nested.go new file mode 100644 index 0000000..ce2f790 --- /dev/null +++ b/search/collector/nested.go @@ -0,0 +1,103 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +type collectStoreNested struct { + // descAdder is used to customize how descendants are merged into their parent + descAdder search.DescendantAdderCallbackFn + // nested reader to retrieve ancestor information + nr index.NestedReader + // the current root document match being built + currRoot *search.DocumentMatch + // the ancestor ID of the current root document being built + currRootAncestorID index.AncestorID + // prealloc slice for ancestor IDs + ancestors []index.AncestorID +} + +func newStoreNested(nr index.NestedReader, descAdder search.DescendantAdderCallbackFn) *collectStoreNested { + rv := &collectStoreNested{ + descAdder: descAdder, + nr: nr, + } + return rv +} + +// ProcessNestedDocument adds a document to the nested store, merging it into its root document +// as needed. If the returned DocumentMatch is nil, the incoming doc has been merged +// into its parent and should not be processed further. If the returned DocumentMatch +// is non-nil, it represents a complete root document that should be processed further. +// NOTE: This implementation assumes that documents are added in increasing order of their internal IDs +// which is guaranteed by all searchers in bleve. +func (c *collectStoreNested) ProcessNestedDocument(ctx *search.SearchContext, doc *search.DocumentMatch) (*search.DocumentMatch, error) { + // find ancestors for the doc + var err error + c.ancestors, err = c.nr.Ancestors(doc.IndexInternalID, c.ancestors[:0]) + if err != nil { + return nil, err + } + if len(c.ancestors) == 0 { + // should not happen, every doc should have at least itself as ancestor + return nil, nil + } + // root docID is the last ancestor + rootID := c.ancestors[len(c.ancestors)-1] + // check if there is an interim root already and if the incoming doc belongs to it + if c.currRoot != nil && c.currRootAncestorID.Equals(rootID) { + // there is an interim root already, and the incoming doc belongs to it + if err := c.descAdder(c.currRoot, doc); err != nil { + return nil, err + } + // recycle the child document now that it's merged into the interim root + ctx.DocumentMatchPool.Put(doc) + return nil, nil + } + // completedRoot is the root document match to return, if any + var completedRoot *search.DocumentMatch + if c.currRoot != nil { + // we have an existing interim root, return it for processing + completedRoot = c.currRoot + } + // no interim root for now so either we have a root document incoming + // or we have a child doc and need to create an interim root + if len(c.ancestors) == 1 { + // incoming doc is the root itself + c.currRoot = doc + c.currRootAncestorID = rootID + return completedRoot, nil + } + // this is a child doc, create interim root + newDM := ctx.DocumentMatchPool.Get() + newDM.IndexInternalID = rootID.ToIndexInternalID(newDM.IndexInternalID) + // merge the incoming doc into the new interim root + c.currRoot = newDM + c.currRootAncestorID = rootID + if err := c.descAdder(c.currRoot, doc); err != nil { + return nil, err + } + // recycle the child document now that it's merged into the interim root + ctx.DocumentMatchPool.Put(doc) + return completedRoot, nil +} + +// Current returns the current interim root document match being built, if any +func (c *collectStoreNested) Current() *search.DocumentMatch { + return c.currRoot +} diff --git a/search/collector/search_test.go b/search/collector/search_test.go new file mode 100644 index 0000000..1f6f882 --- /dev/null +++ b/search/collector/search_test.go @@ -0,0 +1,187 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + "reflect" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +type stubSearcher struct { + index int + matches []*search.DocumentMatch +} + +func (ss *stubSearcher) SetBytesRead(val uint64) { + +} + +func (ss *stubSearcher) BytesRead() uint64 { + return 0 +} + +func (ss *stubSearcher) Size() int { + sizeInBytes := int(reflect.TypeOf(*ss).Size()) + + for _, entry := range ss.matches { + if entry != nil { + sizeInBytes += entry.Size() + } + } + + return sizeInBytes +} + +func (ss *stubSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + if ss.index < len(ss.matches) { + rv := ctx.DocumentMatchPool.Get() + rv.IndexInternalID = ss.matches[ss.index].IndexInternalID + rv.Score = ss.matches[ss.index].Score + ss.index++ + return rv, nil + } + return nil, nil +} + +func (ss *stubSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + + for ss.index < len(ss.matches) && ss.matches[ss.index].IndexInternalID.Compare(ID) < 0 { + ss.index++ + } + if ss.index < len(ss.matches) { + rv := ctx.DocumentMatchPool.Get() + rv.IndexInternalID = ss.matches[ss.index].IndexInternalID + rv.Score = ss.matches[ss.index].Score + ss.index++ + return rv, nil + } + return nil, nil +} + +func (ss *stubSearcher) Close() error { + return nil +} + +func (ss *stubSearcher) Weight() float64 { + return 0.0 +} + +func (ss *stubSearcher) SetQueryNorm(float64) { +} + +func (ss *stubSearcher) Count() uint64 { + return uint64(len(ss.matches)) +} + +func (ss *stubSearcher) Min() int { + return 0 +} + +func (ss *stubSearcher) DocumentMatchPoolSize() int { + return 0 +} + +type stubReader struct{} + +func (sr *stubReader) Size() int { + return 0 +} + +func (sr *stubReader) TermFieldReader(ctx context.Context, term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (index.TermFieldReader, error) { + return nil, nil +} + +func (sr *stubReader) DocIDReaderAll() (index.DocIDReader, error) { + return nil, nil +} + +func (sr *stubReader) DocIDReaderOnly(ids []string) (index.DocIDReader, error) { + return nil, nil +} + +func (sr *stubReader) FieldDict(field string) (index.FieldDict, error) { + return nil, nil +} + +func (sr *stubReader) FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) { + return nil, nil +} + +func (sr *stubReader) FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) { + return nil, nil +} + +func (sr *stubReader) Document(id string) (index.Document, error) { + return nil, nil +} + +func (sr *stubReader) DocumentVisitFieldTerms(id index.IndexInternalID, fields []string, visitor index.DocValueVisitor) error { + return nil +} + +func (sr *stubReader) Fields() ([]string, error) { + return nil, nil +} + +func (sr *stubReader) GetInternal(key []byte) ([]byte, error) { + return nil, nil +} + +func (sr *stubReader) DocCount() (uint64, error) { + return 0, nil +} + +func (sr *stubReader) ExternalID(id index.IndexInternalID) (string, error) { + return string(id), nil +} + +func (sr *stubReader) InternalID(id string) (index.IndexInternalID, error) { + return []byte(id), nil +} + +func (sr *stubReader) DumpAll() chan interface{} { + return nil +} + +func (sr *stubReader) DumpDoc(id string) chan interface{} { + return nil +} + +func (sr *stubReader) DumpFields() chan interface{} { + return nil +} + +func (sr *stubReader) Close() error { + return nil +} + +func (sr *stubReader) DocValueReader(fields []string) (index.DocValueReader, error) { + return &DocValueReader{i: sr, fields: fields}, nil +} + +type DocValueReader struct { + i *stubReader + fields []string +} + +func (dvr *DocValueReader) VisitDocValues(id index.IndexInternalID, visitor index.DocValueVisitor) error { + return dvr.i.DocumentVisitFieldTerms(id, dvr.fields, visitor) +} +func (dvr *DocValueReader) BytesRead() uint64 { + return 0 +} diff --git a/search/collector/slice.go b/search/collector/slice.go new file mode 100644 index 0000000..6120921 --- /dev/null +++ b/search/collector/slice.go @@ -0,0 +1,83 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "github.com/blevesearch/bleve/v2/search" +) + +type collectStoreSlice struct { + slice search.DocumentMatchCollection + compare collectorCompare +} + +func newStoreSlice(capacity int, compare collectorCompare) *collectStoreSlice { + rv := &collectStoreSlice{ + slice: make(search.DocumentMatchCollection, 0, capacity), + compare: compare, + } + return rv +} + +func (c *collectStoreSlice) AddNotExceedingSize(doc *search.DocumentMatch, + size int) *search.DocumentMatch { + c.add(doc) + if c.len() > size { + return c.removeLast() + } + return nil +} + +func (c *collectStoreSlice) add(doc *search.DocumentMatch) { + // find where to insert, starting at end (lowest) + i := len(c.slice) + for ; i > 0; i-- { + cmp := c.compare(doc, c.slice[i-1]) + if cmp >= 0 { + break + } + } + // insert at i + c.slice = append(c.slice, nil) + copy(c.slice[i+1:], c.slice[i:]) + c.slice[i] = doc +} + +func (c *collectStoreSlice) removeLast() *search.DocumentMatch { + var rv *search.DocumentMatch + rv, c.slice = c.slice[len(c.slice)-1], c.slice[:len(c.slice)-1] + return rv +} + +func (c *collectStoreSlice) Final(skip int, fixup collectorFixup) (search.DocumentMatchCollection, error) { + for i := skip; i < len(c.slice); i++ { + err := fixup(c.slice[i]) + if err != nil { + return nil, err + } + } + if skip <= len(c.slice) { + return c.slice[skip:], nil + } + return search.DocumentMatchCollection{}, nil +} + +func (c *collectStoreSlice) Internal() search.DocumentMatchCollection { + return c.slice +} + +func (c *collectStoreSlice) len() int { + return len(c.slice) +} diff --git a/search/collector/topn.go b/search/collector/topn.go new file mode 100644 index 0000000..296984f --- /dev/null +++ b/search/collector/topn.go @@ -0,0 +1,739 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + "reflect" + "strconv" + "time" + + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeTopNCollector int + +func init() { + var coll TopNCollector + reflectStaticSizeTopNCollector = int(reflect.TypeOf(coll).Size()) +} + +type collectorStore interface { + // Add the document, and if the new store size exceeds the provided size + // the last element is removed and returned. If the size has not been + // exceeded, nil is returned. + AddNotExceedingSize(doc *search.DocumentMatch, size int) *search.DocumentMatch + + Final(skip int, fixup collectorFixup) (search.DocumentMatchCollection, error) + + // Provide access the internal heap implementation + Internal() search.DocumentMatchCollection +} + +// PreAllocSizeSkipCap will cap preallocation to this amount when +// size+skip exceeds this value +var PreAllocSizeSkipCap = 1000 + +type collectorCompare func(i, j *search.DocumentMatch) int + +type collectorFixup func(d *search.DocumentMatch) error + +// TopNCollector collects the top N hits, optionally skipping some results +type TopNCollector struct { + size int + skip int + total uint64 + bytesRead uint64 + maxScore float64 + took time.Duration + sort search.SortOrder + results search.DocumentMatchCollection + facetsBuilder *search.FacetsBuilder + + store collectorStore + cmp collectorCompare // specialized or generic; shared by heap + dmHandler + + needDocIds bool + neededFields []string + cachedScoring []bool + cachedDesc []bool + + lowestMatchOutsideResults *search.DocumentMatch + updateFieldVisitor index.DocValueVisitor + dvReader index.DocValueReader + searchAfter *search.DocumentMatch + + knnHits map[string]*search.DocumentMatch + hybridMergeCallback search.HybridMergeCallbackFn + + nestedStore *collectStoreNested + + fastPrepare bool + + earlyStopN int + earlyStopped bool +} + +// CheckDoneEvery controls how frequently we check the context deadline +const CheckDoneEvery = uint64(1024) + +// NewTopNCollector builds a collector to find the top 'size' hits +// skipping over the first 'skip' hits +// ordering hits by the provided sort order +func NewTopNCollector(size int, skip int, sort search.SortOrder) *TopNCollector { + return newTopNCollector(size, skip, sort, nil) +} + +// NewTopNCollectorAfter builds a collector to find the top 'size' hits +// skipping over the first 'skip' hits +// ordering hits by the provided sort order +// starting after the provided 'after' sort values +func NewTopNCollectorAfter(size int, sort search.SortOrder, after []string) *TopNCollector { + rv := newTopNCollector(size, 0, sort, nil) + rv.searchAfter = createSearchAfterDocument(sort, after) + return rv +} + +// NewNestedTopNCollector builds a collector to find the top 'size' hits +// skipping over the first 'skip' hits +// ordering hits by the provided sort order +// while ensuring the nested documents are handled correctly +// (i.e. parent document is returned instead of nested document) +func NewNestedTopNCollector(size int, skip int, sort search.SortOrder, nr index.NestedReader) *TopNCollector { + return newTopNCollector(size, skip, sort, nr) +} + +// NewNestedTopNCollectorAfter builds a collector to find the top 'size' hits +// skipping over the first 'skip' hits +// ordering hits by the provided sort order +// starting after the provided 'after' sort values +// while ensuring the nested documents are handled correctly +// (i.e. parent document is returned instead of nested document) +func NewNestedTopNCollectorAfter(size int, sort search.SortOrder, after []string, nr index.NestedReader) *TopNCollector { + rv := newTopNCollector(size, 0, sort, nr) + rv.searchAfter = createSearchAfterDocument(sort, after) + return rv +} + +func newTopNCollector(size int, skip int, sort search.SortOrder, nr index.NestedReader) *TopNCollector { + hc := &TopNCollector{ + size: size, + skip: skip, + sort: sort, + neededFields: sort.RequiredFields(), + cachedScoring: sort.CacheIsScore(), + cachedDesc: sort.CacheDescending(), + needDocIds: sort.RequiresDocID(), + } + + hc.cmp = getOptimalCollectorCompare(hc) + hc.store = getOptimalCollectorStore(size, skip, hc.cmp) + + if nr != nil { + descAdder := func(parent, child *search.DocumentMatch) error { + // add descendant score to parent score + parent.Score += child.Score + // merge explanations + parent.Expl = parent.Expl.MergeWith(child.Expl) + // merge field term locations + parent.FieldTermLocations = search.MergeFieldTermLocationsFromMatch(parent.FieldTermLocations, child) + // add child's ID to parent's Descendants + // add other as descendant only if it is not the same document + if !parent.IndexInternalID.Equals(child.IndexInternalID) { + // Add a copy of child.IndexInternalID to descendants, because + // child.IndexInternalID will be reset when 'child' is recycled. + var descendantID index.IndexInternalID + // first check if parent's descendants slice has capacity to reuse + if len(parent.Descendants) < cap(parent.Descendants) { + // reuse the buffer element at len(parent.Descendants) + descendantID = parent.Descendants[:len(parent.Descendants)+1][len(parent.Descendants)] + } + // copy the contents of id into descendantID, allocating if needed + parent.Descendants = append(parent.Descendants, index.NewIndexInternalIDFrom(descendantID, child.IndexInternalID)) + } + return nil + } + hc.nestedStore = newStoreNested(nr, search.DescendantAdderCallbackFn(descAdder)) + } + + return hc +} + +func getOptimalCollectorCompare(hc *TopNCollector) collectorCompare { + if len(hc.sort) == 1 && hc.cachedScoring[0] && hc.cachedDesc[0] { + return search.CompareScoreDescending + } + return func(i, j *search.DocumentMatch) int { + return hc.sort.Compare(hc.cachedScoring, hc.cachedDesc, i, j) + } +} + +// Creates a dummy document to compare with for pagination. +func createSearchAfterDocument(sort search.SortOrder, after []string) *search.DocumentMatch { + encodedAfter := make([]string, len(after)) + for i, ss := range sort { + encodedAfter[i] = encodeSearchAfter(ss, after[i]) + } + + rv := &search.DocumentMatch{ + Sort: encodedAfter, + } + for pos, ss := range sort { + if ss.RequiresDocID() { + rv.ID = after[pos] + } + if ss.RequiresScoring() { + if score, err := strconv.ParseFloat(after[pos], 64); err == nil { + rv.Score = score + } + } + } + return rv +} + +// encodeSearchAfter applies prefix-coding to SearchAfter +// if required to enable pagination on numeric, datetime, +// and geo fields +func encodeSearchAfter(ss search.SearchSort, after string) string { + encodeFloat := func() string { + f64, _ := strconv.ParseFloat(after, 64) // error checking in SearchRequest.Validate + i64 := numeric.Float64ToInt64(f64) + return string(numeric.MustNewPrefixCodedInt64(i64, 0)) + } + + encodeDate := func() string { + t, _ := time.Parse(time.RFC3339Nano, after) // error checking in SearchRequest.Validate + i64 := t.UnixNano() + return string(numeric.MustNewPrefixCodedInt64(i64, 0)) + } + + switch ss := ss.(type) { + case *search.SortGeoDistance: + return encodeFloat() + case *search.SortField: + switch ss.Type { + case search.SortFieldAsNumber: + return encodeFloat() + case search.SortFieldAsDate: + return encodeDate() + default: + // For SortFieldAsString and SortFieldAuto + // NOTE: SortFieldAuto is used if you set Sort with a string + // or if the type of the field is not set in the object + // in the Sort slice. We cannot perform type inference in + // this case, so we return the original string, even if + // its actually numeric or date. + return after + } + default: + // For SortDocID and SortScore + return after + } +} + +// Filter document matches based on the SearchAfter field in the SearchRequest. +func FilterHitsBySearchAfter(hits []*search.DocumentMatch, sort search.SortOrder, after []string) []*search.DocumentMatch { + if len(hits) == 0 { + return hits + } + // create a search after document + searchAfter := createSearchAfterDocument(sort, after) + // filter the hits + idx := 0 + cachedScoring := sort.CacheIsScore() + cachedDesc := sort.CacheDescending() + for _, hit := range hits { + if sort.Compare(cachedScoring, cachedDesc, hit, searchAfter) > 0 { + hits[idx] = hit + idx++ + } + } + return hits[:idx] +} + +func getOptimalCollectorStore(size, skip int, comparator collectorCompare) collectorStore { + // pre-allocate space on the store to avoid reslicing + // unless the size + skip is too large, then cap it + // everything should still work, just reslices as necessary + backingSize := size + skip + 1 + if size+skip > PreAllocSizeSkipCap { + backingSize = PreAllocSizeSkipCap + 1 + } + + if size+skip > 10 { + return newStoreHeap(backingSize, comparator) + } else { + return newStoreSlice(backingSize, comparator) + } +} + +func (hc *TopNCollector) Size() int { + sizeInBytes := reflectStaticSizeTopNCollector + size.SizeOfPtr + + if hc.facetsBuilder != nil { + sizeInBytes += hc.facetsBuilder.Size() + } + + for _, entry := range hc.neededFields { + sizeInBytes += len(entry) + size.SizeOfString + } + + sizeInBytes += len(hc.cachedScoring) + len(hc.cachedDesc) + + return sizeInBytes +} + +// Collect goes to the index to find the matching documents +func (hc *TopNCollector) Collect(ctx context.Context, searcher search.Searcher, reader index.IndexReader) error { + startTime := time.Now() + var err error + var next *search.DocumentMatch + + // pre-allocate enough space in the DocumentMatchPool + // unless the size + skip is too large, then cap it + // everything should still work, just allocates DocumentMatches on demand + backingSize := hc.size + hc.skip + 1 + if hc.size+hc.skip > PreAllocSizeSkipCap { + backingSize = PreAllocSizeSkipCap + 1 + } + searchContext := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(backingSize+searcher.DocumentMatchPoolSize(), len(hc.sort)), + Collector: hc, + IndexReader: reader, + } + + hc.dvReader, err = reader.DocValueReader(hc.neededFields) + if err != nil { + return err + } + + hc.updateFieldVisitor = func(field string, term []byte) { + if hc.facetsBuilder != nil { + hc.facetsBuilder.UpdateVisitor(field, term) + } + hc.sort.UpdateVisitor(field, term) + } + + dmHandlerMaker := MakeTopNDocumentMatchHandler + if cv := ctx.Value(search.MakeDocumentMatchHandlerKey); cv != nil { + dmHandlerMaker = cv.(search.MakeDocumentMatchHandler) + } + // use the application given builder for making the custom document match + // handler and perform callbacks/invocations on the newly made handler. + dmHandler, loadID, err := dmHandlerMaker(searchContext) + if err != nil { + return err + } + + hc.needDocIds = hc.needDocIds || loadID + + // fastPrepare is set to true when a hit needs only basicPrepare plus the shared + // score sort value (see canFastPrepare). Applies only to score-sorted queries + // with no field-loading needs. + hc.fastPrepare = hc.canFastPrepare() + select { + case <-ctx.Done(): + search.RecordSearchCost(ctx, search.AbortM, 0) + return ctx.Err() + default: + next, err = searcher.Next(searchContext) + } + // use a local totalDocs for counting total docs seen + // for context deadline checking, as hc.total is only + // incremented for actual(root) collected documents, and + // we need to check deadline for every document seen (root or nested) + var totalDocs uint64 + for err == nil && next != nil { + if totalDocs%CheckDoneEvery == 0 { + select { + case <-ctx.Done(): + search.RecordSearchCost(ctx, search.AbortM, 0) + return ctx.Err() + default: + } + } + totalDocs++ + if hc.nestedStore != nil { + // This may be a nested document — add it to the nested store first. + // If the nested store returns nil, the document was merged into its parent + // and should not be processed further. + // If it returns a non-nil document, it represents a complete root document + // and should be processed further. + next, err = hc.nestedStore.ProcessNestedDocument(searchContext, next) + if err != nil { + break + } + } + if next != nil { + if hc.knnHits != nil { + err = hc.adjustKNNDocumentMatch(searchContext, reader, next) + if err != nil { + break + } + } + hc.basicPrepare(next) + if hc.fastPrepare { + next.Sort = sortByScoreOpt + } else { + err = hc.prepareDocumentMatch(searchContext, reader, next) + if err != nil { + break + } + } + err = dmHandler(next) + if err != nil { + break + } + if hc.earlyStopN > 0 && hc.total >= uint64(hc.earlyStopN) { + hc.earlyStopped = true + break + } + } + next, err = searcher.Next(searchContext) + } + if err != nil { + return err + } + + // if we have a nested store, we may have an interim root + // that needs to be returned for processing + if hc.nestedStore != nil { + currRoot := hc.nestedStore.Current() + if currRoot != nil { + if hc.knnHits != nil { + err = hc.adjustKNNDocumentMatch(searchContext, reader, currRoot) + if err != nil { + return err + } + } + // no descendants at this point + hc.basicPrepare(currRoot) + if hc.fastPrepare { + currRoot.Sort = sortByScoreOpt + } else { + err = hc.prepareDocumentMatch(searchContext, reader, currRoot) + if err != nil { + return err + } + } + + err = dmHandler(currRoot) + if err != nil { + return err + } + } + } + + if hc.knnHits != nil { + // we may have some knn hits left that did not match any of the top N tf-idf hits + // we need to add them to the collector store to consider them as well. + for _, knnDoc := range hc.knnHits { + err = hc.prepareKNNDocumentMatch(searchContext, reader, knnDoc) + if err != nil { + return err + } + err = dmHandler(knnDoc) + if err != nil { + return err + } + } + } + + statsCallbackFn := ctx.Value(search.SearchIOStatsCallbackKey) + if statsCallbackFn != nil { + // hc.bytesRead corresponds to the + // total bytes read as part of docValues being read every hit + // which must be accounted by invoking the callback. + statsCallbackFn.(search.SearchIOStatsCallbackFunc)(hc.bytesRead) + + search.RecordSearchCost(ctx, search.AddM, hc.bytesRead) + } + + // help finalize/flush the results in case + // of custom document match handlers. + err = dmHandler(nil) + if err != nil { + return err + } + + // compute search duration + hc.took = time.Since(startTime) + + // finalize actual results + err = hc.finalizeResults(reader) + if err != nil { + return err + } + return nil +} + +var sortByScoreOpt = []string{"_score"} + +func (hc *TopNCollector) adjustKNNDocumentMatch(ctx *search.SearchContext, + reader index.IndexReader, d *search.DocumentMatch) (err error) { + d.ID, err = reader.ExternalID(d.IndexInternalID) + if err != nil { + return err + } + if knnHit, ok := hc.knnHits[d.ID]; ok { + // merge this document's hit with its knn score, expl etc. and remove it from the map + hc.hybridMergeCallback(d, knnHit) + delete(hc.knnHits, d.ID) + } + return nil +} + +func (hc *TopNCollector) basicPrepare(d *search.DocumentMatch) { + hc.total++ + d.HitNumber = hc.total + if d.Score > hc.maxScore { + hc.maxScore = d.Score + } +} + +func (hc *TopNCollector) canFastPrepare() bool { + return len(hc.neededFields) == 0 && !hc.needDocIds && + len(hc.sort) == 1 && hc.cachedScoring[0] +} + +// prepareDocumentMatch does the non-fast preparation for a regular (non-KNN) +// hit: visiting field terms required for sort/facets, loading the docID if +// needed, and computing the sort value. basicPrepare must be called first. +func (hc *TopNCollector) prepareDocumentMatch(ctx *search.SearchContext, + reader index.IndexReader, d *search.DocumentMatch) (err error) { + + // visit field terms for features that require it (sort, facets) + if len(hc.neededFields) > 0 { + err = hc.visitFieldTerms(reader, d, hc.updateFieldVisitor) + if err != nil { + return err + } + } + + // see if we need to load ID (at this early stage, for example to sort on it) + if hc.needDocIds && d.ID == "" { + d.ID, err = reader.ExternalID(d.IndexInternalID) + if err != nil { + return err + } + } + + // compute this hits sort value + if len(hc.sort) == 1 && hc.cachedScoring[0] { + d.Sort = sortByScoreOpt + } else { + hc.sort.Value(d) + } + + return nil +} + +// prepareKNNDocumentMatch prepares a KNN hit. Its sort value was already +// computed by the KNN collector, so only facet field visiting (if any) and the +// basic bookkeeping are needed. +func (hc *TopNCollector) prepareKNNDocumentMatch(ctx *search.SearchContext, + reader index.IndexReader, d *search.DocumentMatch) (err error) { + + if hc.facetsBuilder != nil { + err = hc.visitFieldTerms(reader, d, func(field string, term []byte) { + hc.facetsBuilder.UpdateVisitor(field, term) + }) + if err != nil { + return err + } + } + + hc.basicPrepare(d) + return nil +} + +func MakeTopNDocumentMatchHandler( + ctx *search.SearchContext) (search.DocumentMatchHandler, bool, error) { + var hc *TopNCollector + var ok bool + if hc, ok = ctx.Collector.(*TopNCollector); ok { + return func(d *search.DocumentMatch) error { + if d == nil { + return nil + } + + // support search after based pagination, + // if this hit is <= the search after sort key + // we should skip it + if hc.searchAfter != nil { + // exact sort order matches use hit number to break tie + // but we want to allow for exact match, so we pretend + hc.searchAfter.HitNumber = d.HitNumber + if hc.cmp(d, hc.searchAfter) <= 0 { + ctx.DocumentMatchPool.Put(d) + return nil + } + } + + // optimization, we track lowest sorting hit already removed from heap + // with this one comparison, we can avoid all heap operations if + // this hit would have been added and then immediately removed + if hc.lowestMatchOutsideResults != nil { + if hc.cmp(d, hc.lowestMatchOutsideResults) >= 0 { + // this hit can't possibly be in the result set, so avoid heap ops + ctx.DocumentMatchPool.Put(d) + return nil + } + } + + removed := hc.store.AddNotExceedingSize(d, hc.size+hc.skip) + if removed != nil { + if hc.lowestMatchOutsideResults == nil { + hc.lowestMatchOutsideResults = removed + } else { + cmp := hc.cmp(removed, hc.lowestMatchOutsideResults) + if cmp < 0 { + tmp := hc.lowestMatchOutsideResults + hc.lowestMatchOutsideResults = removed + ctx.DocumentMatchPool.Put(tmp) + } + } + } + return nil + }, false, nil + } + return nil, false, nil +} + +// visitFieldTerms is responsible for visiting the field terms of the +// search hit, and passing visited terms to the sort and facet builder +func (hc *TopNCollector) visitFieldTerms(reader index.IndexReader, d *search.DocumentMatch, v index.DocValueVisitor) error { + if hc.facetsBuilder != nil { + hc.facetsBuilder.StartDoc() + } + if d.ID != "" && d.IndexInternalID == nil { + // this document may have been sent over as preSearchData and + // we need to look up the internal id to visit the doc values for it + var err error + d.IndexInternalID, err = reader.InternalID(d.ID) + if err != nil { + return err + } + } + + // first visit descendants if any + for _, descID := range d.Descendants { + err := hc.dvReader.VisitDocValues(descID, v) + if err != nil { + return err + } + } + // now visit the doc values for this document + err := hc.dvReader.VisitDocValues(d.IndexInternalID, v) + if hc.facetsBuilder != nil { + hc.facetsBuilder.EndDoc() + } + + hc.bytesRead += hc.dvReader.BytesRead() + + return err +} + +// SetFacetsBuilder registers a facet builder for this collector +func (hc *TopNCollector) SetFacetsBuilder(facetsBuilder *search.FacetsBuilder) { + hc.facetsBuilder = facetsBuilder + fieldsRequiredForFaceting := facetsBuilder.RequiredFields() + // for each of these fields, append only if not already there in hc.neededFields. + for _, field := range fieldsRequiredForFaceting { + found := false + for _, neededField := range hc.neededFields { + if field == neededField { + found = true + break + } + } + if !found { + hc.neededFields = append(hc.neededFields, field) + } + } +} + +// finalizeResults starts with the heap containing the final top size+skip +// it now throws away the results to be skipped +// and does final doc id lookup (if necessary) +func (hc *TopNCollector) finalizeResults(r index.IndexReader) error { + var err error + hc.results, err = hc.store.Final(hc.skip, func(doc *search.DocumentMatch) error { + if doc.ID == "" { + // look up the id since we need it for lookup + var err error + doc.ID, err = r.ExternalID(doc.IndexInternalID) + if err != nil { + return err + } + } + doc.Complete(nil) + return nil + }) + + return err +} + +// Results returns the collected hits +func (hc *TopNCollector) Results() search.DocumentMatchCollection { + return hc.results +} + +// Total returns the total number of hits +func (hc *TopNCollector) Total() uint64 { + return hc.total +} + +// SetEarlyStop makes Collect() stop pulling from the searcher once n hits have +// been collected; n <= 0 disables. Callers must ensure unseen docs cannot +// change the result. +func (hc *TopNCollector) SetEarlyStop(n int) { + hc.earlyStopN = n +} + +// EarlyStopped reports whether Collect() stopped early; if true, Total() is a +// lower bound. +func (hc *TopNCollector) EarlyStopped() bool { + return hc.earlyStopped +} + +// MaxScore returns the maximum score seen across all the hits +func (hc *TopNCollector) MaxScore() float64 { + return hc.maxScore +} + +// Took returns the time spent collecting hits +func (hc *TopNCollector) Took() time.Duration { + return hc.took +} + +// FacetResults returns the computed facets results +func (hc *TopNCollector) FacetResults() search.FacetResults { + if hc.facetsBuilder != nil { + return hc.facetsBuilder.Results() + } + return nil +} + +func (hc *TopNCollector) SetKNNHits(knnHits search.DocumentMatchCollection, hybridMergeCallback search.HybridMergeCallbackFn) { + hc.knnHits = make(map[string]*search.DocumentMatch, len(knnHits)) + for _, hit := range knnHits { + hc.knnHits[hit.ID] = hit + } + hc.hybridMergeCallback = hybridMergeCallback +} diff --git a/search/collector/topn_test.go b/search/collector/topn_test.go new file mode 100644 index 0000000..db39043 --- /dev/null +++ b/search/collector/topn_test.go @@ -0,0 +1,1116 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "bytes" + "context" + "strconv" + "testing" + "time" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/facet" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +func TestTop10Scores(t *testing.T) { + // a stub search with more than 10 matches + // the top-10 scores are > 10 + // everything else is less than 10 + searcher := &stubSearcher{ + matches: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("a"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("b"), + Score: 9, + }, + { + IndexInternalID: index.IndexInternalID("c"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("d"), + Score: 9, + }, + { + IndexInternalID: index.IndexInternalID("e"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("f"), + Score: 9, + }, + { + IndexInternalID: index.IndexInternalID("g"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("h"), + Score: 9, + }, + { + IndexInternalID: index.IndexInternalID("i"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("j"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("k"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("l"), + Score: 99, + }, + { + IndexInternalID: index.IndexInternalID("m"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("n"), + Score: 11, + }, + }, + } + + collector := NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}}) + err := collector.Collect(context.Background(), searcher, &stubReader{}) + if err != nil { + t.Fatal(err) + } + + maxScore := collector.MaxScore() + if maxScore != 99.0 { + t.Errorf("expected max score 99.0, got %f", maxScore) + } + + total := collector.Total() + if total != 14 { + t.Errorf("expected 14 total results, got %d", total) + } + + results := collector.Results() + + if len(results) != 10 { + t.Logf("results: %v", results) + t.Fatalf("expected 10 results, got %d", len(results)) + } + + if results[0].ID != "l" { + t.Errorf("expected first result to have ID 'l', got %s", results[0].ID) + } + + if results[0].Score != 99.0 { + t.Errorf("expected highest score to be 99.0, got %f", results[0].Score) + } + + minScore := 1000.0 + for _, result := range results { + if result.Score < minScore { + minScore = result.Score + } + } + + if minScore < 10 { + t.Errorf("expected minimum score to be higher than 10, got %f", minScore) + } +} + +func TestTop10ScoresSkip10(t *testing.T) { + // a stub search with more than 10 matches + // the top-10 scores are > 10 + // everything else is less than 10 + searcher := &stubSearcher{ + matches: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("a"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("b"), + Score: 9.5, + }, + { + IndexInternalID: index.IndexInternalID("c"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("d"), + Score: 9, + }, + { + IndexInternalID: index.IndexInternalID("e"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("f"), + Score: 9, + }, + { + IndexInternalID: index.IndexInternalID("g"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("h"), + Score: 9, + }, + { + IndexInternalID: index.IndexInternalID("i"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("j"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("k"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("l"), + Score: 99, + }, + { + IndexInternalID: index.IndexInternalID("m"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("n"), + Score: 11, + }, + }, + } + + collector := NewTopNCollector(10, 10, search.SortOrder{&search.SortScore{Desc: true}}) + err := collector.Collect(context.Background(), searcher, &stubReader{}) + if err != nil { + t.Fatal(err) + } + + maxScore := collector.MaxScore() + if maxScore != 99.0 { + t.Errorf("expected max score 99.0, got %f", maxScore) + } + + total := collector.Total() + if total != 14 { + t.Errorf("expected 14 total results, got %d", total) + } + + results := collector.Results() + + if len(results) != 4 { + t.Fatalf("expected 4 results, got %d", len(results)) + } + + if results[0].ID != "b" { + t.Errorf("expected first result to have ID 'b', got %s", results[0].ID) + } + + if results[0].Score != 9.5 { + t.Errorf("expected highest score to be 9.5, got %f", results[0].Score) + } +} + +func TestTop10ScoresSkip10Only9Hits(t *testing.T) { + // a stub search with only 10 matches + searcher := &stubSearcher{ + matches: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("a"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("c"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("e"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("g"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("i"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("j"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("k"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("m"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("n"), + Score: 11, + }, + }, + } + + collector := NewTopNCollector(10, 10, search.SortOrder{&search.SortScore{Desc: true}}) + err := collector.Collect(context.Background(), searcher, &stubReader{}) + if err != nil { + t.Fatal(err) + } + + total := collector.Total() + if total != 9 { + t.Errorf("expected 9 total results, got %d", total) + } + + results := collector.Results() + + if len(results) != 0 { + t.Fatalf("expected 0 results, got %d", len(results)) + } +} + +func TestPaginationSameScores(t *testing.T) { + // a stub search with more than 10 matches + // all documents have the same score + searcher := &stubSearcher{ + matches: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("a"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("b"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("c"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("d"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("e"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("f"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("g"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("h"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("i"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("j"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("k"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("l"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("m"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("n"), + Score: 5, + }, + }, + } + + // first get first 5 hits + collector := NewTopNCollector(5, 0, search.SortOrder{&search.SortScore{Desc: true}}) + err := collector.Collect(context.Background(), searcher, &stubReader{}) + if err != nil { + t.Fatal(err) + } + + total := collector.Total() + if total != 14 { + t.Errorf("expected 14 total results, got %d", total) + } + + results := collector.Results() + + if len(results) != 5 { + t.Fatalf("expected 5 results, got %d", len(results)) + } + + firstResults := make(map[string]struct{}) + for _, hit := range results { + firstResults[hit.ID] = struct{}{} + } + + // a stub search with more than 10 matches + // all documents have the same score + searcher = &stubSearcher{ + matches: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("a"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("b"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("c"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("d"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("e"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("f"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("g"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("h"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("i"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("j"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("k"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("l"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("m"), + Score: 5, + }, + { + IndexInternalID: index.IndexInternalID("n"), + Score: 5, + }, + }, + } + + // now get next 5 hits + collector = NewTopNCollector(5, 5, search.SortOrder{&search.SortScore{Desc: true}}) + err = collector.Collect(context.Background(), searcher, &stubReader{}) + if err != nil { + t.Fatal(err) + } + + total = collector.Total() + if total != 14 { + t.Errorf("expected 14 total results, got %d", total) + } + + results = collector.Results() + + if len(results) != 5 { + t.Fatalf("expected 5 results, got %d", len(results)) + } + + // make sure that none of these hits repeat ones we saw in the top 5 + for _, hit := range results { + if _, ok := firstResults[hit.ID]; ok { + t.Errorf("doc ID %s is in top 5 and next 5 result sets", hit.ID) + } + } +} + +// TestStreamResults verifies the search.DocumentMatchHandler +func TestStreamResults(t *testing.T) { + matches := []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("a"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("b"), + Score: 1, + }, + { + IndexInternalID: index.IndexInternalID("c"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("d"), + Score: 999, + }, + { + IndexInternalID: index.IndexInternalID("e"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("f"), + Score: 9, + }, + { + IndexInternalID: index.IndexInternalID("g"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("h"), + Score: 89, + }, + { + IndexInternalID: index.IndexInternalID("i"), + Score: 101, + }, + { + IndexInternalID: index.IndexInternalID("j"), + Score: 112, + }, + { + IndexInternalID: index.IndexInternalID("k"), + Score: 10, + }, + { + IndexInternalID: index.IndexInternalID("l"), + Score: 99, + }, + { + IndexInternalID: index.IndexInternalID("m"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("n"), + Score: 111, + }, + } + + searcher := &stubSearcher{ + matches: matches, + } + ind := 0 + docMatchHandler := func(hit *search.DocumentMatch) error { + if hit == nil { + return nil // search completed + } + if !bytes.Equal(hit.IndexInternalID, matches[ind].IndexInternalID) { + t.Errorf("%d hit IndexInternalID actual: %s, expected: %s", + ind, hit.IndexInternalID, matches[ind].IndexInternalID) + } + if hit.Score != matches[ind].Score { + t.Errorf("%d hit Score actual: %s, expected: %s", + ind, hit.IndexInternalID, matches[ind].IndexInternalID) + } + ind++ + return nil + } + + var handlerMaker search.MakeDocumentMatchHandler = func(ctx *search.SearchContext) (search.DocumentMatchHandler, bool, error) { + return docMatchHandler, false, nil + } + + ctx := context.WithValue(context.Background(), search.MakeDocumentMatchHandlerKey, handlerMaker) + + collector := NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}}) + err := collector.Collect(ctx, searcher, &stubReader{}) + if err != nil { + t.Fatal(err) + } + + maxScore := collector.MaxScore() + if maxScore != 999.0 { + t.Errorf("expected max score 99.0, got %f", maxScore) + } + + total := collector.Total() + if int(total) != ind { + t.Errorf("expected 14 total results, got %d", total) + } + + results := collector.Results() + + if len(results) != 0 { + t.Fatalf("expected 0 results, got %d", len(results)) + } +} + +// TestCollectorChaining verifies the chaining of collectors. +// The custom DocumentMatchHandler can process every hit for +// the search query and then pass the hit to the topn collector +// to eventually have the sorted top `N` results. +func TestCollectorChaining(t *testing.T) { + matches := []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("a"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("b"), + Score: 1, + }, + { + IndexInternalID: index.IndexInternalID("c"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("d"), + Score: 999, + }, + { + IndexInternalID: index.IndexInternalID("e"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("f"), + Score: 9, + }, + { + IndexInternalID: index.IndexInternalID("g"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("h"), + Score: 89, + }, + { + IndexInternalID: index.IndexInternalID("i"), + Score: 101, + }, + { + IndexInternalID: index.IndexInternalID("j"), + Score: 112, + }, + { + IndexInternalID: index.IndexInternalID("k"), + Score: 10, + }, + { + IndexInternalID: index.IndexInternalID("l"), + Score: 99, + }, + { + IndexInternalID: index.IndexInternalID("m"), + Score: 11, + }, + { + IndexInternalID: index.IndexInternalID("n"), + Score: 111, + }, + } + + searcher := &stubSearcher{ + matches: matches, + } + + var topNHandler search.DocumentMatchHandler + ind := 0 + docMatchHandler := func(hit *search.DocumentMatch) error { + if hit == nil { + return nil // search completed + } + if !bytes.Equal(hit.IndexInternalID, matches[ind].IndexInternalID) { + t.Errorf("%d hit IndexInternalID actual: %s, expected: %s", + ind, hit.IndexInternalID, matches[ind].IndexInternalID) + } + if hit.Score != matches[ind].Score { + t.Errorf("%d hit Score actual: %s, expected: %s", + ind, hit.IndexInternalID, matches[ind].IndexInternalID) + } + ind++ + // give the hit back to the topN collector + err := topNHandler(hit) + if err != nil { + t.Errorf("unexpected err: %v", err) + } + return nil + } + + var handlerMaker search.MakeDocumentMatchHandler = func(ctx *search.SearchContext) (search.DocumentMatchHandler, bool, error) { + topNHandler, _, _ = MakeTopNDocumentMatchHandler(ctx) + return docMatchHandler, false, nil + } + + ctx := context.WithValue(context.Background(), search.MakeDocumentMatchHandlerKey, + handlerMaker) + + collector := NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}}) + err := collector.Collect(ctx, searcher, &stubReader{}) + if err != nil { + t.Fatal(err) + } + + maxScore := collector.MaxScore() + if maxScore != 999.0 { + t.Errorf("expected max score 99.0, got %f", maxScore) + } + + total := collector.Total() + if int(total) != ind { + t.Errorf("expected 14 total results, got %d", total) + } + + results := collector.Results() + + if len(results) != 10 { // as it is paged + t.Fatalf("expected 0 results, got %d", len(results)) + } + + if results[0].ID != "d" { + t.Errorf("expected first result to have ID 'l', got %s", results[0].ID) + } + + if results[0].Score != 999.0 { + t.Errorf("expected highest score to be 999.0, got %f", results[0].Score) + } + + minScore := 1000.0 + for _, result := range results { + if result.Score < minScore { + minScore = result.Score + } + } + + if minScore < 10 { + t.Errorf("expected minimum score to be higher than 10, got %f", minScore) + } +} + +func setupIndex(t *testing.T) index.Index { + analysisQueue := index.NewAnalysisQueue(1) + i, err := scorch.NewScorch( + scorch.Name, + map[string]interface{}{ + "path": "", + }, + analysisQueue) + if err != nil { + t.Fatal(err) + } + err = i.Open() + if err != nil { + t.Fatal(err) + } + + return i +} + +func TestSetFacetsBuilder(t *testing.T) { + // Field common to both sorting and faceting. + sortFacetsField := "locations" + + coll := NewTopNCollector(10, 0, search.SortOrder{&search.SortField{Field: sortFacetsField}}) + + i := setupIndex(t) + indexReader, err := i.Reader() + if err != nil { + t.Fatal(err) + } + + fb := search.NewFacetsBuilder(indexReader) + facetBuilder := facet.NewTermsFacetBuilder(sortFacetsField, 100) + fb.Add("locations_facet", facetBuilder) + coll.SetFacetsBuilder(fb) + + // Should not duplicate the "locations" field in the collector. + if len(coll.neededFields) != 1 || coll.neededFields[0] != sortFacetsField { + t.Errorf("expected fields in collector: %v, observed: %v", []string{sortFacetsField}, coll.neededFields) + } +} + +func TestSearchAfterNumeric(t *testing.T) { + idx := setupIndex(t) + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + docs := []struct { + id string + data int64 + }{ + {"a", 10}, + {"b", 9}, + {"c", 8}, + {"d", 7}, + {"e", 6}, + {"f", 5}, + {"g", 4}, + {"h", 3}, + {"i", 2}, + {"j", 1}, + } + + batch := index.NewBatch() + for _, d := range docs { + doc := document.NewDocument(d.id) + field := document.NewNumericFieldWithIndexingOptions("data", []uint64{}, float64(d.data), index.IndexField|index.StoreField|index.IncludeTermVectors) + doc.AddField(field) + batch.Update(doc) + } + + err := idx.Batch(batch) + if err != nil { + t.Fatal(err) + } + + reader, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + defer func() { + err := reader.Close() + if err != nil { + t.Fatal(err) + } + }() + + searcher, err := searcher.NewMatchAllSearcher(context.Background(), reader, 1.0, search.SearcherOptions{}) + if err != nil { + t.Fatal(err) + } + + sortOrder := search.SortOrder{&search.SortField{Field: "data", Type: search.SortFieldAsNumber, Desc: true}} + + after := []string{"6"} + + collectorAfter := NewTopNCollectorAfter(5, sortOrder, after) + err = collectorAfter.Collect(context.Background(), searcher, reader) + if err != nil { + t.Fatal(err) + } + + resultsAfter := collectorAfter.Results() + if len(resultsAfter) != 5 { + t.Fatalf("expected 5 results, got %d", len(resultsAfter)) + } + for i := range resultsAfter { + raID := resultsAfter[i].ID + docID := docs[i+len(resultsAfter)].id + if raID != docID { + t.Errorf("expected result '%s', got '%s'", docID, raID) + } + } +} + +func TestSearchAfterDateTime(t *testing.T) { + idx := setupIndex(t) + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + docs := []struct { + id string + data time.Time + }{ + {"a", time.Unix(10, 0).UTC()}, + {"b", time.Unix(9, 0).UTC()}, + {"c", time.Unix(8, 0).UTC()}, + {"d", time.Unix(7, 0).UTC()}, + {"e", time.Unix(6, 0).UTC()}, + {"f", time.Unix(5, 0).UTC()}, + {"g", time.Unix(4, 0).UTC()}, + {"h", time.Unix(3, 0).UTC()}, + {"i", time.Unix(2, 0).UTC()}, + {"j", time.Unix(1, 0).UTC()}, + } + + batch := index.NewBatch() + for _, d := range docs { + doc := document.NewDocument(d.id) + field, err := document.NewDateTimeFieldWithIndexingOptions("data", []uint64{}, d.data, time.RFC3339Nano, index.IndexField|index.StoreField|index.IncludeTermVectors) + if err != nil { + t.Fatal(err) + } + doc.AddField(field) + batch.Update(doc) + } + + err := idx.Batch(batch) + if err != nil { + t.Fatal(err) + } + + reader, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + defer func() { + err := reader.Close() + if err != nil { + t.Fatal(err) + } + }() + + searcher, err := searcher.NewMatchAllSearcher(context.Background(), reader, 1.0, search.SearcherOptions{}) + if err != nil { + t.Fatal(err) + } + + sortOrder := search.SortOrder{&search.SortField{Field: "data", Type: search.SortFieldAsDate, Desc: true}} + + afterTime := time.Unix(6, 0).UTC() + after := []string{afterTime.Format(time.RFC3339Nano)} + + collectorAfter := NewTopNCollectorAfter(5, sortOrder, after) + err = collectorAfter.Collect(context.Background(), searcher, reader) + if err != nil { + t.Fatal(err) + } + + resultsAfter := collectorAfter.Results() + if len(resultsAfter) != 5 { + t.Fatalf("expected 5 results, got %d", len(resultsAfter)) + } + for i := range resultsAfter { + raID := resultsAfter[i].ID + docID := docs[i+len(resultsAfter)].id + if raID != docID { + t.Errorf("expected result '%s', got '%s'", docID, raID) + } + } +} + +func TestSearchAfterGeo(t *testing.T) { + idx := setupIndex(t) + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + docs := []struct { + id string + lon float64 + lat float64 + }{ + {"a", 1, 0}, + {"b", 2, 0}, + {"c", 3, 0}, + {"d", 4, 0}, + {"e", 5, 0}, + {"f", 6, 0}, + {"g", 7, 0}, + {"h", 8, 0}, + {"i", 9, 0}, + {"j", 10, 0}, + } + + batch := index.NewBatch() + for _, d := range docs { + doc := document.NewDocument(d.id) + field := document.NewGeoPointFieldWithIndexingOptions("location", []uint64{}, d.lon, d.lat, index.IndexField|index.StoreField|index.IncludeTermVectors) + doc.AddField(field) + batch.Update(doc) + } + + err := idx.Batch(batch) + if err != nil { + t.Fatal(err) + } + + reader, err := idx.Reader() + if err != nil { + t.Fatal(err) + } + defer func() { + err := reader.Close() + if err != nil { + t.Fatal(err) + } + }() + + searcher, err := searcher.NewMatchAllSearcher(context.Background(), reader, 1.0, search.SearcherOptions{}) + if err != nil { + t.Fatal(err) + } + + centerLon, centerLat := 0.0, 0.0 + sortOrder := search.SortOrder{&search.SortGeoDistance{Field: "location", Lon: centerLon, Lat: centerLat, Desc: false}} + + // search after doc "e" which has lon 5, lat 0 + afterLon, afterLat := 5.0, 0.0 + afterDistance := geo.Haversin(centerLon, centerLat, afterLon, afterLat) + // to compensate scaling + afterDistance *= 1000 + after := []string{strconv.FormatFloat(afterDistance, 'f', -1, 64)} + + collectorAfter := NewTopNCollectorAfter(5, sortOrder, after) + err = collectorAfter.Collect(context.Background(), searcher, reader) + if err != nil { + t.Fatal(err) + } + + resultsAfter := collectorAfter.Results() + + if len(resultsAfter) != 5 { + t.Fatalf("expected 5 results, got %d", len(resultsAfter)) + } + for i := range resultsAfter { + raID := resultsAfter[i].ID + docID := docs[i+len(resultsAfter)].id + if raID != docID { + t.Errorf("expected result '%s', got '%s'", docID, raID) + } + } +} + +func BenchmarkTop10of0Scores(b *testing.B) { + benchHelper(0, func() search.Collector { + return NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}}) + }, b) +} + +func BenchmarkTop10of3Scores(b *testing.B) { + benchHelper(3, func() search.Collector { + return NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}}) + }, b) +} + +func BenchmarkTop10of10Scores(b *testing.B) { + benchHelper(10, func() search.Collector { + return NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}}) + }, b) +} + +func BenchmarkTop10of25Scores(b *testing.B) { + benchHelper(25, func() search.Collector { + return NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}}) + }, b) +} + +func BenchmarkTop10of50Scores(b *testing.B) { + benchHelper(50, func() search.Collector { + return NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}}) + }, b) +} + +func BenchmarkTop10of10000Scores(b *testing.B) { + benchHelper(10000, func() search.Collector { + return NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}}) + }, b) +} + +func BenchmarkTop100of0Scores(b *testing.B) { + benchHelper(0, func() search.Collector { + return NewTopNCollector(100, 0, search.SortOrder{&search.SortScore{Desc: true}}) + }, b) +} + +func BenchmarkTop100of3Scores(b *testing.B) { + benchHelper(3, func() search.Collector { + return NewTopNCollector(100, 0, search.SortOrder{&search.SortScore{Desc: true}}) + }, b) +} + +func BenchmarkTop100of10Scores(b *testing.B) { + benchHelper(10, func() search.Collector { + return NewTopNCollector(100, 0, search.SortOrder{&search.SortScore{Desc: true}}) + }, b) +} + +func BenchmarkTop100of25Scores(b *testing.B) { + benchHelper(25, func() search.Collector { + return NewTopNCollector(100, 0, search.SortOrder{&search.SortScore{Desc: true}}) + }, b) +} + +func BenchmarkTop100of50Scores(b *testing.B) { + benchHelper(50, func() search.Collector { + return NewTopNCollector(100, 0, search.SortOrder{&search.SortScore{Desc: true}}) + }, b) +} + +func BenchmarkTop100of10000Scores(b *testing.B) { + benchHelper(10000, func() search.Collector { + return NewTopNCollector(100, 0, search.SortOrder{&search.SortScore{Desc: true}}) + }, b) +} + +func BenchmarkTop1000of10000Scores(b *testing.B) { + benchHelper(10000, func() search.Collector { + return NewTopNCollector(1000, 0, search.SortOrder{&search.SortScore{Desc: true}}) + }, b) +} + +func BenchmarkTop10000of100000Scores(b *testing.B) { + benchHelper(100000, func() search.Collector { + return NewTopNCollector(10000, 0, search.SortOrder{&search.SortScore{Desc: true}}) + }, b) +} + +func BenchmarkTop10of100000Scores(b *testing.B) { + benchHelper(100000, func() search.Collector { + return NewTopNCollector(10, 0, search.SortOrder{&search.SortScore{Desc: true}}) + }, b) +} + +func BenchmarkTop100of100000Scores(b *testing.B) { + benchHelper(100000, func() search.Collector { + return NewTopNCollector(100, 0, search.SortOrder{&search.SortScore{Desc: true}}) + }, b) +} + +func BenchmarkTop1000of100000Scores(b *testing.B) { + benchHelper(100000, func() search.Collector { + return NewTopNCollector(1000, 0, search.SortOrder{&search.SortScore{Desc: true}}) + }, b) +} + +func BenchmarkTop10000of1000000Scores(b *testing.B) { + benchHelper(1000000, func() search.Collector { + return NewTopNCollector(10000, 0, search.SortOrder{&search.SortScore{Desc: true}}) + }, b) +} diff --git a/search/explanation.go b/search/explanation.go new file mode 100644 index 0000000..98c5e09 --- /dev/null +++ b/search/explanation.go @@ -0,0 +1,105 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import ( + "encoding/json" + "fmt" + "reflect" + + "github.com/blevesearch/bleve/v2/size" +) + +var reflectStaticSizeExplanation int + +func init() { + var e Explanation + reflectStaticSizeExplanation = int(reflect.TypeOf(e).Size()) +} + +const MergedExplMessage = "sum of merged explanations:" + +type Explanation struct { + Value float64 `json:"value"` + Message string `json:"message"` + PartialMatch bool `json:"partial_match,omitempty"` + Children []*Explanation `json:"children,omitempty"` +} + +func (expl *Explanation) String() string { + js, err := json.MarshalIndent(expl, "", " ") + if err != nil { + return fmt.Sprintf("error serializing explanation to json: %v", err) + } + return string(js) +} + +func (expl *Explanation) Size() int { + sizeInBytes := reflectStaticSizeExplanation + size.SizeOfPtr + + len(expl.Message) + + for _, entry := range expl.Children { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} + +// MergeExpl merges two explanations into one. +// If either explanation is nil, the other is returned. +// If the first explanation is already a merged explanation, +// the second explanation is appended to its children. +// Otherwise, a new merged explanation is created +// with the two explanations as its children. +func (expl *Explanation) MergeWith(other *Explanation) *Explanation { + if expl == nil { + return other + } + if other == nil || expl == other { + return expl + } + + newScore := expl.Value + other.Value + + // if both are merged explanations, combine children + if expl.Message == MergedExplMessage && other.Message == MergedExplMessage { + expl.Value = newScore + expl.Children = append(expl.Children, other.Children...) + return expl + } + + // atleast one is not a merged explanation see which one it is + // if expl is merged, append other + if expl.Message == MergedExplMessage { + // append other as a child to first + expl.Value = newScore + expl.Children = append(expl.Children, other) + return expl + } + + // if other is merged, append expl + if other.Message == MergedExplMessage { + other.Value = newScore + other.Children = append(other.Children, expl) + return other + } + // create a new explanation to hold the merged one + rv := &Explanation{ + Value: expl.Value + other.Value, + Message: MergedExplMessage, + Children: []*Explanation{expl, other}, + } + return rv +} diff --git a/search/facet/facet_builder_datetime.go b/search/facet/facet_builder_datetime.go new file mode 100644 index 0000000..9fe4cf4 --- /dev/null +++ b/search/facet/facet_builder_datetime.go @@ -0,0 +1,163 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package facet + +import ( + "reflect" + "sort" + "time" + + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" +) + +var ( + reflectStaticSizeDateTimeFacetBuilder int + reflectStaticSizedateTimeRange int +) + +func init() { + var dtfb DateTimeFacetBuilder + reflectStaticSizeDateTimeFacetBuilder = int(reflect.TypeOf(dtfb).Size()) + var dtr dateTimeRange + reflectStaticSizedateTimeRange = int(reflect.TypeOf(dtr).Size()) +} + +type dateTimeRange struct { + start time.Time + end time.Time +} + +type DateTimeFacetBuilder struct { + size int + field string + termsCount map[string]int + total int + missing int + ranges map[string]*dateTimeRange + sawValue bool +} + +func NewDateTimeFacetBuilder(field string, size int) *DateTimeFacetBuilder { + return &DateTimeFacetBuilder{ + size: size, + field: field, + termsCount: make(map[string]int), + ranges: make(map[string]*dateTimeRange, 0), + } +} + +func (fb *DateTimeFacetBuilder) Size() int { + sizeInBytes := reflectStaticSizeDateTimeFacetBuilder + size.SizeOfPtr + + len(fb.field) + + for k := range fb.termsCount { + sizeInBytes += size.SizeOfString + len(k) + + size.SizeOfInt + } + + for k := range fb.ranges { + sizeInBytes += size.SizeOfString + len(k) + + size.SizeOfPtr + reflectStaticSizedateTimeRange + } + + return sizeInBytes +} + +func (fb *DateTimeFacetBuilder) AddRange(name string, start, end time.Time) { + r := dateTimeRange{ + start: start, + end: end, + } + fb.ranges[name] = &r +} + +func (fb *DateTimeFacetBuilder) Field() string { + return fb.field +} + +func (fb *DateTimeFacetBuilder) UpdateVisitor(term []byte) { + fb.sawValue = true + // only consider the values which are shifted 0 + prefixCoded := numeric.PrefixCoded(term) + shift, err := prefixCoded.Shift() + if err == nil && shift == 0 { + i64, err := prefixCoded.Int64() + if err == nil { + t := time.Unix(0, i64) + + // look at each of the ranges for a match + for rangeName, r := range fb.ranges { + if (r.start.IsZero() || t.After(r.start) || t.Equal(r.start)) && (r.end.IsZero() || t.Before(r.end)) { + fb.termsCount[rangeName] = fb.termsCount[rangeName] + 1 + fb.total++ + } + } + } + } +} + +func (fb *DateTimeFacetBuilder) StartDoc() { + fb.sawValue = false +} + +func (fb *DateTimeFacetBuilder) EndDoc() { + if !fb.sawValue { + fb.missing++ + } +} + +func (fb *DateTimeFacetBuilder) Result() *search.FacetResult { + rv := search.FacetResult{ + Field: fb.field, + Total: fb.total, + Missing: fb.missing, + } + + rv.DateRanges = make([]*search.DateRangeFacet, 0, len(fb.termsCount)) + + for term, count := range fb.termsCount { + dateRange := fb.ranges[term] + tf := &search.DateRangeFacet{ + Name: term, + Count: count, + } + if !dateRange.start.IsZero() { + start := dateRange.start.Format(time.RFC3339Nano) + tf.Start = &start + } + if !dateRange.end.IsZero() { + end := dateRange.end.Format(time.RFC3339Nano) + tf.End = &end + } + rv.DateRanges = append(rv.DateRanges, tf) + } + + sort.Sort(rv.DateRanges) + + // we now have the list of the top N facets + if fb.size < len(rv.DateRanges) { + rv.DateRanges = rv.DateRanges[:fb.size] + } + + notOther := 0 + for _, nr := range rv.DateRanges { + notOther += nr.Count + } + rv.Other = fb.total - notOther + + return &rv +} diff --git a/search/facet/facet_builder_numeric.go b/search/facet/facet_builder_numeric.go new file mode 100644 index 0000000..1383942 --- /dev/null +++ b/search/facet/facet_builder_numeric.go @@ -0,0 +1,157 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package facet + +import ( + "reflect" + "sort" + + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" +) + +var ( + reflectStaticSizeNumericFacetBuilder int + reflectStaticSizenumericRange int +) + +func init() { + var nfb NumericFacetBuilder + reflectStaticSizeNumericFacetBuilder = int(reflect.TypeOf(nfb).Size()) + var nr numericRange + reflectStaticSizenumericRange = int(reflect.TypeOf(nr).Size()) +} + +type numericRange struct { + min *float64 + max *float64 +} + +type NumericFacetBuilder struct { + size int + field string + termsCount map[string]int + total int + missing int + ranges map[string]*numericRange + sawValue bool +} + +func NewNumericFacetBuilder(field string, size int) *NumericFacetBuilder { + return &NumericFacetBuilder{ + size: size, + field: field, + termsCount: make(map[string]int), + ranges: make(map[string]*numericRange, 0), + } +} + +func (fb *NumericFacetBuilder) Size() int { + sizeInBytes := reflectStaticSizeNumericFacetBuilder + size.SizeOfPtr + + len(fb.field) + + for k := range fb.termsCount { + sizeInBytes += size.SizeOfString + len(k) + + size.SizeOfInt + } + + for k := range fb.ranges { + sizeInBytes += size.SizeOfString + len(k) + + size.SizeOfPtr + reflectStaticSizenumericRange + } + + return sizeInBytes +} + +func (fb *NumericFacetBuilder) AddRange(name string, min, max *float64) { + r := numericRange{ + min: min, + max: max, + } + fb.ranges[name] = &r +} + +func (fb *NumericFacetBuilder) Field() string { + return fb.field +} + +func (fb *NumericFacetBuilder) UpdateVisitor(term []byte) { + fb.sawValue = true + // only consider the values which are shifted 0 + prefixCoded := numeric.PrefixCoded(term) + shift, err := prefixCoded.Shift() + if err == nil && shift == 0 { + i64, err := prefixCoded.Int64() + if err == nil { + f64 := numeric.Int64ToFloat64(i64) + + // look at each of the ranges for a match + for rangeName, r := range fb.ranges { + if (r.min == nil || f64 >= *r.min) && (r.max == nil || f64 < *r.max) { + fb.termsCount[rangeName] = fb.termsCount[rangeName] + 1 + fb.total++ + } + } + } + } +} + +func (fb *NumericFacetBuilder) StartDoc() { + fb.sawValue = false +} + +func (fb *NumericFacetBuilder) EndDoc() { + if !fb.sawValue { + fb.missing++ + } +} + +func (fb *NumericFacetBuilder) Result() *search.FacetResult { + rv := search.FacetResult{ + Field: fb.field, + Total: fb.total, + Missing: fb.missing, + } + + rv.NumericRanges = make([]*search.NumericRangeFacet, 0, len(fb.termsCount)) + + for term, count := range fb.termsCount { + numericRange := fb.ranges[term] + tf := &search.NumericRangeFacet{ + Name: term, + Count: count, + Min: numericRange.min, + Max: numericRange.max, + } + + rv.NumericRanges = append(rv.NumericRanges, tf) + } + + sort.Sort(rv.NumericRanges) + + // we now have the list of the top N facets + if fb.size < len(rv.NumericRanges) { + rv.NumericRanges = rv.NumericRanges[:fb.size] + } + + notOther := 0 + for _, nr := range rv.NumericRanges { + notOther += nr.Count + } + rv.Other = fb.total - notOther + + return &rv +} diff --git a/search/facet/facet_builder_numeric_test.go b/search/facet/facet_builder_numeric_test.go new file mode 100644 index 0000000..dea2e04 --- /dev/null +++ b/search/facet/facet_builder_numeric_test.go @@ -0,0 +1,64 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package facet + +import ( + "strconv" + "testing" + + "github.com/blevesearch/bleve/v2/numeric" +) + +var pcodedvalues []numeric.PrefixCoded + +func init() { + pcodedvalues = []numeric.PrefixCoded{{0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, {0x20, 0x0, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f}, {0x20, 0x0, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7a, 0x1d, 0xa}, {0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x1, 0x16, 0x9, 0x4a, 0x7b}} +} + +func BenchmarkNumericFacet10(b *testing.B) { + numericFacetN(b, 10) +} + +func BenchmarkNumericFacet100(b *testing.B) { + numericFacetN(b, 100) +} + +func BenchmarkNumericFacet1000(b *testing.B) { + numericFacetN(b, 1000) +} + +func numericFacetN(b *testing.B, numTerms int) { + field := "test" + nfb := NewNumericFacetBuilder(field, numTerms) + min, max := 0.0, 9999999998.0 + + for i := 0; i <= numTerms; i++ { + max++ + min-- + + nfb.AddRange("rangename"+strconv.Itoa(i), &min, &max) + + for _, pv := range pcodedvalues { + nfb.StartDoc() + nfb.UpdateVisitor(pv) + nfb.EndDoc() + } + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + nfb.Result() + } +} diff --git a/search/facet/facet_builder_terms.go b/search/facet/facet_builder_terms.go new file mode 100644 index 0000000..43cfcad --- /dev/null +++ b/search/facet/facet_builder_terms.go @@ -0,0 +1,157 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package facet + +import ( + "bytes" + "reflect" + "regexp" + "sort" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" +) + +var reflectStaticSizeTermsFacetBuilder int + +func init() { + var tfb TermsFacetBuilder + reflectStaticSizeTermsFacetBuilder = int(reflect.TypeOf(tfb).Size()) +} + +type TermsFacetBuilder struct { + size int + field string + prefixBytes []byte + regex *regexp.Regexp + termsCount map[string]int + total int + missing int + sawValue bool +} + +func NewTermsFacetBuilder(field string, size int) *TermsFacetBuilder { + return &TermsFacetBuilder{ + size: size, + field: field, + termsCount: make(map[string]int), + } +} + +func (fb *TermsFacetBuilder) Size() int { + sizeInBytes := reflectStaticSizeTermsFacetBuilder + size.SizeOfPtr + + len(fb.field) + + len(fb.prefixBytes) + + size.SizeOfPtr // regex pointer (does not include actual regexp.Regexp object size) + + // Estimate regex object size if present. + if fb.regex != nil { + // This is only the static size of regexp.Regexp struct, not including heap allocations. + sizeInBytes += int(reflect.TypeOf(*fb.regex).Size()) + // NOTE: Actual memory usage of regexp.Regexp may be higher due to internal allocations. + } + + for k := range fb.termsCount { + sizeInBytes += size.SizeOfString + len(k) + + size.SizeOfInt + } + + return sizeInBytes +} + +func (fb *TermsFacetBuilder) Field() string { + return fb.field +} + +// SetPrefixFilter sets the prefix filter for term facets. +func (fb *TermsFacetBuilder) SetPrefixFilter(prefix string) { + if prefix != "" { + fb.prefixBytes = []byte(prefix) + } else { + fb.prefixBytes = nil + } +} + +// SetRegexFilter sets the compiled regex filter for term facets. +func (fb *TermsFacetBuilder) SetRegexFilter(regex *regexp.Regexp) { + fb.regex = regex +} + +func (fb *TermsFacetBuilder) UpdateVisitor(term []byte) { + // Total represents all terms visited, not just matching ones. + // This is necessary for the "Other" calculation. + fb.total++ + + // Fast prefix check on []byte - zero allocation + if len(fb.prefixBytes) > 0 && !bytes.HasPrefix(term, fb.prefixBytes) { + return + } + + // Fast regex check on []byte - zero allocation + if fb.regex != nil && !fb.regex.Match(term) { + return + } + + // Only convert to string if term matches filters + termStr := string(term) + fb.sawValue = true + fb.termsCount[termStr] = fb.termsCount[termStr] + 1 +} + +func (fb *TermsFacetBuilder) StartDoc() { + fb.sawValue = false +} + +func (fb *TermsFacetBuilder) EndDoc() { + if !fb.sawValue { + fb.missing++ + } +} + +func (fb *TermsFacetBuilder) Result() *search.FacetResult { + rv := search.FacetResult{ + Field: fb.field, + Total: fb.total, + Missing: fb.missing, + } + + rv.Terms = &search.TermFacets{} + + for term, count := range fb.termsCount { + tf := &search.TermFacet{ + Term: term, + Count: count, + } + + rv.Terms.Add(tf) + } + + sort.Sort(rv.Terms) + + // we now have the list of the top N facets + trimTopN := fb.size + if trimTopN > rv.Terms.Len() { + trimTopN = rv.Terms.Len() + } + rv.Terms.TrimToTopN(trimTopN) + + notOther := 0 + for _, tf := range rv.Terms.Terms() { + notOther += tf.Count + } + rv.Other = fb.total - notOther + + return &rv +} diff --git a/search/facet/facet_builder_terms_test.go b/search/facet/facet_builder_terms_test.go new file mode 100644 index 0000000..3ed2fcc --- /dev/null +++ b/search/facet/facet_builder_terms_test.go @@ -0,0 +1,290 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package facet + +import ( + "os" + "regexp" + "testing" +) + +var terms []string + +func init() { + wsRegexp := regexp.MustCompile(`\W+`) + input, err := os.ReadFile("benchmark_data.txt") + if err != nil { + panic(err) + } + terms = wsRegexp.Split(string(input), -1) +} + +func BenchmarkTermsFacet10(b *testing.B) { + termsFacetN(b, 10) +} + +func BenchmarkTermsFacet100(b *testing.B) { + termsFacetN(b, 100) +} + +func BenchmarkTermsFacet1000(b *testing.B) { + termsFacetN(b, 1000) +} + +func BenchmarkTermsFacet10000(b *testing.B) { + termsFacetN(b, 10000) +} + +// func BenchmarkTermsFacet100000(b *testing.B) { +// termsFacetN(b, 100000) +// } + +func termsFacetN(b *testing.B, numTerms int) { + field := "test" + termsLen := len(terms) + tfb := NewTermsFacetBuilder(field, 3) + i := 0 + for len(tfb.termsCount) < numTerms && i <= termsLen { + j := i % termsLen + term := terms[j] + tfb.StartDoc() + tfb.UpdateVisitor([]byte(term)) + tfb.EndDoc() + i++ + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + tfb.Result() + } +} + +func TestTermsFacetPrefix(t *testing.T) { + field := "category" + tfb := NewTermsFacetBuilder(field, 10) + tfb.SetPrefixFilter("prod-") + + // Add terms with various prefixes + terms := []string{ + "prod-server", + "prod-database", + "dev-server", + "dev-database", + "test-server", + "prod-cache", + } + + for _, term := range terms { + tfb.StartDoc() + tfb.UpdateVisitor([]byte(term)) + tfb.EndDoc() + } + + result := tfb.Result() + + // Should only have terms with "prod-" prefix + if result.Terms.Len() != 3 { + t.Fatalf("expected 3 matching terms, got %d", result.Terms.Len()) + } + + // Verify the terms are correct + expectedTerms := map[string]bool{ + "prod-server": true, + "prod-database": true, + "prod-cache": true, + } + + for _, facet := range result.Terms.Terms() { + if !expectedTerms[facet.Term] { + t.Errorf("unexpected term in results: %s", facet.Term) + } + if facet.Count != 1 { + t.Errorf("expected count 1 for %s, got %d", facet.Term, facet.Count) + } + } + + // Total should include all terms (matching + non-matching) + if result.Total != 6 { + t.Errorf("expected total 6, got %d", result.Total) + } + + // Other should be 3 (the non-matching terms) + if result.Other != 3 { + t.Errorf("expected other 3, got %d", result.Other) + } +} + +func TestTermsFacetRegex(t *testing.T) { + field := "product_code" + // Match pattern: ABC-#### (3 letters, dash, 4 digits) - pattern: ^[A-Z]{3}-\\d{4}$ + tfb := NewTermsFacetBuilder(field, 10) + regex, err := regexp.Compile("^[A-Z]{3}-\\d{4}$") + if err != nil { + t.Fatal(err) + } + tfb.SetRegexFilter(regex) + + // Add terms with various formats + terms := []string{ + "ABC-1234", + "XYZ-5678", + "ABC-999", // too few digits + "ABCD-1234", // too many letters + "ABC-ABCD", // letters instead of digits + "DEF-0000", + } + + for _, term := range terms { + tfb.StartDoc() + tfb.UpdateVisitor([]byte(term)) + tfb.EndDoc() + } + + result := tfb.Result() + + // Should only have 3 terms matching the pattern + if result.Terms.Len() != 3 { + t.Fatalf("expected 3 matching terms, got %d", result.Terms.Len()) + } + + // Verify the terms are correct + expectedTerms := map[string]bool{ + "ABC-1234": true, + "XYZ-5678": true, + "DEF-0000": true, + } + + for _, facet := range result.Terms.Terms() { + if !expectedTerms[facet.Term] { + t.Errorf("unexpected term in results: %s", facet.Term) + } + if facet.Count != 1 { + t.Errorf("expected count 1 for %s, got %d", facet.Term, facet.Count) + } + } + + // Total should include all terms + if result.Total != 6 { + t.Errorf("expected total 6, got %d", result.Total) + } + + // Other should be 3 (the non-matching terms) + if result.Other != 3 { + t.Errorf("expected other 3, got %d", result.Other) + } +} + +func TestTermsFacetPrefixAndRegex(t *testing.T) { + field := "tag" + // Both prefix "env:" and regex pattern for prod/staging only + tfb := NewTermsFacetBuilder(field, 10) + tfb.SetPrefixFilter("env:") + regex, err := regexp.Compile("^env:(prod|staging)$") + if err != nil { + t.Fatal(err) + } + tfb.SetRegexFilter(regex) + + // Add various terms + terms := []string{ + "env:prod", + "env:staging", + "env:dev", // has prefix but doesn't match regex + "env:test", // has prefix but doesn't match regex + "type:server", // no prefix + "env:prod", // duplicate + "env:staging", // duplicate + } + + for _, term := range terms { + tfb.StartDoc() + tfb.UpdateVisitor([]byte(term)) + tfb.EndDoc() + } + + result := tfb.Result() + + // Should only have 2 unique terms (env:prod and env:staging) + if result.Terms.Len() != 2 { + t.Fatalf("expected 2 matching terms, got %d", result.Terms.Len()) + } + + // Verify the terms and counts + termCounts := make(map[string]int) + for _, facet := range result.Terms.Terms() { + termCounts[facet.Term] = facet.Count + } + + if termCounts["env:prod"] != 2 { + t.Errorf("expected count 2 for env:prod, got %d", termCounts["env:prod"]) + } + if termCounts["env:staging"] != 2 { + t.Errorf("expected count 2 for env:staging, got %d", termCounts["env:staging"]) + } + + // Total should be all 7 terms + if result.Total != 7 { + t.Errorf("expected total 7, got %d", result.Total) + } + + // Other should be 3 (env:dev, env:test, type:server) + if result.Other != 3 { + t.Errorf("expected other 3, got %d", result.Other) + } +} + +func TestTermsFacetInvalidRegex(t *testing.T) { + // Invalid regex pattern (unmatched bracket) + _, err := regexp.Compile("[invalid") + if err == nil { + t.Fatal("expected error for invalid regex, got nil") + } +} + +func TestTermsFacetNoFilter(t *testing.T) { + field := "tag" + tfb := NewTermsFacetBuilder(field, 2) + + terms := []string{"apple", "banana", "cherry", "apple"} + + for _, term := range terms { + tfb.StartDoc() + tfb.UpdateVisitor([]byte(term)) + tfb.EndDoc() + } + + result := tfb.Result() + + // Should return top 2 by count + if result.Terms.Len() != 2 { + t.Fatalf("expected 2 terms, got %d", result.Terms.Len()) + } + + // Apple should be first with count 2 + facets := result.Terms.Terms() + if facets[0].Term != "apple" || facets[0].Count != 2 { + t.Errorf("expected apple with count 2, got %s with count %d", facets[0].Term, facets[0].Count) + } + + // Total should be 4 + if result.Total != 4 { + t.Errorf("expected total 4, got %d", result.Total) + } + + // Other should be 1 (cherry was trimmed) + if result.Other != 1 { + t.Errorf("expected other 1, got %d", result.Other) + } +} diff --git a/search/facets_builder.go b/search/facets_builder.go new file mode 100644 index 0000000..4b1f2db --- /dev/null +++ b/search/facets_builder.go @@ -0,0 +1,411 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import ( + "reflect" + "sort" + + "github.com/blevesearch/bleve/v2/size" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeFacetsBuilder int +var reflectStaticSizeFacetResult int +var reflectStaticSizeTermFacet int +var reflectStaticSizeNumericRangeFacet int +var reflectStaticSizeDateRangeFacet int + +func init() { + var fb FacetsBuilder + reflectStaticSizeFacetsBuilder = int(reflect.TypeOf(fb).Size()) + var fr FacetResult + reflectStaticSizeFacetResult = int(reflect.TypeOf(fr).Size()) + var tf TermFacet + reflectStaticSizeTermFacet = int(reflect.TypeOf(tf).Size()) + var nrf NumericRangeFacet + reflectStaticSizeNumericRangeFacet = int(reflect.TypeOf(nrf).Size()) + var drf DateRangeFacet + reflectStaticSizeDateRangeFacet = int(reflect.TypeOf(drf).Size()) +} + +type FacetBuilder interface { + StartDoc() + UpdateVisitor(term []byte) + EndDoc() + + Result() *FacetResult + Field() string + + Size() int +} + +type FacetsBuilder struct { + indexReader index.IndexReader + facetNames []string + facets []FacetBuilder + facetsByField map[string][]FacetBuilder + fields []string +} + +func NewFacetsBuilder(indexReader index.IndexReader) *FacetsBuilder { + return &FacetsBuilder{ + indexReader: indexReader, + } +} + +func (fb *FacetsBuilder) Size() int { + sizeInBytes := reflectStaticSizeFacetsBuilder + size.SizeOfPtr + + for k, v := range fb.facets { + sizeInBytes += size.SizeOfString + v.Size() + len(fb.facetNames[k]) + } + + for _, entry := range fb.fields { + sizeInBytes += size.SizeOfString + len(entry) + } + + return sizeInBytes +} + +func (fb *FacetsBuilder) Add(name string, facetBuilder FacetBuilder) { + if fb.facetsByField == nil { + fb.facetsByField = map[string][]FacetBuilder{} + } + + fb.facetNames = append(fb.facetNames, name) + fb.facets = append(fb.facets, facetBuilder) + fb.facetsByField[facetBuilder.Field()] = append(fb.facetsByField[facetBuilder.Field()], facetBuilder) + fb.fields = append(fb.fields, facetBuilder.Field()) +} + +func (fb *FacetsBuilder) RequiredFields() []string { + return fb.fields +} + +func (fb *FacetsBuilder) StartDoc() { + for _, facetBuilder := range fb.facets { + facetBuilder.StartDoc() + } +} + +func (fb *FacetsBuilder) EndDoc() { + for _, facetBuilder := range fb.facets { + facetBuilder.EndDoc() + } +} + +func (fb *FacetsBuilder) UpdateVisitor(field string, term []byte) { + if facetBuilders, ok := fb.facetsByField[field]; ok { + for _, facetBuilder := range facetBuilders { + facetBuilder.UpdateVisitor(term) + } + } +} + +type TermFacet struct { + Term string `json:"term"` + Count int `json:"count"` +} + +type TermFacets struct { + termFacets []*TermFacet + termLookup map[string]*TermFacet +} + +func (tf *TermFacets) Terms() []*TermFacet { + if tf == nil { + return []*TermFacet{} + } + return tf.termFacets +} + +func (tf *TermFacets) TrimToTopN(n int) { + tf.termFacets = tf.termFacets[:n] +} + +func (tf *TermFacets) Add(termFacets ...*TermFacet) { + for _, termFacet := range termFacets { + if tf.termLookup == nil { + tf.termLookup = map[string]*TermFacet{} + } + + if term, ok := tf.termLookup[termFacet.Term]; ok { + term.Count += termFacet.Count + return + } + + // if we got here it wasn't already in the existing terms + tf.termFacets = append(tf.termFacets, termFacet) + tf.termLookup[termFacet.Term] = termFacet + } +} + +func (tf *TermFacets) Len() int { + // Handle case where *TermFacets is not fully initialized in index_impl.go.init() + if tf == nil { + return 0 + } + + return len(tf.termFacets) +} +func (tf *TermFacets) Swap(i, j int) { + tf.termFacets[i], tf.termFacets[j] = tf.termFacets[j], tf.termFacets[i] +} +func (tf *TermFacets) Less(i, j int) bool { + if tf.termFacets[i].Count == tf.termFacets[j].Count { + return tf.termFacets[i].Term < tf.termFacets[j].Term + } + return tf.termFacets[i].Count > tf.termFacets[j].Count +} + +// TermFacets used to be a type alias for []*TermFacet. +// To maintain backwards compatibility, we have to implement custom +// JSON marshalling. +func (tf *TermFacets) MarshalJSON() ([]byte, error) { + return util.MarshalJSON(tf.termFacets) +} + +func (tf *TermFacets) UnmarshalJSON(b []byte) error { + termFacets := []*TermFacet{} + err := util.UnmarshalJSON(b, &termFacets) + if err != nil { + return err + } + + for _, termFacet := range termFacets { + tf.Add(termFacet) + } + + return nil +} + +type NumericRangeFacet struct { + Name string `json:"name"` + Min *float64 `json:"min,omitempty"` + Max *float64 `json:"max,omitempty"` + Count int `json:"count"` +} + +func (nrf *NumericRangeFacet) Same(other *NumericRangeFacet) bool { + if nrf.Min == nil && other.Min != nil { + return false + } + if nrf.Min != nil && other.Min == nil { + return false + } + if nrf.Min != nil && other.Min != nil && *nrf.Min != *other.Min { + return false + } + if nrf.Max == nil && other.Max != nil { + return false + } + if nrf.Max != nil && other.Max == nil { + return false + } + if nrf.Max != nil && other.Max != nil && *nrf.Max != *other.Max { + return false + } + + return true +} + +type NumericRangeFacets []*NumericRangeFacet + +func (nrf NumericRangeFacets) Add(numericRangeFacet *NumericRangeFacet) NumericRangeFacets { + for _, existingNr := range nrf { + if numericRangeFacet.Same(existingNr) { + existingNr.Count += numericRangeFacet.Count + return nrf + } + } + // if we got here it wasn't already in the existing terms + nrf = append(nrf, numericRangeFacet) + return nrf +} + +func (nrf NumericRangeFacets) Len() int { return len(nrf) } +func (nrf NumericRangeFacets) Swap(i, j int) { nrf[i], nrf[j] = nrf[j], nrf[i] } +func (nrf NumericRangeFacets) Less(i, j int) bool { + if nrf[i].Count == nrf[j].Count { + return nrf[i].Name < nrf[j].Name + } + return nrf[i].Count > nrf[j].Count +} + +type DateRangeFacet struct { + Name string `json:"name"` + Start *string `json:"start,omitempty"` + End *string `json:"end,omitempty"` + Count int `json:"count"` +} + +func (drf *DateRangeFacet) Same(other *DateRangeFacet) bool { + if drf.Start == nil && other.Start != nil { + return false + } + if drf.Start != nil && other.Start == nil { + return false + } + if drf.Start != nil && other.Start != nil && *drf.Start != *other.Start { + return false + } + if drf.End == nil && other.End != nil { + return false + } + if drf.End != nil && other.End == nil { + return false + } + if drf.End != nil && other.End != nil && *drf.End != *other.End { + return false + } + + return true +} + +type DateRangeFacets []*DateRangeFacet + +func (drf DateRangeFacets) Add(dateRangeFacet *DateRangeFacet) DateRangeFacets { + for _, existingDr := range drf { + if dateRangeFacet.Same(existingDr) { + existingDr.Count += dateRangeFacet.Count + return drf + } + } + // if we got here it wasn't already in the existing terms + drf = append(drf, dateRangeFacet) + return drf +} + +func (drf DateRangeFacets) Len() int { return len(drf) } +func (drf DateRangeFacets) Swap(i, j int) { drf[i], drf[j] = drf[j], drf[i] } +func (drf DateRangeFacets) Less(i, j int) bool { + if drf[i].Count == drf[j].Count { + return drf[i].Name < drf[j].Name + } + return drf[i].Count > drf[j].Count +} + +type FacetResult struct { + Field string `json:"field"` + Total int `json:"total"` + Missing int `json:"missing"` + Other int `json:"other"` + Terms *TermFacets `json:"terms,omitempty"` + NumericRanges NumericRangeFacets `json:"numeric_ranges,omitempty"` + DateRanges DateRangeFacets `json:"date_ranges,omitempty"` +} + +func (fr *FacetResult) Size() int { + return reflectStaticSizeFacetResult + size.SizeOfPtr + + len(fr.Field) + + fr.Terms.Len()*(reflectStaticSizeTermFacet+size.SizeOfPtr) + + len(fr.NumericRanges)*(reflectStaticSizeNumericRangeFacet+size.SizeOfPtr) + + len(fr.DateRanges)*(reflectStaticSizeDateRangeFacet+size.SizeOfPtr) +} + +func (fr *FacetResult) Merge(other *FacetResult) { + fr.Total += other.Total + fr.Missing += other.Missing + fr.Other += other.Other + if other.Terms != nil { + if fr.Terms == nil { + fr.Terms = other.Terms + return + } + for _, term := range other.Terms.termFacets { + fr.Terms.Add(term) + } + } + if other.NumericRanges != nil { + if fr.NumericRanges == nil { + fr.NumericRanges = other.NumericRanges + return + } + for _, nr := range other.NumericRanges { + fr.NumericRanges = fr.NumericRanges.Add(nr) + } + } + if other.DateRanges != nil { + if fr.DateRanges == nil { + fr.DateRanges = other.DateRanges + return + } + for _, dr := range other.DateRanges { + fr.DateRanges = fr.DateRanges.Add(dr) + } + } +} + +func (fr *FacetResult) Fixup(size int) { + if fr.Terms != nil { + sort.Sort(fr.Terms) + if fr.Terms.Len() > size { + moveToOther := fr.Terms.termFacets[size:] + for _, mto := range moveToOther { + fr.Other += mto.Count + } + fr.Terms.termFacets = fr.Terms.termFacets[0:size] + } + } else if fr.NumericRanges != nil { + sort.Sort(fr.NumericRanges) + if len(fr.NumericRanges) > size { + moveToOther := fr.NumericRanges[size:] + for _, mto := range moveToOther { + fr.Other += mto.Count + } + fr.NumericRanges = fr.NumericRanges[0:size] + } + } else if fr.DateRanges != nil { + sort.Sort(fr.DateRanges) + if len(fr.DateRanges) > size { + moveToOther := fr.DateRanges[size:] + for _, mto := range moveToOther { + fr.Other += mto.Count + } + fr.DateRanges = fr.DateRanges[0:size] + } + } +} + +type FacetResults map[string]*FacetResult + +func (fr FacetResults) Merge(other FacetResults) { + for name, oFacetResult := range other { + facetResult, ok := fr[name] + if ok { + facetResult.Merge(oFacetResult) + } else { + fr[name] = oFacetResult + } + } +} + +func (fr FacetResults) Fixup(name string, size int) { + facetResult, ok := fr[name] + if ok { + facetResult.Fixup(size) + } +} + +func (fb *FacetsBuilder) Results() FacetResults { + fr := make(FacetResults) + for i, facetBuilder := range fb.facets { + facetResult := facetBuilder.Result() + fr[fb.facetNames[i]] = facetResult + } + return fr +} diff --git a/search/facets_builder_test.go b/search/facets_builder_test.go new file mode 100644 index 0000000..6815b1a --- /dev/null +++ b/search/facets_builder_test.go @@ -0,0 +1,424 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import ( + "fmt" + "reflect" + "testing" +) + +func TestTermFacetResultsMerge(t *testing.T) { + type testCase struct { + // Input + frs1 FacetResults // first facet results + frs2 FacetResults // second facet results (to be merged into first) + fixups map[string]int // {facetName:size} (to be applied after merge) + + // Expected output + expFrs FacetResults // facet results after merge and fixup + } + + tests := []*testCase{ + func() *testCase { + rv := &testCase{} + + rv.frs1 = FacetResults{ + "types": &FacetResult{ + Field: "type", + Total: 100, + Missing: 25, + Other: 25, + Terms: func() *TermFacets { + tfs := &TermFacets{} + tfs.Add( + &TermFacet{ + Term: "blog", + Count: 25, + }, + &TermFacet{ + Term: "comment", + Count: 24, + }, + &TermFacet{ + Term: "feedback", + Count: 1, + }, + ) + return tfs + }(), + }, + "categories": &FacetResult{ + Field: "category", + Total: 97, + Missing: 22, + Other: 15, + Terms: func() *TermFacets { + tfs := &TermFacets{} + tfs.Add( + &TermFacet{ + Term: "clothing", + Count: 35, + }, + &TermFacet{ + Term: "electronics", + Count: 25, + }, + ) + return tfs + }(), + }, + } + rv.frs2 = FacetResults{ + "types": &FacetResult{ + Field: "type", + Total: 100, + Missing: 25, + Other: 25, + Terms: func() *TermFacets { + tfs := &TermFacets{} + tfs.Add( + &TermFacet{ + Term: "blog", + Count: 25, + }, + &TermFacet{ + Term: "comment", + Count: 22, + }, + &TermFacet{ + Term: "flag", + Count: 3, + }, + ) + return tfs + }(), + }, + } + rv.fixups = map[string]int{ + "types": 3, // we want top 3 terms based on count + } + + rv.expFrs = FacetResults{ + "types": &FacetResult{ + Field: "type", + Total: 200, + Missing: 50, + Other: 51, + Terms: &TermFacets{ + termFacets: []*TermFacet{ + { + Term: "blog", + Count: 50, + }, + { + Term: "comment", + Count: 46, + }, + { + Term: "flag", + Count: 3, + }, + }, + }, + }, + "categories": rv.frs1["categories"], + } + + return rv + }(), + func() *testCase { + rv := &testCase{} + + rv.frs1 = FacetResults{ + "facetName": &FacetResult{ + Field: "docField", + Total: 0, + Missing: 0, + Other: 0, + Terms: nil, + }, + } + rv.frs2 = FacetResults{ + "facetName": &FacetResult{ + Field: "docField", + Total: 3, + Missing: 0, + Other: 0, + Terms: &TermFacets{ + termFacets: []*TermFacet{ + { + Term: "firstTerm", + Count: 1, + }, + { + Term: "secondTerm", + Count: 2, + }, + }, + }, + }, + } + rv.fixups = map[string]int{ + "facetName": 1, + } + + rv.expFrs = FacetResults{ + "facetName": &FacetResult{ + Field: "docField", + Total: 3, + Missing: 0, + Other: 1, + Terms: &TermFacets{ + termFacets: []*TermFacet{ + { + Term: "secondTerm", + Count: 2, + }, + }, + }, + }, + } + return rv + }(), + } + + for tcIdx, tc := range tests { + t.Run(fmt.Sprintf("T#%d", tcIdx), func(t *testing.T) { + tc.frs1.Merge(tc.frs2) + for facetName, size := range tc.fixups { + tc.frs1.Fixup(facetName, size) + } + + // clear termLookup, so we can compare the facet results + for _, fr := range tc.frs1 { + if fr.Terms != nil { + fr.Terms.termLookup = nil + } + } + + if !reflect.DeepEqual(tc.frs1, tc.expFrs) { + t.Errorf("expected %v, got %v", tc.expFrs, tc.frs1) + } + }) + } +} + +func TestNumericFacetResultsMerge(t *testing.T) { + + lowmed := 3.0 + medhi := 6.0 + hihigher := 9.0 + + // why second copy? the pointers may be different, but values the same + lowmed2 := 3.0 + medhi2 := 6.0 + hihigher2 := 9.0 + + fr1 := &FacetResult{ + Field: "rating", + Total: 100, + Missing: 25, + Other: 25, + NumericRanges: []*NumericRangeFacet{ + { + Name: "low", + Max: &lowmed, + Count: 25, + }, + { + Name: "med", + Count: 24, + Max: &lowmed, + Min: &medhi, + }, + { + Name: "hi", + Count: 1, + Min: &medhi, + Max: &hihigher, + }, + }, + } + frs1 := FacetResults{ + "ratings": fr1, + } + + fr2 := &FacetResult{ + Field: "rating", + Total: 100, + Missing: 25, + Other: 25, + NumericRanges: []*NumericRangeFacet{ + { + Name: "low", + Max: &lowmed2, + Count: 25, + }, + { + Name: "med", + Max: &lowmed2, + Min: &medhi2, + Count: 22, + }, + { + Name: "highest", + Min: &hihigher2, + Count: 3, + }, + }, + } + frs2 := FacetResults{ + "ratings": fr2, + } + + expectedFr := &FacetResult{ + Field: "rating", + Total: 200, + Missing: 50, + Other: 51, + NumericRanges: []*NumericRangeFacet{ + { + Name: "low", + Count: 50, + Max: &lowmed, + }, + { + Name: "med", + Max: &lowmed, + Min: &medhi, + Count: 46, + }, + { + Name: "highest", + Min: &hihigher, + Count: 3, + }, + }, + } + expectedFrs := FacetResults{ + "ratings": expectedFr, + } + + frs1.Merge(frs2) + frs1.Fixup("ratings", 3) + if !reflect.DeepEqual(frs1, expectedFrs) { + t.Errorf("expected %#v, got %#v", expectedFrs, frs1) + } +} + +func TestDateFacetResultsMerge(t *testing.T) { + + lowmed := "2010-01-01" + medhi := "2011-01-01" + hihigher := "2012-01-01" + + // why second copy? the pointer are to strings done by date time parsing + // inside the facet generation, so comparing pointers will not work + lowmed2 := "2010-01-01" + medhi2 := "2011-01-01" + hihigher2 := "2012-01-01" + + fr1 := &FacetResult{ + Field: "birthday", + Total: 100, + Missing: 25, + Other: 25, + DateRanges: []*DateRangeFacet{ + { + Name: "low", + End: &lowmed, + Count: 25, + }, + { + Name: "med", + Count: 24, + Start: &lowmed, + End: &medhi, + }, + { + Name: "hi", + Count: 1, + Start: &medhi, + End: &hihigher, + }, + }, + } + frs1 := FacetResults{ + "birthdays": fr1, + } + + fr2 := &FacetResult{ + Field: "birthday", + Total: 100, + Missing: 25, + Other: 25, + DateRanges: []*DateRangeFacet{ + { + Name: "low", + End: &lowmed2, + Count: 25, + }, + { + Name: "med", + Start: &lowmed2, + End: &medhi2, + Count: 22, + }, + { + Name: "highest", + Start: &hihigher2, + Count: 3, + }, + }, + } + frs2 := FacetResults{ + "birthdays": fr2, + } + + expectedFr := &FacetResult{ + Field: "birthday", + Total: 200, + Missing: 50, + Other: 51, + DateRanges: []*DateRangeFacet{ + { + Name: "low", + Count: 50, + End: &lowmed, + }, + { + Name: "med", + Start: &lowmed, + End: &medhi, + Count: 46, + }, + { + Name: "highest", + Start: &hihigher, + Count: 3, + }, + }, + } + expectedFrs := FacetResults{ + "birthdays": expectedFr, + } + + frs1.Merge(frs2) + frs1.Fixup("birthdays", 3) + if !reflect.DeepEqual(frs1, expectedFrs) { + t.Errorf("expected %#v, got %#v", expectedFrs, frs1) + } +} diff --git a/search/highlight/format/ansi/ansi.go b/search/highlight/format/ansi/ansi.go new file mode 100644 index 0000000..4a0e1eb --- /dev/null +++ b/search/highlight/format/ansi/ansi.go @@ -0,0 +1,112 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ansi + +import ( + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search/highlight" +) + +const Name = "ansi" + +const DefaultAnsiHighlight = BgYellow + +type FragmentFormatter struct { + color string +} + +func NewFragmentFormatter(color string) *FragmentFormatter { + return &FragmentFormatter{ + color: color, + } +} + +func (a *FragmentFormatter) Format(f *highlight.Fragment, orderedTermLocations highlight.TermLocations) string { + rv := "" + curr := f.Start + for _, termLocation := range orderedTermLocations { + if termLocation == nil { + continue + } + // make sure the array positions match + if !termLocation.ArrayPositions.Equals(f.ArrayPositions) { + continue + } + if termLocation.Start < curr { + continue + } + if termLocation.End > f.End { + break + } + // add the stuff before this location + rv += string(f.Orig[curr:termLocation.Start]) + // add the color + rv += a.color + // add the term itself + rv += string(f.Orig[termLocation.Start:termLocation.End]) + // reset the color + rv += Reset + // update current + curr = termLocation.End + } + // add any remaining text after the last token + rv += string(f.Orig[curr:f.End]) + + return rv +} + +// ANSI color control escape sequences. +// Shamelessly copied from https://github.com/sqp/godock/blob/master/libs/log/colors.go +const ( + Reset = "\x1b[0m" + Bright = "\x1b[1m" + Dim = "\x1b[2m" + Underscore = "\x1b[4m" + Blink = "\x1b[5m" + Reverse = "\x1b[7m" + Hidden = "\x1b[8m" + FgBlack = "\x1b[30m" + FgRed = "\x1b[31m" + FgGreen = "\x1b[32m" + FgYellow = "\x1b[33m" + FgBlue = "\x1b[34m" + FgMagenta = "\x1b[35m" + FgCyan = "\x1b[36m" + FgWhite = "\x1b[37m" + BgBlack = "\x1b[40m" + BgRed = "\x1b[41m" + BgGreen = "\x1b[42m" + BgYellow = "\x1b[43m" + BgBlue = "\x1b[44m" + BgMagenta = "\x1b[45m" + BgCyan = "\x1b[46m" + BgWhite = "\x1b[47m" +) + +func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.FragmentFormatter, error) { + color := DefaultAnsiHighlight + colorVal, ok := config["color"].(string) + if ok { + color = colorVal + } + return NewFragmentFormatter(color), nil +} + +func init() { + err := registry.RegisterFragmentFormatter(Name, Constructor) + if err != nil { + panic(err) + } +} diff --git a/search/highlight/format/html/html.go b/search/highlight/format/html/html.go new file mode 100644 index 0000000..92b6f61 --- /dev/null +++ b/search/highlight/format/html/html.go @@ -0,0 +1,94 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package html + +import ( + "html" + + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search/highlight" +) + +const Name = "html" + +const defaultHTMLHighlightBefore = "" +const defaultHTMLHighlightAfter = "" + +type FragmentFormatter struct { + before string + after string +} + +func NewFragmentFormatter(before, after string) *FragmentFormatter { + return &FragmentFormatter{ + before: before, + after: after, + } +} + +func (a *FragmentFormatter) Format(f *highlight.Fragment, orderedTermLocations highlight.TermLocations) string { + rv := "" + curr := f.Start + for _, termLocation := range orderedTermLocations { + if termLocation == nil { + continue + } + // make sure the array positions match + if !termLocation.ArrayPositions.Equals(f.ArrayPositions) { + continue + } + if termLocation.Start < curr { + continue + } + if termLocation.End > f.End { + break + } + // add the stuff before this location + rv += html.EscapeString(string(f.Orig[curr:termLocation.Start])) + // start the tag + rv += a.before + // add the term itself + rv += html.EscapeString(string(f.Orig[termLocation.Start:termLocation.End])) + // end the tag + rv += a.after + // update current + curr = termLocation.End + } + // add any remaining text after the last token + rv += html.EscapeString(string(f.Orig[curr:f.End])) + + return rv +} + +func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.FragmentFormatter, error) { + before := defaultHTMLHighlightBefore + beforeVal, ok := config["before"].(string) + if ok { + before = beforeVal + } + after := defaultHTMLHighlightAfter + afterVal, ok := config["after"].(string) + if ok { + after = afterVal + } + return NewFragmentFormatter(before, after), nil +} + +func init() { + err := registry.RegisterFragmentFormatter(Name, Constructor) + if err != nil { + panic(err) + } +} diff --git a/search/highlight/format/html/html_test.go b/search/highlight/format/html/html_test.go new file mode 100644 index 0000000..e746568 --- /dev/null +++ b/search/highlight/format/html/html_test.go @@ -0,0 +1,120 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package html + +import ( + "testing" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/highlight" +) + +func TestHTMLFragmentFormatter(t *testing.T) { + tests := []struct { + fragment *highlight.Fragment + tlm search.TermLocationMap + output string + start string + end string + }{ + { + fragment: &highlight.Fragment{ + Orig: []byte("the quick brown fox"), + Start: 0, + End: 19, + }, + tlm: search.TermLocationMap{ + "quick": []*search.Location{ + { + Pos: 2, + Start: 4, + End: 9, + }, + }, + }, + output: "the quick brown fox", + start: "", + end: "", + }, + { + fragment: &highlight.Fragment{ + Orig: []byte("the quick brown fox"), + Start: 0, + End: 19, + }, + tlm: search.TermLocationMap{ + "quick": []*search.Location{ + { + Pos: 2, + Start: 4, + End: 9, + }, + }, + }, + output: "the quick brown fox", + start: "", + end: "", + }, + // test html escaping + { + fragment: &highlight.Fragment{ + Orig: []byte(" quick brown & fox"), + Start: 0, + End: 23, + }, + tlm: search.TermLocationMap{ + "quick": []*search.Location{ + { + Pos: 2, + Start: 6, + End: 11, + }, + }, + }, + output: "<the> quick brown & fox", + start: "", + end: "", + }, + // test html escaping inside search term + { + fragment: &highlight.Fragment{ + Orig: []byte(" qu&ick brown & fox"), + Start: 0, + End: 24, + }, + tlm: search.TermLocationMap{ + "qu&ick": []*search.Location{ + { + Pos: 2, + Start: 6, + End: 12, + }, + }, + }, + output: "<the> qu&ick brown & fox", + start: "", + end: "", + }, + } + + for _, test := range tests { + emHTMLFormatter := NewFragmentFormatter(test.start, test.end) + otl := highlight.OrderTermLocations(test.tlm) + result := emHTMLFormatter.Format(test.fragment, otl) + if result != test.output { + t.Errorf("expected `%s`, got `%s`", test.output, result) + } + } +} diff --git a/search/highlight/format/plain/plain.go b/search/highlight/format/plain/plain.go new file mode 100644 index 0000000..f65d1b7 --- /dev/null +++ b/search/highlight/format/plain/plain.go @@ -0,0 +1,92 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plain + +import ( + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search/highlight" +) + +const Name = "plain" + +const defaultPlainHighlightBefore = "" +const defaultPlainHighlightAfter = "" + +type FragmentFormatter struct { + before string + after string +} + +func NewFragmentFormatter(before, after string) *FragmentFormatter { + return &FragmentFormatter{ + before: before, + after: after, + } +} + +func (a *FragmentFormatter) Format(f *highlight.Fragment, orderedTermLocations highlight.TermLocations) string { + rv := "" + curr := f.Start + for _, termLocation := range orderedTermLocations { + if termLocation == nil { + continue + } + // make sure the array positions match + if !termLocation.ArrayPositions.Equals(f.ArrayPositions) { + continue + } + if termLocation.Start < curr { + continue + } + if termLocation.End > f.End { + break + } + // add the stuff before this location + rv += string(f.Orig[curr:termLocation.Start]) + // start the highlight tag + rv += a.before + // add the term itself + rv += string(f.Orig[termLocation.Start:termLocation.End]) + // end the highlight tag + rv += a.after + // update current + curr = termLocation.End + } + // add any remaining text after the last token + rv += string(f.Orig[curr:f.End]) + + return rv +} + +func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.FragmentFormatter, error) { + before := defaultPlainHighlightBefore + beforeVal, ok := config["before"].(string) + if ok { + before = beforeVal + } + after := defaultPlainHighlightAfter + afterVal, ok := config["after"].(string) + if ok { + after = afterVal + } + return NewFragmentFormatter(before, after), nil +} + +func init() { + err := registry.RegisterFragmentFormatter(Name, Constructor) + if err != nil { + panic(err) + } +} diff --git a/search/highlight/format/plain/plain_test.go b/search/highlight/format/plain/plain_test.go new file mode 100644 index 0000000..7786f4e --- /dev/null +++ b/search/highlight/format/plain/plain_test.go @@ -0,0 +1,80 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plain + +import ( + "testing" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/highlight" +) + +func TestPlainFragmentFormatter(t *testing.T) { + tests := []struct { + fragment *highlight.Fragment + tlm search.TermLocationMap + output string + start string + end string + }{ + { + fragment: &highlight.Fragment{ + Orig: []byte("the quick brown fox"), + Start: 0, + End: 19, + }, + tlm: search.TermLocationMap{ + "quick": []*search.Location{ + { + Pos: 2, + Start: 4, + End: 9, + }, + }, + }, + output: "the quick brown fox", + start: "", + end: "", + }, + { + fragment: &highlight.Fragment{ + Orig: []byte("the quick brown fox"), + Start: 0, + End: 19, + }, + tlm: search.TermLocationMap{ + "quick": []*search.Location{ + { + Pos: 2, + Start: 4, + End: 9, + }, + }, + }, + output: "the quick brown fox", + start: "", + end: "", + }, + } + + for _, test := range tests { + plainFormatter := NewFragmentFormatter(test.start, test.end) + otl := highlight.OrderTermLocations(test.tlm) + result := plainFormatter.Format(test.fragment, otl) + if result != test.output { + t.Errorf("expected `%s`, got `%s`", test.output, result) + } + } +} diff --git a/search/highlight/fragmenter/simple/simple.go b/search/highlight/fragmenter/simple/simple.go new file mode 100644 index 0000000..1c34b01 --- /dev/null +++ b/search/highlight/fragmenter/simple/simple.go @@ -0,0 +1,156 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package simple + +import ( + "unicode/utf8" + + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search/highlight" +) + +const Name = "simple" + +const defaultFragmentSize = 200 + +type Fragmenter struct { + fragmentSize int +} + +func NewFragmenter(fragmentSize int) *Fragmenter { + return &Fragmenter{ + fragmentSize: fragmentSize, + } +} + +func (s *Fragmenter) Fragment(orig []byte, ot highlight.TermLocations) []*highlight.Fragment { + var rv []*highlight.Fragment + maxbegin := 0 +OUTER: + for currTermIndex, termLocation := range ot { + // start with this + // it should be the highest scoring fragment with this term first + start := termLocation.Start + end := start + used := 0 + for end < len(orig) && used < s.fragmentSize { + r, size := utf8.DecodeRune(orig[end:]) + if r == utf8.RuneError { + continue OUTER // bail + } + end += size + used++ + } + + // if we still have more characters available to us + // push back towards beginning + // without cross maxbegin + for start > 0 && used < s.fragmentSize { + if start > len(orig) { + // bail if out of bounds, possibly due to token replacement + // e.g with a regexp replacement + continue OUTER + } + r, size := utf8.DecodeLastRune(orig[0:start]) + if r == utf8.RuneError { + continue OUTER // bail + } + if start-size >= maxbegin { + start -= size + used++ + } else { + break + } + } + + // however, we'd rather have the tokens centered more in the frag + // lets try to do that as best we can, without affecting the score + // find the end of the last term in this fragment + minend := end + for _, innerTermLocation := range ot[currTermIndex:] { + if innerTermLocation.End > end { + break + } + minend = innerTermLocation.End + } + + // find the smaller of the two rooms to move + roomToMove := utf8.RuneCount(orig[minend:end]) + roomToMoveStart := 0 + if start >= maxbegin { + roomToMoveStart = utf8.RuneCount(orig[maxbegin:start]) + } + if roomToMoveStart < roomToMove { + roomToMove = roomToMoveStart + } + + offset := roomToMove / 2 + + for offset > 0 { + r, size := utf8.DecodeLastRune(orig[0:start]) + if r == utf8.RuneError { + continue OUTER // bail + } + start -= size + + r, size = utf8.DecodeLastRune(orig[0:end]) + if r == utf8.RuneError { + continue OUTER // bail + } + end -= size + offset-- + } + + rv = append(rv, &highlight.Fragment{Orig: orig, Start: start - offset, End: end - offset}) + // set maxbegin to the end of the current term location + // so that next one won't back up to include it + maxbegin = termLocation.End + + } + if len(ot) == 0 { + // if there were no terms to highlight + // produce a single fragment from the beginning + start := 0 + end := start + used := 0 + for end < len(orig) && used < s.fragmentSize { + r, size := utf8.DecodeRune(orig[end:]) + if r == utf8.RuneError { + break + } + end += size + used++ + } + rv = append(rv, &highlight.Fragment{Orig: orig, Start: start, End: end}) + } + + return rv +} + +func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.Fragmenter, error) { + size := defaultFragmentSize + sizeVal, ok := config["size"].(float64) + if ok { + size = int(sizeVal) + } + return NewFragmenter(size), nil +} + +func init() { + err := registry.RegisterFragmenter(Name, Constructor) + if err != nil { + panic(err) + } +} diff --git a/search/highlight/fragmenter/simple/simple_test.go b/search/highlight/fragmenter/simple/simple_test.go new file mode 100644 index 0000000..5106b65 --- /dev/null +++ b/search/highlight/fragmenter/simple/simple_test.go @@ -0,0 +1,311 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package simple + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/search/highlight" +) + +func TestSimpleFragmenter(t *testing.T) { + + tests := []struct { + orig []byte + fragments []*highlight.Fragment + ot highlight.TermLocations + size int + }{ + { + orig: []byte("this is a test"), + fragments: []*highlight.Fragment{ + { + Orig: []byte("this is a test"), + Start: 0, + End: 14, + }, + }, + ot: highlight.TermLocations{ + &highlight.TermLocation{ + Term: "test", + Pos: 4, + Start: 10, + End: 14, + }, + }, + size: 100, + }, + { + orig: []byte("0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"), + fragments: []*highlight.Fragment{ + { + Orig: []byte("0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"), + Start: 0, + End: 100, + }, + }, + ot: highlight.TermLocations{ + &highlight.TermLocation{ + Term: "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", + Pos: 1, + Start: 0, + End: 100, + }, + }, + size: 100, + }, + { + orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), + fragments: []*highlight.Fragment{ + { + Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), + Start: 0, + End: 100, + }, + { + Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), + Start: 10, + End: 101, + }, + { + Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), + Start: 20, + End: 101, + }, + { + Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), + Start: 30, + End: 101, + }, + { + Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), + Start: 40, + End: 101, + }, + { + Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), + Start: 50, + End: 101, + }, + { + Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), + Start: 60, + End: 101, + }, + { + Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), + Start: 70, + End: 101, + }, + { + Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), + Start: 80, + End: 101, + }, + { + Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), + Start: 90, + End: 101, + }, + }, + ot: highlight.TermLocations{ + &highlight.TermLocation{ + Term: "0123456789", + Pos: 1, + Start: 0, + End: 10, + }, + &highlight.TermLocation{ + Term: "0123456789", + Pos: 2, + Start: 10, + End: 20, + }, + &highlight.TermLocation{ + Term: "0123456789", + Pos: 3, + Start: 20, + End: 30, + }, + &highlight.TermLocation{ + Term: "0123456789", + Pos: 4, + Start: 30, + End: 40, + }, + &highlight.TermLocation{ + Term: "0123456789", + Pos: 5, + Start: 40, + End: 50, + }, + &highlight.TermLocation{ + Term: "0123456789", + Pos: 6, + Start: 50, + End: 60, + }, + &highlight.TermLocation{ + Term: "0123456789", + Pos: 7, + Start: 60, + End: 70, + }, + &highlight.TermLocation{ + Term: "0123456789", + Pos: 8, + Start: 70, + End: 80, + }, + &highlight.TermLocation{ + Term: "0123456789", + Pos: 9, + Start: 80, + End: 90, + }, + &highlight.TermLocation{ + Term: "0123456789", + Pos: 10, + Start: 90, + End: 100, + }, + }, + size: 100, + }, + { + orig: []byte("[[पानी का स्वाद]] [[नीलेश रघुवंशी]] का कविता संग्रह हैं। इस कृति के लिए उन्हें २००४ में [[केदार सम्मान]] से सम्मानित किया गया है।{{केदार सम्मान से सम्मानित कृतियाँ}}"), + fragments: []*highlight.Fragment{ + { + Orig: []byte("[[पानी का स्वाद]] [[नीलेश रघुवंशी]] का कविता संग्रह हैं। इस कृति के लिए उन्हें २००४ में [[केदार सम्मान]] से सम्मानित किया गया है।{{केदार सम्मान से सम्मानित कृतियाँ}}"), + Start: 0, + End: 411, + }, + }, + ot: highlight.TermLocations{ + &highlight.TermLocation{ + Term: "पानी", + Pos: 1, + Start: 2, + End: 14, + }, + }, + size: 200, + }, + { + orig: []byte("交换机"), + fragments: []*highlight.Fragment{ + { + Orig: []byte("交换机"), + Start: 0, + End: 9, + }, + { + Orig: []byte("交换机"), + Start: 3, + End: 9, + }, + }, + ot: highlight.TermLocations{ + &highlight.TermLocation{ + Term: "交换", + Pos: 1, + Start: 0, + End: 6, + }, + &highlight.TermLocation{ + Term: "换机", + Pos: 2, + Start: 3, + End: 9, + }, + }, + size: 200, + }, + } + + for _, test := range tests { + fragmenter := NewFragmenter(test.size) + fragments := fragmenter.Fragment(test.orig, test.ot) + if !reflect.DeepEqual(fragments, test.fragments) { + t.Errorf("expected %#v, got %#v", test.fragments, fragments) + for _, fragment := range fragments { + t.Logf("frag: %s", fragment.Orig[fragment.Start:fragment.End]) + t.Logf("frag: %d - %d", fragment.Start, fragment.End) + } + } + } +} + +func TestSimpleFragmenterWithSize(t *testing.T) { + + tests := []struct { + orig []byte + fragments []*highlight.Fragment + ot highlight.TermLocations + }{ + { + orig: []byte("this is a test"), + fragments: []*highlight.Fragment{ + { + Orig: []byte("this is a test"), + Start: 0, + End: 5, + }, + { + Orig: []byte("this is a test"), + Start: 9, + End: 14, + }, + }, + ot: highlight.TermLocations{ + &highlight.TermLocation{ + Term: "this", + Pos: 1, + Start: 0, + End: 5, + }, + &highlight.TermLocation{ + Term: "test", + Pos: 4, + Start: 10, + End: 14, + }, + }, + }, + { + orig: []byte("避免出现 rune 越界问题"), + fragments: []*highlight.Fragment{ + { + Orig: []byte("避免出现 rune 越界问题"), + Start: 0, + End: 13, + }, + }, + ot: nil, + }, + } + + fragmenter := NewFragmenter(5) + for _, test := range tests { + fragments := fragmenter.Fragment(test.orig, test.ot) + if !reflect.DeepEqual(fragments, test.fragments) { + t.Errorf("expected %#v, got %#v", test.fragments, fragments) + for _, fragment := range fragments { + t.Logf("frag: %#v", fragment) + } + } + } +} diff --git a/search/highlight/highlighter.go b/search/highlight/highlighter.go new file mode 100644 index 0000000..3dd9ce0 --- /dev/null +++ b/search/highlight/highlighter.go @@ -0,0 +1,64 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package highlight + +import ( + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +type Fragment struct { + Orig []byte + ArrayPositions []uint64 + Start int + End int + Score float64 + Index int // used by heap +} + +func (f *Fragment) Overlaps(other *Fragment) bool { + if other.Start >= f.Start && other.Start < f.End { + return true + } else if f.Start >= other.Start && f.Start < other.End { + return true + } + return false +} + +type Fragmenter interface { + Fragment([]byte, TermLocations) []*Fragment +} + +type FragmentFormatter interface { + Format(f *Fragment, orderedTermLocations TermLocations) string +} + +type FragmentScorer interface { + Score(f *Fragment) float64 +} + +type Highlighter interface { + Fragmenter() Fragmenter + SetFragmenter(Fragmenter) + + FragmentFormatter() FragmentFormatter + SetFragmentFormatter(FragmentFormatter) + + Separator() string + SetSeparator(string) + + BestFragmentInField(*search.DocumentMatch, index.Document, string) string + BestFragmentsInField(*search.DocumentMatch, index.Document, string, int) []string +} diff --git a/search/highlight/highlighter/ansi/ansi.go b/search/highlight/highlighter/ansi/ansi.go new file mode 100644 index 0000000..99c1575 --- /dev/null +++ b/search/highlight/highlighter/ansi/ansi.go @@ -0,0 +1,53 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ansi + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search/highlight" + ansiFormatter "github.com/blevesearch/bleve/v2/search/highlight/format/ansi" + simpleFragmenter "github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple" + simpleHighlighter "github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple" +) + +const Name = "ansi" + +func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.Highlighter, error) { + + fragmenter, err := cache.FragmenterNamed(simpleFragmenter.Name) + if err != nil { + return nil, fmt.Errorf("error building fragmenter: %v", err) + } + + formatter, err := cache.FragmentFormatterNamed(ansiFormatter.Name) + if err != nil { + return nil, fmt.Errorf("error building fragment formatter: %v", err) + } + + return simpleHighlighter.NewHighlighter( + fragmenter, + formatter, + simpleHighlighter.DefaultSeparator), + nil +} + +func init() { + err := registry.RegisterHighlighter(Name, Constructor) + if err != nil { + panic(err) + } +} diff --git a/search/highlight/highlighter/html/html.go b/search/highlight/highlighter/html/html.go new file mode 100644 index 0000000..02eca0a --- /dev/null +++ b/search/highlight/highlighter/html/html.go @@ -0,0 +1,53 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package html + +import ( + "fmt" + + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search/highlight" + htmlFormatter "github.com/blevesearch/bleve/v2/search/highlight/format/html" + simpleFragmenter "github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple" + simpleHighlighter "github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple" +) + +const Name = "html" + +func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.Highlighter, error) { + + fragmenter, err := cache.FragmenterNamed(simpleFragmenter.Name) + if err != nil { + return nil, fmt.Errorf("error building fragmenter: %v", err) + } + + formatter, err := cache.FragmentFormatterNamed(htmlFormatter.Name) + if err != nil { + return nil, fmt.Errorf("error building fragment formatter: %v", err) + } + + return simpleHighlighter.NewHighlighter( + fragmenter, + formatter, + simpleHighlighter.DefaultSeparator), + nil +} + +func init() { + err := registry.RegisterHighlighter(Name, Constructor) + if err != nil { + panic(err) + } +} diff --git a/search/highlight/highlighter/simple/fragment_scorer_simple.go b/search/highlight/highlighter/simple/fragment_scorer_simple.go new file mode 100644 index 0000000..786e33c --- /dev/null +++ b/search/highlight/highlighter/simple/fragment_scorer_simple.go @@ -0,0 +1,49 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package simple + +import ( + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/highlight" +) + +// FragmentScorer will score fragments by how many +// unique terms occur in the fragment with no regard for +// any boost values used in the original query +type FragmentScorer struct { + tlm search.TermLocationMap +} + +func NewFragmentScorer(tlm search.TermLocationMap) *FragmentScorer { + return &FragmentScorer{ + tlm: tlm, + } +} + +func (s *FragmentScorer) Score(f *highlight.Fragment) { + score := 0.0 +OUTER: + for _, locations := range s.tlm { + for _, location := range locations { + if location.ArrayPositions.Equals(f.ArrayPositions) && int(location.Start) >= f.Start && int(location.End) <= f.End { + score += 1.0 + // once we find a term in the fragment + // don't care about additional matches + continue OUTER + } + } + } + f.Score = score +} diff --git a/search/highlight/highlighter/simple/fragment_scorer_simple_test.go b/search/highlight/highlighter/simple/fragment_scorer_simple_test.go new file mode 100644 index 0000000..116c7bd --- /dev/null +++ b/search/highlight/highlighter/simple/fragment_scorer_simple_test.go @@ -0,0 +1,82 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package simple + +import ( + "testing" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/highlight" +) + +func TestSimpleFragmentScorer(t *testing.T) { + + tests := []struct { + fragment *highlight.Fragment + tlm search.TermLocationMap + score float64 + }{ + { + fragment: &highlight.Fragment{ + Orig: []byte("cat in the hat"), + Start: 0, + End: 14, + }, + tlm: search.TermLocationMap{ + "cat": []*search.Location{ + { + Pos: 0, + Start: 0, + End: 3, + }, + }, + }, + score: 1, + }, + { + fragment: &highlight.Fragment{ + Orig: []byte("cat in the hat"), + Start: 0, + End: 14, + }, + tlm: search.TermLocationMap{ + "cat": []*search.Location{ + { + Pos: 1, + Start: 0, + End: 3, + }, + }, + "hat": []*search.Location{ + { + Pos: 4, + Start: 11, + End: 14, + }, + }, + }, + score: 2, + }, + } + + for _, test := range tests { + scorer := NewFragmentScorer(test.tlm) + scorer.Score(test.fragment) + if test.fragment.Score != test.score { + t.Errorf("expected score %f, got %f", test.score, test.fragment.Score) + } + } + +} diff --git a/search/highlight/highlighter/simple/highlighter_simple.go b/search/highlight/highlighter/simple/highlighter_simple.go new file mode 100644 index 0000000..d0adfa8 --- /dev/null +++ b/search/highlight/highlighter/simple/highlighter_simple.go @@ -0,0 +1,221 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package simple + +import ( + "container/heap" + "fmt" + + index "github.com/blevesearch/bleve_index_api" + + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/highlight" +) + +const Name = "simple" +const DefaultSeparator = "…" + +type Highlighter struct { + fragmenter highlight.Fragmenter + formatter highlight.FragmentFormatter + sep string +} + +func NewHighlighter(fragmenter highlight.Fragmenter, formatter highlight.FragmentFormatter, separator string) *Highlighter { + return &Highlighter{ + fragmenter: fragmenter, + formatter: formatter, + sep: separator, + } +} + +func (s *Highlighter) Fragmenter() highlight.Fragmenter { + return s.fragmenter +} + +func (s *Highlighter) SetFragmenter(f highlight.Fragmenter) { + s.fragmenter = f +} + +func (s *Highlighter) FragmentFormatter() highlight.FragmentFormatter { + return s.formatter +} + +func (s *Highlighter) SetFragmentFormatter(f highlight.FragmentFormatter) { + s.formatter = f +} + +func (s *Highlighter) Separator() string { + return s.sep +} + +func (s *Highlighter) SetSeparator(sep string) { + s.sep = sep +} + +func (s *Highlighter) BestFragmentInField(dm *search.DocumentMatch, doc index.Document, field string) string { + fragments := s.BestFragmentsInField(dm, doc, field, 1) + if len(fragments) > 0 { + return fragments[0] + } + return "" +} + +func (s *Highlighter) BestFragmentsInField(dm *search.DocumentMatch, doc index.Document, field string, num int) []string { + tlm := dm.Locations[field] + orderedTermLocations := highlight.OrderTermLocations(tlm) + scorer := NewFragmentScorer(tlm) + + // score the fragments and put them into a priority queue ordered by score + fq := make(FragmentQueue, 0) + heap.Init(&fq) + doc.VisitFields(func(f index.Field) { + if f.Name() == field { + _, ok := f.(index.TextField) + if ok { + termLocationsSameArrayPosition := make(highlight.TermLocations, 0) + for _, otl := range orderedTermLocations { + if otl.ArrayPositions.Equals(f.ArrayPositions()) { + termLocationsSameArrayPosition = append(termLocationsSameArrayPosition, otl) + } + } + + fieldData := f.Value() + fragments := s.fragmenter.Fragment(fieldData, termLocationsSameArrayPosition) + for _, fragment := range fragments { + fragment.ArrayPositions = f.ArrayPositions() + scorer.Score(fragment) + heap.Push(&fq, fragment) + } + } + } + }) + + // now find the N best non-overlapping fragments + var bestFragments []*highlight.Fragment + if len(fq) > 0 { + candidate := heap.Pop(&fq) + OUTER: + for candidate != nil && len(bestFragments) < num { + // see if this overlaps with any of the best already identified + if len(bestFragments) > 0 { + for _, frag := range bestFragments { + if candidate.(*highlight.Fragment).Overlaps(frag) { + if len(fq) < 1 { + break OUTER + } + candidate = heap.Pop(&fq) + continue OUTER + } + } + bestFragments = append(bestFragments, candidate.(*highlight.Fragment)) + } else { + bestFragments = append(bestFragments, candidate.(*highlight.Fragment)) + } + + if len(fq) < 1 { + break + } + candidate = heap.Pop(&fq) + } + } + + // now that we have the best fragments, we can format them + orderedTermLocations.MergeOverlapping() + formattedFragments := make([]string, len(bestFragments)) + for i, fragment := range bestFragments { + formattedFragments[i] = "" + if fragment.Start != 0 { + formattedFragments[i] += s.sep + } + formattedFragments[i] += s.formatter.Format(fragment, orderedTermLocations) + if fragment.End != len(fragment.Orig) { + formattedFragments[i] += s.sep + } + } + if len(formattedFragments) > 0 { + dm.AddFragments(field, formattedFragments) + } + + return formattedFragments +} + +// FragmentQueue implements heap.Interface and holds Items. +type FragmentQueue []*highlight.Fragment + +func (fq FragmentQueue) Len() int { return len(fq) } + +func (fq FragmentQueue) Less(i, j int) bool { + // We want Pop to give us the highest, not lowest, priority so we use greater-than here. + return fq[i].Score > fq[j].Score +} + +func (fq FragmentQueue) Swap(i, j int) { + fq[i], fq[j] = fq[j], fq[i] + fq[i].Index = i + fq[j].Index = j +} + +func (fq *FragmentQueue) Push(x interface{}) { + n := len(*fq) + item := x.(*highlight.Fragment) + item.Index = n + *fq = append(*fq, item) +} + +func (fq *FragmentQueue) Pop() interface{} { + old := *fq + n := len(old) + item := old[n-1] + item.Index = -1 // for safety + *fq = old[0 : n-1] + return item +} + +func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.Highlighter, error) { + separator := DefaultSeparator + separatorVal, ok := config["separator"].(string) + if ok { + separator = separatorVal + } + + fragmenterName, ok := config["fragmenter"].(string) + if !ok { + return nil, fmt.Errorf("must specify fragmenter") + } + fragmenter, err := cache.FragmenterNamed(fragmenterName) + if err != nil { + return nil, fmt.Errorf("error building fragmenter: %v", err) + } + + formatterName, ok := config["formatter"].(string) + if !ok { + return nil, fmt.Errorf("must specify formatter") + } + formatter, err := cache.FragmentFormatterNamed(formatterName) + if err != nil { + return nil, fmt.Errorf("error building fragment formatter: %v", err) + } + + return NewHighlighter(fragmenter, formatter, separator), nil +} + +func init() { + err := registry.RegisterHighlighter(Name, Constructor) + if err != nil { + panic(err) + } +} diff --git a/search/highlight/highlighter/simple/highlighter_simple_test.go b/search/highlight/highlighter/simple/highlighter_simple_test.go new file mode 100644 index 0000000..e93166a --- /dev/null +++ b/search/highlight/highlighter/simple/highlighter_simple_test.go @@ -0,0 +1,169 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package simple + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/highlight/format/ansi" + sfrag "github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple" +) + +const ( + reset = "\x1b[0m" + DefaultAnsiHighlight = "\x1b[43m" +) + +func TestSimpleHighlighter(t *testing.T) { + fragmenter := sfrag.NewFragmenter(100) + formatter := ansi.NewFragmentFormatter(ansi.DefaultAnsiHighlight) + highlighter := NewHighlighter(fragmenter, formatter, DefaultSeparator) + + docMatch := search.DocumentMatch{ + ID: "a", + Score: 1.0, + Locations: search.FieldTermLocationMap{ + "desc": search.TermLocationMap{ + "quick": []*search.Location{ + { + Pos: 2, + Start: 4, + End: 9, + }, + }, + "fox": []*search.Location{ + { + Pos: 4, + Start: 16, + End: 19, + }, + }, + }, + }, + } + + expectedFragment := "the " + DefaultAnsiHighlight + "quick" + reset + " brown " + DefaultAnsiHighlight + "fox" + reset + " jumps over the lazy dog" + doc := document.NewDocument("a").AddField(document.NewTextField("desc", []uint64{}, []byte("the quick brown fox jumps over the lazy dog"))) + + fragment := highlighter.BestFragmentInField(&docMatch, doc, "desc") + if fragment != expectedFragment { + t.Errorf("expected `%s`, got `%s`", expectedFragment, fragment) + } +} + +func TestSimpleHighlighterLonger(t *testing.T) { + + fieldBytes := []byte(`Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris sed semper nulla, sed pellentesque urna. Suspendisse potenti. Aliquam dignissim pulvinar erat vel ullamcorper. Nullam sed diam at dolor dapibus varius. Vestibulum at semper nunc. Integer ullamcorper enim ut nisi condimentum lacinia. Nulla ipsum ipsum, dictum in dapibus non, bibendum eget neque. Vestibulum malesuada erat quis malesuada dictum. Mauris luctus viverra lorem, nec hendrerit lacus lacinia ut. Donec suscipit sit amet nisi et dictum. Maecenas ultrices mollis diam, vel commodo libero lobortis nec. Nunc non dignissim dolor. Nulla non tempus risus, eget porttitor lectus. Suspendisse vitae gravida magna, a sagittis urna. Curabitur nec dui volutpat, hendrerit nisi non, adipiscing erat. Maecenas aliquet sem sit amet nibh ultrices accumsan. + +Mauris lobortis sem sed blandit bibendum. In scelerisque eros sed metus aliquet convallis ac eget metus. Donec eget feugiat sem. Quisque venenatis, augue et blandit vulputate, velit odio viverra dolor, eu iaculis eros urna ut nunc. Duis faucibus mattis enim ut ultricies. Donec scelerisque volutpat elit, vel varius ante porttitor vel. Duis neque nulla, ultrices vel est id, molestie semper odio. Maecenas condimentum felis vitae nibh venenatis, ut feugiat risus vehicula. Suspendisse non sapien neque. Etiam et lorem consequat lorem aliquam ullamcorper. Pellentesque id vestibulum neque, at aliquam turpis. Aenean ultrices nec erat sit amet aliquam. Morbi eu sem in augue cursus ullamcorper a sed dolor. Integer et lobortis nulla, sit amet laoreet elit. In elementum, nibh nec volutpat pretium, lectus est pulvinar arcu, vehicula lobortis tellus sem id mauris. Maecenas ac blandit purus, sit amet scelerisque magna. + +In hac habitasse platea dictumst. In lacinia elit non risus venenatis viverra. Nulla vestibulum laoreet turpis ac accumsan. Vivamus eros felis, rhoncus vel interdum bibendum, imperdiet nec diam. Etiam sed eros sed orci pellentesque sagittis. Praesent a fermentum leo. Vivamus ipsum risus, faucibus a dignissim ut, ullamcorper nec risus. Etiam quis adipiscing velit. Nam ac cursus arcu. Sed bibendum lectus quis massa dapibus dapibus. Vestibulum fermentum eros vitae hendrerit condimentum. + +Fusce viverra eleifend iaculis. Maecenas tempor dictum cursus. Mauris faucibus, tortor in bibendum ornare, nibh lorem sollicitudin est, sed consectetur nulla dui imperdiet urna. Fusce aliquet odio fermentum massa mollis, id feugiat lacus egestas. Integer et eleifend metus. Duis neque tellus, vulputate nec dui eu, euismod sodales orci. Vivamus turpis erat, consectetur et pulvinar nec, ornare a quam. Maecenas fermentum, ligula vitae consectetur lobortis, mi lacus fermentum ante, ut semper lacus lectus porta orci. Nulla vehicula sodales eros, in iaculis ante laoreet at. Sed venenatis interdum metus, egestas scelerisque orci laoreet ut. Donec fermentum enim eget nibh blandit laoreet. Proin lacinia adipiscing lorem vel ornare. Donec ullamcorper massa elementum urna varius viverra. Proin pharetra, erat at feugiat rhoncus, velit eros condimentum mi, ac mattis sapien dolor non elit. Aenean viverra purus id tincidunt vulputate. + +Etiam vel augue vel nisl commodo suscipit et ac nisl. Quisque eros diam, porttitor et aliquet sed, vulputate in odio. Aenean feugiat est quis neque vehicula, eget vulputate nunc tempor. Donec quis nulla ut quam feugiat consectetur ut et justo. Nulla congue, metus auctor facilisis scelerisque, nunc risus vulputate urna, in blandit urna nibh et neque. Etiam quis tortor ut nulla dignissim dictum non sed ligula. Vivamus accumsan ligula eget ipsum ultrices, a tincidunt urna blandit. In hac habitasse platea dictumst.`) + + doc := document.NewDocument("a").AddField(document.NewTextField("full", []uint64{}, fieldBytes)) + docMatch := search.DocumentMatch{ + ID: "a", + Score: 1.0, + Locations: search.FieldTermLocationMap{ + "full": search.TermLocationMap{ + "metus": []*search.Location{ + { + Pos: 0, + Start: 883, + End: 888, + }, + { + Pos: 0, + Start: 915, + End: 920, + }, + { + Pos: 0, + Start: 2492, + End: 2497, + }, + { + Pos: 0, + Start: 2822, + End: 2827, + }, + { + Pos: 0, + Start: 3417, + End: 3422, + }, + }, + "interdum": []*search.Location{ + { + Pos: 0, + Start: 1891, + End: 1899, + }, + { + Pos: 0, + Start: 2813, + End: 2821, + }, + }, + "venenatis": []*search.Location{ + { + Pos: 0, + Start: 954, + End: 963, + }, + { + Pos: 0, + Start: 1252, + End: 1261, + }, + { + Pos: 0, + Start: 1795, + End: 1804, + }, + { + Pos: 0, + Start: 2803, + End: 2812, + }, + }, + }, + }, + } + + expectedFragments := []string{ + "…eros, in iaculis ante laoreet at. Sed " + DefaultAnsiHighlight + "venenatis" + reset + " " + DefaultAnsiHighlight + "interdum" + reset + " " + DefaultAnsiHighlight + "metus" + reset + ", egestas scelerisque orci laoreet ut.…", + "… eros sed " + DefaultAnsiHighlight + "metus" + reset + " aliquet convallis ac eget " + DefaultAnsiHighlight + "metus" + reset + ". Donec eget feugiat sem. Quisque " + DefaultAnsiHighlight + "venenatis" + reset + ", augue et…", + "… odio. Maecenas condimentum felis vitae nibh " + DefaultAnsiHighlight + "venenatis" + reset + ", ut feugiat risus vehicula. Suspendisse non s…", + "… id feugiat lacus egestas. Integer et eleifend " + DefaultAnsiHighlight + "metus" + reset + ". Duis neque tellus, vulputate nec dui eu, euism…", + "… accumsan. Vivamus eros felis, rhoncus vel " + DefaultAnsiHighlight + "interdum" + reset + " bibendum, imperdiet nec diam. Etiam sed eros sed…", + } + + fragmenter := sfrag.NewFragmenter(100) + formatter := ansi.NewFragmentFormatter(ansi.DefaultAnsiHighlight) + highlighter := NewHighlighter(fragmenter, formatter, DefaultSeparator) + fragments := highlighter.BestFragmentsInField(&docMatch, doc, "full", 5) + + if !reflect.DeepEqual(fragments, expectedFragments) { + t.Errorf("expected %#v, got %#v", expectedFragments, fragments) + } + +} diff --git a/search/highlight/term_locations.go b/search/highlight/term_locations.go new file mode 100644 index 0000000..6bf385c --- /dev/null +++ b/search/highlight/term_locations.go @@ -0,0 +1,105 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package highlight + +import ( + "reflect" + "sort" + + "github.com/blevesearch/bleve/v2/search" +) + +type TermLocation struct { + Term string + ArrayPositions search.ArrayPositions + Pos int + Start int + End int +} + +func (tl *TermLocation) Overlaps(other *TermLocation) bool { + if reflect.DeepEqual(tl.ArrayPositions, other.ArrayPositions) { + if other.Start >= tl.Start && other.Start < tl.End { + return true + } else if tl.Start >= other.Start && tl.Start < other.End { + return true + } + } + return false +} + +type TermLocations []*TermLocation + +func (t TermLocations) Len() int { return len(t) } +func (t TermLocations) Swap(i, j int) { t[i], t[j] = t[j], t[i] } +func (t TermLocations) Less(i, j int) bool { + + shortestArrayPositions := len(t[i].ArrayPositions) + if len(t[j].ArrayPositions) < shortestArrayPositions { + shortestArrayPositions = len(t[j].ArrayPositions) + } + + // compare all the common array positions + for api := 0; api < shortestArrayPositions; api++ { + if t[i].ArrayPositions[api] < t[j].ArrayPositions[api] { + return true + } + if t[i].ArrayPositions[api] > t[j].ArrayPositions[api] { + return false + } + } + // all the common array positions are the same + if len(t[i].ArrayPositions) < len(t[j].ArrayPositions) { + return true // j array positions, longer so greater + } else if len(t[i].ArrayPositions) > len(t[j].ArrayPositions) { + return false // j array positions, shorter so less + } + + // array positions the same, compare starts + return t[i].Start < t[j].Start +} + +func (t TermLocations) MergeOverlapping() { + var lastTl *TermLocation + for i, tl := range t { + if lastTl == nil && tl != nil { + lastTl = tl + } else if lastTl != nil && tl != nil { + if lastTl.Overlaps(tl) { + // ok merge this with previous + lastTl.End = tl.End + t[i] = nil + } + } + } +} + +func OrderTermLocations(tlm search.TermLocationMap) TermLocations { + rv := make(TermLocations, 0) + for term, locations := range tlm { + for _, location := range locations { + tl := TermLocation{ + Term: term, + ArrayPositions: location.ArrayPositions, + Pos: int(location.Pos), + Start: int(location.Start), + End: int(location.End), + } + rv = append(rv, &tl) + } + } + sort.Sort(rv) + return rv +} diff --git a/search/highlight/term_locations_test.go b/search/highlight/term_locations_test.go new file mode 100644 index 0000000..6daa5cc --- /dev/null +++ b/search/highlight/term_locations_test.go @@ -0,0 +1,512 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package highlight + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/search" +) + +func TestTermLocationOverlaps(t *testing.T) { + + tests := []struct { + left *TermLocation + right *TermLocation + expected bool + }{ + { + left: &TermLocation{ + Start: 0, + End: 5, + }, + right: &TermLocation{ + Start: 3, + End: 7, + }, + expected: true, + }, + { + left: &TermLocation{ + Start: 0, + End: 5, + }, + right: &TermLocation{ + Start: 5, + End: 7, + }, + expected: false, + }, + { + left: &TermLocation{ + Start: 0, + End: 5, + }, + right: &TermLocation{ + Start: 7, + End: 11, + }, + expected: false, + }, + // with array positions + { + left: &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Start: 0, + End: 5, + }, + right: &TermLocation{ + ArrayPositions: search.ArrayPositions{1}, + Start: 7, + End: 11, + }, + expected: false, + }, + { + left: &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Start: 0, + End: 5, + }, + right: &TermLocation{ + ArrayPositions: search.ArrayPositions{1}, + Start: 3, + End: 11, + }, + expected: false, + }, + { + left: &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Start: 0, + End: 5, + }, + right: &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Start: 3, + End: 11, + }, + expected: true, + }, + { + left: &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Start: 0, + End: 5, + }, + right: &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Start: 7, + End: 11, + }, + expected: false, + }, + } + + for _, test := range tests { + actual := test.left.Overlaps(test.right) + if actual != test.expected { + t.Errorf("expected %t got %t for %#v", test.expected, actual, test) + } + } +} + +func TestTermLocationsMergeOverlapping(t *testing.T) { + + tests := []struct { + input TermLocations + output TermLocations + }{ + { + input: TermLocations{}, + output: TermLocations{}, + }, + { + input: TermLocations{ + &TermLocation{ + Start: 0, + End: 5, + }, + &TermLocation{ + Start: 7, + End: 11, + }, + }, + output: TermLocations{ + &TermLocation{ + Start: 0, + End: 5, + }, + &TermLocation{ + Start: 7, + End: 11, + }, + }, + }, + { + input: TermLocations{ + &TermLocation{ + Start: 0, + End: 5, + }, + &TermLocation{ + Start: 4, + End: 11, + }, + }, + output: TermLocations{ + &TermLocation{ + Start: 0, + End: 11, + }, + nil, + }, + }, + { + input: TermLocations{ + &TermLocation{ + Start: 0, + End: 5, + }, + &TermLocation{ + Start: 4, + End: 11, + }, + &TermLocation{ + Start: 9, + End: 13, + }, + }, + output: TermLocations{ + &TermLocation{ + Start: 0, + End: 13, + }, + nil, + nil, + }, + }, + { + input: TermLocations{ + &TermLocation{ + Start: 0, + End: 5, + }, + &TermLocation{ + Start: 4, + End: 11, + }, + &TermLocation{ + Start: 9, + End: 13, + }, + &TermLocation{ + Start: 15, + End: 21, + }, + }, + output: TermLocations{ + &TermLocation{ + Start: 0, + End: 13, + }, + nil, + nil, + &TermLocation{ + Start: 15, + End: 21, + }, + }, + }, + // with array positions + { + input: TermLocations{ + &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Start: 0, + End: 5, + }, + &TermLocation{ + ArrayPositions: search.ArrayPositions{1}, + Start: 7, + End: 11, + }, + }, + output: TermLocations{ + &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Start: 0, + End: 5, + }, + &TermLocation{ + ArrayPositions: search.ArrayPositions{1}, + Start: 7, + End: 11, + }, + }, + }, + { + input: TermLocations{ + &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Start: 0, + End: 5, + }, + &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Start: 7, + End: 11, + }, + }, + output: TermLocations{ + &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Start: 0, + End: 5, + }, + &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Start: 7, + End: 11, + }, + }, + }, + { + input: TermLocations{ + &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Start: 0, + End: 5, + }, + &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Start: 3, + End: 11, + }, + }, + output: TermLocations{ + &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Start: 0, + End: 11, + }, + nil, + }, + }, + { + input: TermLocations{ + &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Start: 0, + End: 5, + }, + &TermLocation{ + ArrayPositions: search.ArrayPositions{1}, + Start: 3, + End: 11, + }, + }, + output: TermLocations{ + &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Start: 0, + End: 5, + }, + &TermLocation{ + ArrayPositions: search.ArrayPositions{1}, + Start: 3, + End: 11, + }, + }, + }, + } + + for _, test := range tests { + test.input.MergeOverlapping() + if !reflect.DeepEqual(test.input, test.output) { + t.Errorf("expected: %#v got %#v", test.output, test.input) + } + } +} + +func TestTermLocationsOrder(t *testing.T) { + + tests := []struct { + input search.TermLocationMap + output TermLocations + }{ + { + input: search.TermLocationMap{}, + output: TermLocations{}, + }, + { + input: search.TermLocationMap{ + "term": []*search.Location{ + { + Start: 0, + }, + { + Start: 5, + }, + }, + }, + output: TermLocations{ + &TermLocation{ + Term: "term", + Start: 0, + }, + &TermLocation{ + Term: "term", + Start: 5, + }, + }, + }, + { + input: search.TermLocationMap{ + "term": []*search.Location{ + { + Start: 5, + }, + { + Start: 0, + }, + }, + }, + output: TermLocations{ + &TermLocation{ + Term: "term", + Start: 0, + }, + &TermLocation{ + Term: "term", + Start: 5, + }, + }, + }, + // with array positions + { + input: search.TermLocationMap{ + "term": []*search.Location{ + { + ArrayPositions: search.ArrayPositions{0}, + Start: 0, + }, + { + ArrayPositions: search.ArrayPositions{0}, + Start: 5, + }, + }, + }, + output: TermLocations{ + &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Term: "term", + Start: 0, + }, + &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Term: "term", + Start: 5, + }, + }, + }, + { + input: search.TermLocationMap{ + "term": []*search.Location{ + { + ArrayPositions: search.ArrayPositions{0}, + Start: 5, + }, + { + ArrayPositions: search.ArrayPositions{0}, + Start: 0, + }, + }, + }, + output: TermLocations{ + &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Term: "term", + Start: 0, + }, + &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Term: "term", + Start: 5, + }, + }, + }, + { + input: search.TermLocationMap{ + "term": []*search.Location{ + { + ArrayPositions: search.ArrayPositions{0}, + Start: 5, + }, + { + ArrayPositions: search.ArrayPositions{1}, + Start: 0, + }, + }, + }, + output: TermLocations{ + &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Term: "term", + Start: 5, + }, + &TermLocation{ + ArrayPositions: search.ArrayPositions{1}, + Term: "term", + Start: 0, + }, + }, + }, + { + input: search.TermLocationMap{ + "term": []*search.Location{ + { + ArrayPositions: search.ArrayPositions{0}, + Start: 5, + }, + { + ArrayPositions: search.ArrayPositions{0, 1}, + Start: 0, + }, + }, + }, + output: TermLocations{ + &TermLocation{ + ArrayPositions: search.ArrayPositions{0}, + Term: "term", + Start: 5, + }, + &TermLocation{ + ArrayPositions: search.ArrayPositions{0, 1}, + Term: "term", + Start: 0, + }, + }, + }, + } + + for _, test := range tests { + actual := OrderTermLocations(test.input) + if !reflect.DeepEqual(actual, test.output) { + t.Errorf("expected: %#v got %#v", test.output, actual) + } + } +} diff --git a/search/levenshtein.go b/search/levenshtein.go new file mode 100644 index 0000000..dadab25 --- /dev/null +++ b/search/levenshtein.go @@ -0,0 +1,118 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import ( + "math" +) + +func LevenshteinDistance(a, b string) int { + la := len(a) + lb := len(b) + d := make([]int, la+1) + var lastdiag, olddiag, temp int + + for i := 1; i <= la; i++ { + d[i] = i + } + for i := 1; i <= lb; i++ { + d[0] = i + lastdiag = i - 1 + for j := 1; j <= la; j++ { + olddiag = d[j] + min := d[j] + 1 + if (d[j-1] + 1) < min { + min = d[j-1] + 1 + } + if a[j-1] == b[i-1] { + temp = 0 + } else { + temp = 1 + } + if (lastdiag + temp) < min { + min = lastdiag + temp + } + d[j] = min + lastdiag = olddiag + } + } + return d[la] +} + +// LevenshteinDistanceMax same as LevenshteinDistance but +// attempts to bail early once we know the distance +// will be greater than max +// in which case the first return val will be the max +// and the second will be true, indicating max was exceeded +func LevenshteinDistanceMax(a, b string, max int) (int, bool) { + v, wasMax, _ := LevenshteinDistanceMaxReuseSlice(a, b, max, nil) + return v, wasMax +} + +func LevenshteinDistanceMaxReuseSlice(a, b string, max int, d []int) (int, bool, []int) { + la := len(a) + lb := len(b) + + ld := int(math.Abs(float64(la - lb))) + if ld > max { + return max, true, d + } else if la == 0 || lb == 0 { + // if one string of the two strings is empty, then ld is + // the length of the other string and as such is <= max + return ld, false, d + } + + if cap(d) < la+1 { + d = make([]int, la+1) + } + d = d[:la+1] + + var lastdiag, olddiag, temp int + + for i := 1; i <= la; i++ { + d[i] = i + } + for i := 1; i <= lb; i++ { + d[0] = i + lastdiag = i - 1 + rowmin := max + 1 + for j := 1; j <= la; j++ { + olddiag = d[j] + min := d[j] + 1 + if (d[j-1] + 1) < min { + min = d[j-1] + 1 + } + if a[j-1] == b[i-1] { + temp = 0 + } else { + temp = 1 + } + if (lastdiag + temp) < min { + min = lastdiag + temp + } + if min < rowmin { + rowmin = min + } + d[j] = min + + lastdiag = olddiag + } + // after each row if rowmin isn't less than max stop + if rowmin > max { + return max, true, d + } + } + return d[la], false, d +} diff --git a/search/levenshtein_test.go b/search/levenshtein_test.go new file mode 100644 index 0000000..ef23980 --- /dev/null +++ b/search/levenshtein_test.go @@ -0,0 +1,126 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import ( + "testing" +) + +func TestLevenshteinDistance(t *testing.T) { + + tests := []struct { + a string + b string + dist int + }{ + { + "water", + "atec", + 2, + }, + { + "water", + "aphex", + 4, + }, + } + + for _, test := range tests { + actual := LevenshteinDistance(test.a, test.b) + if actual != test.dist { + t.Errorf("expected %d, got %d for %s and %s", test.dist, actual, test.a, test.b) + } + } +} + +func TestLevenshteinDistanceMax(t *testing.T) { + + tests := []struct { + a string + b string + max int + dist int + exceeded bool + }{ + { + a: "water", + b: "atec", + max: 1, + dist: 1, + exceeded: true, + }, + { + a: "water", + b: "christmas", + max: 3, + dist: 3, + exceeded: true, + }, + { + a: "", + b: "water", + max: 10, + dist: 5, + exceeded: false, + }, + { + a: "water", + b: "", + max: 3, + dist: 3, + exceeded: true, + }, + } + + for _, test := range tests { + actual, exceeded := LevenshteinDistanceMax(test.a, test.b, test.max) + if actual != test.dist || exceeded != test.exceeded { + t.Errorf("expected %d %t, got %d %t for %s and %s", test.dist, test.exceeded, actual, exceeded, test.a, test.b) + } + } +} + +// 5 terms that are less than 2 +// 5 terms that are more than 2 +var benchmarkTerms = []string{ + "watex", + "aters", + "wayer", + "wbter", + "yater", + "christmas", + "waterwaterwater", + "watcatdogfish", + "q", + "couchbase", +} + +func BenchmarkLevenshteinDistance(b *testing.B) { + a := "water" + for i := 0; i < b.N; i++ { + for _, t := range benchmarkTerms { + LevenshteinDistance(a, t) + } + } +} + +func BenchmarkLevenshteinDistanceMax(b *testing.B) { + a := "water" + for i := 0; i < b.N; i++ { + for _, t := range benchmarkTerms { + LevenshteinDistanceMax(a, t, 2) + } + } +} diff --git a/search/pool.go b/search/pool.go new file mode 100644 index 0000000..81c5ba1 --- /dev/null +++ b/search/pool.go @@ -0,0 +1,91 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import ( + "reflect" +) + +var reflectStaticSizeDocumentMatchPool int + +func init() { + var dmp DocumentMatchPool + reflectStaticSizeDocumentMatchPool = int(reflect.TypeOf(dmp).Size()) +} + +// DocumentMatchPoolTooSmall is a callback function that can be executed +// when the DocumentMatchPool does not have sufficient capacity +// By default we just perform just-in-time allocation, but you could log +// a message, or panic, etc. +type DocumentMatchPoolTooSmall func(p *DocumentMatchPool) *DocumentMatch + +// DocumentMatchPool manages use/reuse of DocumentMatch instances +// it pre-allocates space from a single large block with the expected +// number of instances. It is not thread-safe as currently all +// aspects of search take place in a single goroutine. +type DocumentMatchPool struct { + avail DocumentMatchCollection + TooSmall DocumentMatchPoolTooSmall +} + +func defaultDocumentMatchPoolTooSmall(p *DocumentMatchPool) *DocumentMatch { + return &DocumentMatch{} +} + +// NewDocumentMatchPool will build a DocumentMatchPool with memory +// pre-allocated to accommodate the requested number of DocumentMatch +// instances +func NewDocumentMatchPool(size, sortsize int) *DocumentMatchPool { + avail := make(DocumentMatchCollection, size) + // pre-allocate the expected number of instances + startBlock := make([]DocumentMatch, size) + startSorts := make([]string, size*sortsize) + // make these initial instances available + i, j := 0, 0 + for i < size { + avail[i] = &startBlock[i] + avail[i].Sort = startSorts[j:j] + i += 1 + j += sortsize + } + return &DocumentMatchPool{ + avail: avail, + TooSmall: defaultDocumentMatchPoolTooSmall, + } +} + +// Get returns an available DocumentMatch from the pool +// if the pool was not allocated with sufficient size, an allocation will +// occur to satisfy this request. As a side-effect this will grow the size +// of the pool. +func (p *DocumentMatchPool) Get() *DocumentMatch { + var rv *DocumentMatch + if len(p.avail) > 0 { + rv, p.avail = p.avail[len(p.avail)-1], p.avail[:len(p.avail)-1] + } else { + rv = p.TooSmall(p) + } + return rv +} + +// Put returns a DocumentMatch to the pool +func (p *DocumentMatchPool) Put(d *DocumentMatch) { + if d == nil { + return + } + // reset DocumentMatch before returning it to available pool + d.Reset() + p.avail = append(p.avail, d) +} diff --git a/search/pool_test.go b/search/pool_test.go new file mode 100644 index 0000000..21df38b --- /dev/null +++ b/search/pool_test.go @@ -0,0 +1,70 @@ +// Copyright (c) 2013 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import "testing" + +func TestDocumentMatchPool(t *testing.T) { + + tooManyCalled := false + + // create a pool + dmp := NewDocumentMatchPool(10, 0) + dmp.TooSmall = func(inner *DocumentMatchPool) *DocumentMatch { + tooManyCalled = true + return &DocumentMatch{} + } + + // get 10 instances without returning + returned := make(DocumentMatchCollection, 10) + + for i := 0; i < 10; i++ { + returned[i] = dmp.Get() + if tooManyCalled { + t.Fatal("too many function called before expected") + } + } + + // get one more and see if too many function is called + extra := dmp.Get() + if !tooManyCalled { + t.Fatal("expected too many function to be called, but wasn't") + } + + // return the first 10 + for i := 0; i < 10; i++ { + dmp.Put(returned[i]) + } + + // check len and cap + if len(dmp.avail) != 10 { + t.Fatalf("expected 10 available, got %d", len(dmp.avail)) + } + if cap(dmp.avail) != 10 { + t.Fatalf("expected avail cap still 10, got %d", cap(dmp.avail)) + } + + // return the extra + dmp.Put(extra) + + // check len and cap grown to 11 + if len(dmp.avail) != 11 { + t.Fatalf("expected 11 available, got %d", len(dmp.avail)) + } + // cap grows, but not by 1 (append behavior) + if cap(dmp.avail) <= 10 { + t.Fatalf("expected avail cap mpore than 10, got %d", cap(dmp.avail)) + } +} diff --git a/search/query/bool_field.go b/search/query/bool_field.go new file mode 100644 index 0000000..5aa7bb8 --- /dev/null +++ b/search/query/bool_field.go @@ -0,0 +1,66 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +type BoolFieldQuery struct { + Bool bool `json:"bool"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewBoolFieldQuery creates a new Query for boolean fields +func NewBoolFieldQuery(val bool) *BoolFieldQuery { + return &BoolFieldQuery{ + Bool: val, + } +} + +func (q *BoolFieldQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *BoolFieldQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *BoolFieldQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *BoolFieldQuery) Field() string { + return q.FieldVal +} + +func (q *BoolFieldQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + term := "F" + if q.Bool { + term = "T" + } + return searcher.NewTermSearcher(ctx, i, term, field, q.BoostVal.Value(), options) +} diff --git a/search/query/boolean.go b/search/query/boolean.go new file mode 100644 index 0000000..96df4a6 --- /dev/null +++ b/search/query/boolean.go @@ -0,0 +1,355 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +type BooleanQuery struct { + Must Query `json:"must,omitempty"` + Should Query `json:"should,omitempty"` + MustNot Query `json:"must_not,omitempty"` + Filter Query `json:"filter,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + queryStringMode bool +} + +// NewBooleanQuery creates a compound Query composed +// of several other Query objects. +// Result documents must satisfy ALL of the +// must Queries. +// Result documents must satisfy NONE of the must not +// Queries. +// Result documents that ALSO satisfy any of the should +// Queries will score higher. +func NewBooleanQuery(must []Query, should []Query, mustNot []Query) *BooleanQuery { + + rv := BooleanQuery{} + if len(must) > 0 { + rv.Must = NewConjunctionQuery(must) + } + if len(should) > 0 { + rv.Should = NewDisjunctionQuery(should) + } + if len(mustNot) > 0 { + rv.MustNot = NewDisjunctionQuery(mustNot) + } + + return &rv +} + +func NewBooleanQueryForQueryString(must []Query, should []Query, mustNot []Query) *BooleanQuery { + rv := NewBooleanQuery(nil, nil, nil) + rv.queryStringMode = true + rv.AddMust(must...) + rv.AddShould(should...) + rv.AddMustNot(mustNot...) + return rv +} + +// SetMinShould requires that at least minShould of the +// should Queries must be satisfied. +func (q *BooleanQuery) SetMinShould(minShould float64) { + q.Should.(*DisjunctionQuery).SetMin(minShould) +} + +func (q *BooleanQuery) AddMust(m ...Query) { + if m == nil { + return + } + if q.Must == nil { + tmp := NewConjunctionQuery([]Query{}) + tmp.queryStringMode = q.queryStringMode + q.Must = tmp + } + for _, mq := range m { + q.Must.(*ConjunctionQuery).AddQuery(mq) + } +} + +func (q *BooleanQuery) AddShould(m ...Query) { + if m == nil { + return + } + if q.Should == nil { + tmp := NewDisjunctionQuery([]Query{}) + tmp.queryStringMode = q.queryStringMode + q.Should = tmp + } + for _, mq := range m { + q.Should.(*DisjunctionQuery).AddQuery(mq) + } +} + +func (q *BooleanQuery) AddMustNot(m ...Query) { + if m == nil { + return + } + if q.MustNot == nil { + tmp := NewDisjunctionQuery([]Query{}) + tmp.queryStringMode = q.queryStringMode + q.MustNot = tmp + } + for _, mq := range m { + q.MustNot.(*DisjunctionQuery).AddQuery(mq) + } +} + +func (q *BooleanQuery) AddFilter(m Query) { + if m == nil { + return + } + q.Filter = m +} + +func (q *BooleanQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *BooleanQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *BooleanQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + var err error + var mustNotSearcher search.Searcher + if q.MustNot != nil { + mustNotSearcher, err = q.MustNot.Searcher(ctx, i, m, options) + if err != nil { + return nil, err + } + // if must not is MatchNone, reset it to nil + if _, ok := mustNotSearcher.(*searcher.MatchNoneSearcher); ok { + mustNotSearcher = nil + } + } + + var mustSearcher search.Searcher + if q.Must != nil { + mustSearcher, err = q.Must.Searcher(ctx, i, m, options) + if err != nil { + return nil, err + } + // if must searcher is MatchNone, reset it to nil + if _, ok := mustSearcher.(*searcher.MatchNoneSearcher); ok { + mustSearcher = nil + } + } + + var shouldSearcher search.Searcher + if q.Should != nil { + shouldSearcher, err = q.Should.Searcher(ctx, i, m, options) + if err != nil { + return nil, err + } + // if should searcher is MatchNone, reset it to nil + if _, ok := shouldSearcher.(*searcher.MatchNoneSearcher); ok { + shouldSearcher = nil + } + } + + var filterFunc searcher.FilterFunc + if q.Filter != nil { + // create a new searcher options with disabled scoring, since filter should not affect scoring + // and we don't want to pay the cost of scoring if we don't need it, also disable term vectors + // and explain, since we don't need them for filters + filterOptions := search.SearcherOptions{ + Explain: false, + IncludeTermVectors: false, + Score: "none", + } + filterSearcher, err := q.Filter.Searcher(ctx, i, m, filterOptions) + if err != nil { + return nil, err + } + var init bool + var refDoc *search.DocumentMatch + filterFunc = func(sctx *search.SearchContext, d *search.DocumentMatch) bool { + // Initialize the reference document to point + // to the first document in the filterSearcher + var err error + if !init { + refDoc, err = filterSearcher.Next(sctx) + if err != nil { + return false + } + init = true + } + if refDoc == nil { + // filterSearcher is exhausted, d is not in filter + return false + } + // Compare document IDs + cmp := refDoc.IndexInternalID.Compare(d.IndexInternalID) + if cmp < 0 { + // recycle refDoc now that we do not need it + sctx.DocumentMatchPool.Put(refDoc) + // filterSearcher is behind the current document, Advance() it + refDoc, err = filterSearcher.Advance(sctx, d.IndexInternalID) + if err != nil || refDoc == nil { + return false + } + // After advance, check if they're now equal + cmp = refDoc.IndexInternalID.Compare(d.IndexInternalID) + } + // cmp >= 0: either equal (match) or filterSearcher is ahead (no match) + return cmp == 0 + } + } + + // if all 4 are nil, return MatchNone + if mustSearcher == nil && shouldSearcher == nil && mustNotSearcher == nil && filterFunc == nil { + return searcher.NewMatchNoneSearcher(i) + } + + // optimization, if only must searcher, just return it instead + if mustSearcher != nil && shouldSearcher == nil && mustNotSearcher == nil && filterFunc == nil { + return mustSearcher, nil + } + + // optimization, if only should searcher, just return it instead + if mustSearcher == nil && shouldSearcher != nil && mustNotSearcher == nil && filterFunc == nil { + return shouldSearcher, nil + } + + // optimization, if only filter searcher, wrap around a MatchAllSearcher + if mustSearcher == nil && shouldSearcher == nil && mustNotSearcher == nil && filterFunc != nil { + mustSearcher, err = searcher.NewMatchAllSearcher(ctx, i, 1.0, options) + if err != nil { + return nil, err + } + return searcher.NewFilteringSearcher(ctx, + mustSearcher, + filterFunc, + ), nil + } + + // if only mustNotSearcher, start with MatchAll + if mustSearcher == nil && shouldSearcher == nil && mustNotSearcher != nil { + mustSearcher, err = searcher.NewMatchAllSearcher(ctx, i, 1.0, options) + if err != nil { + return nil, err + } + } + + bs, err := searcher.NewBooleanSearcher(ctx, i, mustSearcher, shouldSearcher, mustNotSearcher, options) + if err != nil { + return nil, err + } + + if filterFunc != nil { + return searcher.NewFilteringSearcher(ctx, bs, filterFunc), nil + } + return bs, nil +} + +func (q *BooleanQuery) Validate() error { + if qm, ok := q.Must.(ValidatableQuery); ok { + err := qm.Validate() + if err != nil { + return err + } + } + if qs, ok := q.Should.(ValidatableQuery); ok { + err := qs.Validate() + if err != nil { + return err + } + } + if qmn, ok := q.MustNot.(ValidatableQuery); ok { + err := qmn.Validate() + if err != nil { + return err + } + } + if qf, ok := q.Filter.(ValidatableQuery); ok { + err := qf.Validate() + if err != nil { + return err + } + } + if q.Must == nil && q.Should == nil && q.MustNot == nil && q.Filter == nil { + return fmt.Errorf("boolean query must contain at least one must or should or not must or filter clause") + } + return nil +} + +func (q *BooleanQuery) UnmarshalJSON(data []byte) error { + tmp := struct { + Must json.RawMessage `json:"must,omitempty"` + Should json.RawMessage `json:"should,omitempty"` + MustNot json.RawMessage `json:"must_not,omitempty"` + Filter json.RawMessage `json:"filter,omitempty"` + Boost *Boost `json:"boost,omitempty"` + }{} + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return err + } + + if tmp.Must != nil { + q.Must, err = ParseQuery(tmp.Must) + if err != nil { + return err + } + _, isConjunctionQuery := q.Must.(*ConjunctionQuery) + if !isConjunctionQuery { + return fmt.Errorf("must clause must be conjunction") + } + } + + if tmp.Should != nil { + q.Should, err = ParseQuery(tmp.Should) + if err != nil { + return err + } + _, isDisjunctionQuery := q.Should.(*DisjunctionQuery) + if !isDisjunctionQuery { + return fmt.Errorf("should clause must be disjunction") + } + } + + if tmp.MustNot != nil { + q.MustNot, err = ParseQuery(tmp.MustNot) + if err != nil { + return err + } + _, isDisjunctionQuery := q.MustNot.(*DisjunctionQuery) + if !isDisjunctionQuery { + return fmt.Errorf("must not clause must be disjunction") + } + } + + if tmp.Filter != nil { + q.Filter, err = ParseQuery(tmp.Filter) + if err != nil { + return err + } + } + + q.BoostVal = tmp.Boost + + return nil +} diff --git a/search/query/boost.go b/search/query/boost.go new file mode 100644 index 0000000..1365994 --- /dev/null +++ b/search/query/boost.go @@ -0,0 +1,33 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import "fmt" + +type Boost float64 + +func (b *Boost) Value() float64 { + if b == nil { + return 1.0 + } + return float64(*b) +} + +func (b *Boost) GoString() string { + if b == nil { + return "boost unspecified" + } + return fmt.Sprintf("%f", *b) +} diff --git a/search/query/conjunction.go b/search/query/conjunction.go new file mode 100644 index 0000000..631956d --- /dev/null +++ b/search/query/conjunction.go @@ -0,0 +1,154 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "encoding/json" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +type ConjunctionQuery struct { + Conjuncts []Query `json:"conjuncts"` + BoostVal *Boost `json:"boost,omitempty"` + queryStringMode bool +} + +// NewConjunctionQuery creates a new compound Query. +// Result documents must satisfy all of the queries. +func NewConjunctionQuery(conjuncts []Query) *ConjunctionQuery { + return &ConjunctionQuery{ + Conjuncts: conjuncts, + } +} + +func (q *ConjunctionQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *ConjunctionQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *ConjunctionQuery) AddQuery(aq ...Query) { + q.Conjuncts = append(q.Conjuncts, aq...) +} + +func (q *ConjunctionQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + ss := make([]search.Searcher, 0, len(q.Conjuncts)) + cleanup := func() { + for _, searcher := range ss { + if searcher != nil { + _ = searcher.Close() + } + } + } + nestedMode, _ := ctx.Value(search.NestedSearchKey).(bool) + var nm mapping.NestedMapping + if nestedMode { + var ok bool + // get the nested mapping + if nm, ok = m.(mapping.NestedMapping); !ok { + // shouldn't be in nested mode if no nested mapping + nestedMode = false + } + } + // set of fields used in this query + var qfs search.FieldSet + var err error + + for _, conjunct := range q.Conjuncts { + // Gather fields when nested mode is enabled + if nestedMode { + qfs, err = ExtractFields(conjunct, m, qfs) + if err != nil { + cleanup() + return nil, err + } + } + sr, err := conjunct.Searcher(ctx, i, m, options) + if err != nil { + cleanup() + return nil, err + } + if _, ok := sr.(*searcher.MatchNoneSearcher); ok && q.queryStringMode { + // in query string mode, skip match none + continue + } + ss = append(ss, sr) + } + + if len(ss) < 1 { + return searcher.NewMatchNoneSearcher(i) + } + + if nestedMode { + // first determine the nested depth info for the query fields + commonDepth, maxDepth := nm.NestedDepth(qfs) + // if we have common depth == max depth then we can just use + // the normal conjunction searcher, as all fields share the same + // nested context, otherwise we need to use the nested conjunction searcher + // also, if we are querying the _all or _id fields, we need to use + // the nested conjunction searcher as well, with common depth 0 + // indicating matches happen only at the root level + if qfs.HasAll() || qfs.HasID() { + commonDepth = 0 + } + if commonDepth < maxDepth { + return searcher.NewNestedConjunctionSearcher(ctx, i, ss, commonDepth, options) + } + } + + return searcher.NewConjunctionSearcher(ctx, i, ss, options) +} + +func (q *ConjunctionQuery) Validate() error { + for _, q := range q.Conjuncts { + if q, ok := q.(ValidatableQuery); ok { + err := q.Validate() + if err != nil { + return err + } + } + } + return nil +} + +func (q *ConjunctionQuery) UnmarshalJSON(data []byte) error { + tmp := struct { + Conjuncts []json.RawMessage `json:"conjuncts"` + Boost *Boost `json:"boost,omitempty"` + }{} + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return err + } + q.Conjuncts = make([]Query, len(tmp.Conjuncts)) + for i, term := range tmp.Conjuncts { + query, err := ParseQuery(term) + if err != nil { + return err + } + q.Conjuncts[i] = query + } + q.BoostVal = tmp.Boost + return nil +} diff --git a/search/query/custom_filter.go b/search/query/custom_filter.go new file mode 100644 index 0000000..4be78de --- /dev/null +++ b/search/query/custom_filter.go @@ -0,0 +1,137 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +// CustomFilterQuery wraps a child query and filters its candidate matches via +// an embedder-provided per-hit callback. +type CustomFilterQuery struct { + Query Query `json:"query"` + Fields []string `json:"fields,omitempty"` + + filterFunc searcher.CustomFilterFunc + payload map[string]interface{} +} + +// CustomFilterQueryParser lets an embedder override parsing of +// {"custom_filter": ...} nodes. It is intended to be assigned once during +// process startup or init, before any queries are parsed; callers must not +// mutate it concurrently with ParseQuery(). For example: +// +// func init() { +// query.CustomFilterQueryParser = parseCustomFilterQuery +// } +var CustomFilterQueryParser func([]byte) (Query, error) + +func NewCustomFilterQueryWithFilter(query Query, filter searcher.CustomFilterFunc, fields []string, payload map[string]interface{}) *CustomFilterQuery { + return &CustomFilterQuery{ + Query: query, + Fields: fields, + filterFunc: filter, + payload: payload, + } +} + +func (q *CustomFilterQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + if q == nil { + return nil, fmt.Errorf("custom filter query is nil") + } + if q.Query == nil { + return nil, fmt.Errorf("custom filter query must have a query") + } + if q.filterFunc == nil { + return nil, fmt.Errorf("custom filter query must have a filter callback") + } + + // Build the inner searcher first; custom filtering wraps its output. + childSearcher, err := q.Query.Searcher(ctx, i, m, options) + if err != nil { + return nil, err + } + + // Create a doc value reader for the requested fields (if any) so the + // searcher can populate d.Fields before invoking the callback. + var dvReader index.DocValueReader + var fieldTypes map[string]string + if len(q.Fields) > 0 { + // Expand a "*" wildcard to the concrete field set; the doc-value + // reader matches field names literally and would otherwise load + // nothing, leaving d.Fields empty in the callback. + fields, err2 := expandFieldWildcard(q.Fields, i) + if err2 != nil { + _ = childSearcher.Close() + return nil, err2 + } + dvReader, err2 = i.DocValueReader(fields) + if err2 != nil { + _ = childSearcher.Close() + return nil, err2 + } + fieldTypes = resolveFieldTypes(fields, m) + } + + return searcher.NewCustomFilterSearcher(ctx, childSearcher, q.filterFunc, dvReader, i, fieldTypes), nil +} + +func (q *CustomFilterQuery) Validate() error { + if q == nil { + return fmt.Errorf("custom filter query is nil") + } + if q.Query == nil { + return fmt.Errorf("custom filter query must have a query") + } + if q.filterFunc == nil { + return fmt.Errorf("custom filter query must have a filter callback") + } + if vq, ok := q.Query.(ValidatableQuery); ok { + return vq.Validate() + } + return nil +} + +func (q *CustomFilterQuery) MarshalJSON() ([]byte, error) { + inner := make(map[string]interface{}, len(q.payload)+2) + for k, v := range q.payload { + inner[k] = v + } + inner["query"] = q.Query + if len(q.Fields) > 0 { + inner["fields"] = q.Fields + } + return json.Marshal(map[string]interface{}{ + "custom_filter": inner, + }) +} + +func (q *CustomFilterQuery) UnmarshalJSON(data []byte) error { + child, fields, payload, err := unmarshalCustomQueryPayload(data, "custom_filter") + if err != nil { + return err + } + q.Query = child + q.Fields = fields + q.payload = payload + return nil +} diff --git a/search/query/custom_payload.go b/search/query/custom_payload.go new file mode 100644 index 0000000..6f0a052 --- /dev/null +++ b/search/query/custom_payload.go @@ -0,0 +1,136 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "encoding/json" + "fmt" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +func unmarshalCustomQueryPayload(data []byte, key string) (Query, []string, map[string]interface{}, error) { + tmp := map[string]json.RawMessage{} + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return nil, nil, nil, err + } + + innerRaw, ok := tmp[key] + if !ok || innerRaw == nil { + return nil, nil, nil, nil + } + + var inner map[string]json.RawMessage + err = util.UnmarshalJSON(innerRaw, &inner) + if err != nil || inner == nil { + return nil, nil, nil, fmt.Errorf("%s query must be a JSON object", key) + } + + var child Query + if childQuery, ok := inner["query"]; ok && childQuery != nil { + child, err = ParseQuery(childQuery) + if err != nil { + return nil, nil, nil, err + } + } + + var fields []string + if rawFields, ok := inner["fields"]; ok && rawFields != nil { + if err := util.UnmarshalJSON(rawFields, &fields); err != nil { + return nil, nil, nil, fmt.Errorf("%s query has invalid %q: %w", + key, "fields", err) + } + } + + payload := make(map[string]interface{}, len(inner)) + for k, raw := range inner { + if k == "query" || k == "fields" { + continue + } + var v interface{} + if raw != nil { + err = util.UnmarshalJSON(raw, &v) + if err != nil { + return nil, nil, nil, fmt.Errorf("%s query has invalid %q payload: %w", + key, k, err) + } + } + payload[k] = v + } + + return child, fields, payload, nil +} + +// resolveFieldTypes looks up each field name in the index mapping and returns +// a map of field name → mapping type (e.g. "datetime", "number", "text"). +// This is used by the searcher layer to correctly decode doc value bytes. +func resolveFieldTypes(fields []string, m mapping.IndexMapping) map[string]string { + if m == nil || len(fields) == 0 { + return nil + } + types := make(map[string]string, len(fields)) + for _, f := range fields { + fm := m.FieldMappingForPath(f) + if fm.Type != "" { + types[f] = fm.Type + } + } + if len(types) == 0 { + return nil + } + return types +} + +// expandFieldWildcard replaces a "*" entry in the requested fields with the +// concrete set of fields in the index, mirroring how the standard stored-field +// loader treats "*" (see LoadAndHighlightFields). Custom-query callbacks read +// field values via the doc-value reader, which matches field names literally +// and has no wildcard handling of its own; without this expansion a request for +// fields:["*"] resolves to a single field literally named "*", matches nothing, +// and leaves d.Fields empty in the callback. +// +// The internal composite/identifier fields ("_all", "_id") are excluded so the +// callback sees the same fields the stored-field "*" path would surface rather +// than the tokenized _all blob or the document id (already available as the +// hit's ID). When fields does not contain "*", it is returned unchanged. +func expandFieldWildcard(fields []string, i index.IndexReader) ([]string, error) { + hasWildcard := false + for _, f := range fields { + if f == "*" { + hasWildcard = true + break + } + } + if !hasWildcard { + return fields, nil + } + + allFields, err := i.Fields() + if err != nil { + return nil, err + } + + expanded := make([]string, 0, len(allFields)) + for _, f := range allFields { + if f == "_all" || f == "_id" { + continue + } + expanded = append(expanded, f) + } + return expanded, nil +} diff --git a/search/query/custom_query_test.go b/search/query/custom_query_test.go new file mode 100644 index 0000000..9c68a76 --- /dev/null +++ b/search/query/custom_query_test.go @@ -0,0 +1,199 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/blevesearch/bleve/v2/search" +) + +func TestCustomFilterQueryUnmarshalJSON(t *testing.T) { + jsonBytes := []byte(`{ + "custom_filter": { + "query": { + "match": "beer" + } + } + }`) + + var cfq CustomFilterQuery + err := cfq.UnmarshalJSON(jsonBytes) + if err != nil { + t.Fatal(err) + } + + mq, ok := cfq.Query.(*MatchQuery) + if !ok { + t.Fatalf("expected inner query to be MatchQuery, got %T", cfq.Query) + } + + if mq.Match != "beer" { + t.Errorf("expected match 'beer', got '%s'", mq.Match) + } +} + +func TestCustomScoreQueryUnmarshalJSON(t *testing.T) { + jsonBytes := []byte(`{ + "custom_score": { + "query": { + "match": "beer" + } + } + }`) + + var csq CustomScoreQuery + err := csq.UnmarshalJSON(jsonBytes) + if err != nil { + t.Fatal(err) + } + + if _, ok := csq.Query.(*MatchQuery); !ok { + t.Fatalf("expected inner query to be MatchQuery, got %T", csq.Query) + } +} + +func TestCustomFilterQueryUnmarshalJSONRejectsNonObjectPayload(t *testing.T) { + jsonBytes := []byte(`{ + "custom_filter": "beer" + }`) + + var cfq CustomFilterQuery + err := cfq.UnmarshalJSON(jsonBytes) + if err == nil { + t.Fatal("expected error for malformed custom_filter payload") + } + if !strings.Contains(err.Error(), "custom_filter query must be a JSON object") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestCustomScoreQueryUnmarshalJSONRejectsNonObjectPayload(t *testing.T) { + jsonBytes := []byte(`{ + "custom_score": "beer" + }`) + + var csq CustomScoreQuery + err := csq.UnmarshalJSON(jsonBytes) + if err == nil { + t.Fatal("expected error for malformed custom_score payload") + } + if !strings.Contains(err.Error(), "custom_score query must be a JSON object") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestCustomFilterQueryMarshalJSONPreservesPayloadAndRewritesChild(t *testing.T) { + payload := map[string]interface{}{ + "params": map[string]interface{}{"min": float64(5)}, + "source": "function keep(doc, params){ return true; }", + } + + q := NewCustomFilterQueryWithFilter(NewMatchQuery("ipa"), + func(ctx context.Context, d *search.DocumentMatch) (bool, error) { return true, nil }, []string{"abv"}, payload) + + out, err := q.MarshalJSON() + if err != nil { + t.Fatal(err) + } + + var decoded struct { + CustomFilter struct { + Query json.RawMessage `json:"query"` + Fields []string `json:"fields"` + Params map[string]interface{} `json:"params"` + Source string `json:"source"` + } `json:"custom_filter"` + } + err = json.Unmarshal(out, &decoded) + if err != nil { + t.Fatal(err) + } + + if decoded.CustomFilter.Source != "function keep(doc, params){ return true; }" { + t.Fatalf("unexpected source: %q", decoded.CustomFilter.Source) + } + if len(decoded.CustomFilter.Fields) != 1 || decoded.CustomFilter.Fields[0] != "abv" { + t.Fatalf("unexpected fields: %v", decoded.CustomFilter.Fields) + } + if got := decoded.CustomFilter.Params["min"]; got != float64(5) { + t.Fatalf("unexpected params: %v", decoded.CustomFilter.Params) + } + + child, err := ParseQuery(decoded.CustomFilter.Query) + if err != nil { + t.Fatal(err) + } + mq, ok := child.(*MatchQuery) + if !ok { + t.Fatalf("expected match query child, got %T", child) + } + if mq.Match != "ipa" { + t.Fatalf("expected marshaled child query to be rewritten to ipa, got %q", mq.Match) + } +} + +func TestCustomScoreQueryMarshalJSONPreservesPayloadAndRewritesChild(t *testing.T) { + payload := map[string]interface{}{ + "params": map[string]interface{}{"weight": 0.05}, + "source": "function score(doc, params){ return doc.score; }", + } + + q := NewCustomScoreQueryWithScorer(NewMatchQuery("ipa"), + func(ctx context.Context, d *search.DocumentMatch) (float64, error) { return d.Score, nil }, []string{"ibu"}, payload) + + out, err := q.MarshalJSON() + if err != nil { + t.Fatal(err) + } + + var decoded struct { + CustomScore struct { + Query json.RawMessage `json:"query"` + Fields []string `json:"fields"` + Params map[string]interface{} `json:"params"` + Source string `json:"source"` + } `json:"custom_score"` + } + err = json.Unmarshal(out, &decoded) + if err != nil { + t.Fatal(err) + } + + if decoded.CustomScore.Source != "function score(doc, params){ return doc.score; }" { + t.Fatalf("unexpected source: %q", decoded.CustomScore.Source) + } + if len(decoded.CustomScore.Fields) != 1 || decoded.CustomScore.Fields[0] != "ibu" { + t.Fatalf("unexpected fields: %v", decoded.CustomScore.Fields) + } + if got := decoded.CustomScore.Params["weight"]; got != 0.05 { + t.Fatalf("unexpected params: %v", decoded.CustomScore.Params) + } + + child, err := ParseQuery(decoded.CustomScore.Query) + if err != nil { + t.Fatal(err) + } + mq, ok := child.(*MatchQuery) + if !ok { + t.Fatalf("expected match query child, got %T", child) + } + if mq.Match != "ipa" { + t.Fatalf("expected marshaled child query to be rewritten to ipa, got %q", mq.Match) + } +} diff --git a/search/query/custom_score.go b/search/query/custom_score.go new file mode 100644 index 0000000..5392519 --- /dev/null +++ b/search/query/custom_score.go @@ -0,0 +1,137 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +// CustomScoreQuery wraps a child query and re-scores its candidate matches via +// an embedder-provided per-hit callback. +type CustomScoreQuery struct { + Query Query `json:"query"` + Fields []string `json:"fields,omitempty"` + + scoreFunc searcher.CustomScoreFunc + payload map[string]interface{} +} + +// CustomScoreQueryParser lets an embedder override parsing of +// {"custom_score": ...} nodes. It is intended to be assigned once during +// process startup or init, before any queries are parsed; callers must not +// mutate it concurrently with ParseQuery(). For example: +// +// func init() { +// query.CustomScoreQueryParser = parseCustomScoreQuery +// } +var CustomScoreQueryParser func([]byte) (Query, error) + +func NewCustomScoreQueryWithScorer(query Query, score searcher.CustomScoreFunc, fields []string, payload map[string]interface{}) *CustomScoreQuery { + return &CustomScoreQuery{ + Query: query, + Fields: fields, + scoreFunc: score, + payload: payload, + } +} + +func (q *CustomScoreQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + if q == nil { + return nil, fmt.Errorf("custom score query is nil") + } + if q.Query == nil { + return nil, fmt.Errorf("custom score query must have a query") + } + if q.scoreFunc == nil { + return nil, fmt.Errorf("custom score query must have a score callback") + } + + // Build the inner searcher first; custom scoring wraps its output. + childSearcher, err := q.Query.Searcher(ctx, i, m, options) + if err != nil { + return nil, err + } + + // Create a doc value reader for the requested fields (if any) so the + // searcher can populate d.Fields before invoking the callback. + var dvReader index.DocValueReader + var fieldTypes map[string]string + if len(q.Fields) > 0 { + // Expand a "*" wildcard to the concrete field set; the doc-value + // reader matches field names literally and would otherwise load + // nothing, leaving d.Fields empty in the callback. + fields, err2 := expandFieldWildcard(q.Fields, i) + if err2 != nil { + _ = childSearcher.Close() + return nil, err2 + } + dvReader, err2 = i.DocValueReader(fields) + if err2 != nil { + _ = childSearcher.Close() + return nil, err2 + } + fieldTypes = resolveFieldTypes(fields, m) + } + + return searcher.NewCustomScoreSearcher(ctx, childSearcher, q.scoreFunc, dvReader, i, fieldTypes, options.Explain), nil +} + +func (q *CustomScoreQuery) Validate() error { + if q == nil { + return fmt.Errorf("custom score query is nil") + } + if q.Query == nil { + return fmt.Errorf("custom score query must have a query") + } + if q.scoreFunc == nil { + return fmt.Errorf("custom score query must have a score callback") + } + if vq, ok := q.Query.(ValidatableQuery); ok { + return vq.Validate() + } + return nil +} + +func (q *CustomScoreQuery) MarshalJSON() ([]byte, error) { + inner := make(map[string]interface{}, len(q.payload)+2) + for k, v := range q.payload { + inner[k] = v + } + inner["query"] = q.Query + if len(q.Fields) > 0 { + inner["fields"] = q.Fields + } + return json.Marshal(map[string]interface{}{ + "custom_score": inner, + }) +} + +func (q *CustomScoreQuery) UnmarshalJSON(data []byte) error { + child, fields, payload, err := unmarshalCustomQueryPayload(data, "custom_score") + if err != nil { + return err + } + q.Query = child + q.Fields = fields + q.payload = payload + return nil +} diff --git a/search/query/date_range.go b/search/query/date_range.go new file mode 100644 index 0000000..47012fb --- /dev/null +++ b/search/query/date_range.go @@ -0,0 +1,192 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + "math" + "time" + + "github.com/blevesearch/bleve/v2/analysis/datetime/optional" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +// QueryDateTimeParser controls the default query date time parser. +var QueryDateTimeParser = optional.Name + +// QueryDateTimeFormat controls the format when Marshaling to JSON. +var QueryDateTimeFormat = time.RFC3339 + +var cache = registry.NewCache() + +type BleveQueryTime struct { + time.Time +} + +var MinRFC3339CompatibleTime time.Time +var MaxRFC3339CompatibleTime time.Time + +func init() { + MinRFC3339CompatibleTime, _ = time.Parse(time.RFC3339, "1677-12-01T00:00:00Z") + MaxRFC3339CompatibleTime, _ = time.Parse(time.RFC3339, "2262-04-11T11:59:59Z") +} + +func queryTimeFromString(t string) (time.Time, error) { + dateTimeParser, err := cache.DateTimeParserNamed(QueryDateTimeParser) + if err != nil { + return time.Time{}, err + } + rv, _, err := dateTimeParser.ParseDateTime(t) + if err != nil { + return time.Time{}, err + } + return rv, nil +} + +func (t *BleveQueryTime) MarshalJSON() ([]byte, error) { + tt := time.Time(t.Time) + return []byte("\"" + tt.Format(QueryDateTimeFormat) + "\""), nil +} + +func (t *BleveQueryTime) UnmarshalJSON(data []byte) error { + var timeString string + err := util.UnmarshalJSON(data, &timeString) + if err != nil { + return err + } + dateTimeParser, err := cache.DateTimeParserNamed(QueryDateTimeParser) + if err != nil { + return err + } + t.Time, _, err = dateTimeParser.ParseDateTime(timeString) + if err != nil { + return err + } + return nil +} + +type DateRangeQuery struct { + Start BleveQueryTime `json:"start,omitempty"` + End BleveQueryTime `json:"end,omitempty"` + InclusiveStart *bool `json:"inclusive_start,omitempty"` + InclusiveEnd *bool `json:"inclusive_end,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewDateRangeQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser configured in the +// top-level config.QueryDateTimeParser +// Either, but not both endpoints can be nil. +func NewDateRangeQuery(start, end time.Time) *DateRangeQuery { + return NewDateRangeInclusiveQuery(start, end, nil, nil) +} + +// NewDateRangeInclusiveQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser configured in the +// top-level config.QueryDateTimeParser +// Either, but not both endpoints can be nil. +// startInclusive and endInclusive control inclusion of the endpoints. +func NewDateRangeInclusiveQuery(start, end time.Time, startInclusive, endInclusive *bool) *DateRangeQuery { + return &DateRangeQuery{ + Start: BleveQueryTime{start}, + End: BleveQueryTime{end}, + InclusiveStart: startInclusive, + InclusiveEnd: endInclusive, + } +} + +func (q *DateRangeQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *DateRangeQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *DateRangeQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *DateRangeQuery) Field() string { + return q.FieldVal +} + +func (q *DateRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + min, max, err := q.parseEndpoints() + if err != nil { + return nil, err + } + + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + return searcher.NewNumericRangeSearcher(ctx, i, min, max, q.InclusiveStart, q.InclusiveEnd, field, q.BoostVal.Value(), options) +} + +func (q *DateRangeQuery) parseEndpoints() (*float64, *float64, error) { + min := math.Inf(-1) + max := math.Inf(1) + if !q.Start.IsZero() { + if !isDatetimeCompatible(q.Start) { + // overflow + return nil, nil, fmt.Errorf("invalid/unsupported date range, start: %v", q.Start) + } + startInt64 := q.Start.UnixNano() + min = numeric.Int64ToFloat64(startInt64) + } + if !q.End.IsZero() { + if !isDatetimeCompatible(q.End) { + // overflow + return nil, nil, fmt.Errorf("invalid/unsupported date range, end: %v", q.End) + } + endInt64 := q.End.UnixNano() + max = numeric.Int64ToFloat64(endInt64) + } + + return &min, &max, nil +} + +func (q *DateRangeQuery) Validate() error { + if q.Start.IsZero() && q.End.IsZero() { + return fmt.Errorf("must specify start or end") + } + _, _, err := q.parseEndpoints() + if err != nil { + return err + } + return nil +} + +func isDatetimeCompatible(t BleveQueryTime) bool { + if QueryDateTimeFormat == time.RFC3339 && + (t.Before(MinRFC3339CompatibleTime) || t.After(MaxRFC3339CompatibleTime)) { + return false + } + + return true +} diff --git a/search/query/date_range_string.go b/search/query/date_range_string.go new file mode 100644 index 0000000..ac10719 --- /dev/null +++ b/search/query/date_range_string.go @@ -0,0 +1,176 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + "math" + "time" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +// DateRangeStringQuery represents a query for a range of date values. +// Start and End are the range endpoints, as strings. +// Start and End are parsed using DateTimeParser, which is a custom date time parser +// defined in the index mapping. If DateTimeParser is not specified, then the +// top-level config.QueryDateTimeParser is used. +type DateRangeStringQuery struct { + Start string `json:"start,omitempty"` + End string `json:"end,omitempty"` + InclusiveStart *bool `json:"inclusive_start,omitempty"` + InclusiveEnd *bool `json:"inclusive_end,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + DateTimeParser string `json:"datetime_parser,omitempty"` +} + +// NewDateRangeStringQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser field of the query struct, +// which is a custom date time parser defined in the index mapping. +// if DateTimeParser is not specified, then the +// top-level config.QueryDateTimeParser is used. +// Either, but not both endpoints can be nil. +func NewDateRangeStringQuery(start, end string) *DateRangeStringQuery { + return NewDateRangeStringInclusiveQuery(start, end, nil, nil) +} + +// NewDateRangeStringInclusiveQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser field of the query struct, +// which is a custom date time parser defined in the index mapping. +// if DateTimeParser is not specified, then the +// top-level config.QueryDateTimeParser is used. +// Either, but not both endpoints can be nil. +// startInclusive and endInclusive control inclusion of the endpoints. +func NewDateRangeStringInclusiveQuery(start, end string, startInclusive, endInclusive *bool) *DateRangeStringQuery { + return &DateRangeStringQuery{ + Start: start, + End: end, + InclusiveStart: startInclusive, + InclusiveEnd: endInclusive, + } +} + +func (q *DateRangeStringQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *DateRangeStringQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *DateRangeStringQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *DateRangeStringQuery) Field() string { + return q.FieldVal +} + +func (q *DateRangeStringQuery) SetDateTimeParser(d string) { + q.DateTimeParser = d +} + +func (q *DateRangeStringQuery) DateTimeParserName() string { + return q.DateTimeParser +} + +func (q *DateRangeStringQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + dateTimeParserName := QueryDateTimeParser + if q.DateTimeParser != "" { + dateTimeParserName = q.DateTimeParser + } + dateTimeParser := m.DateTimeParserNamed(dateTimeParserName) + if dateTimeParser == nil { + return nil, fmt.Errorf("no dateTimeParser named '%s' registered", dateTimeParserName) + } + + var startTime, endTime time.Time + var err error + if q.Start != "" { + startTime, _, err = dateTimeParser.ParseDateTime(q.Start) + if err != nil { + return nil, fmt.Errorf("%v, date time parser name: %s", err, dateTimeParserName) + } + } + if q.End != "" { + endTime, _, err = dateTimeParser.ParseDateTime(q.End) + if err != nil { + return nil, fmt.Errorf("%v, date time parser name: %s", err, dateTimeParserName) + } + } + + min, max, err := q.parseEndpoints(startTime, endTime) + if err != nil { + return nil, err + } + return searcher.NewNumericRangeSearcher(ctx, i, min, max, q.InclusiveStart, q.InclusiveEnd, field, q.BoostVal.Value(), options) +} + +func (q *DateRangeStringQuery) parseEndpoints(startTime, endTime time.Time) (*float64, *float64, error) { + min := math.Inf(-1) + max := math.Inf(1) + + if startTime.IsZero() && endTime.IsZero() { + return nil, nil, fmt.Errorf("date range query must specify at least one of start/end") + } + + if !startTime.IsZero() { + if !isDateTimeWithinRange(startTime) { + // overflow + return nil, nil, fmt.Errorf("invalid/unsupported date range, start: %v", q.Start) + } + startInt64 := startTime.UnixNano() + min = numeric.Int64ToFloat64(startInt64) + } + if !endTime.IsZero() { + if !isDateTimeWithinRange(endTime) { + // overflow + return nil, nil, fmt.Errorf("invalid/unsupported date range, end: %v", q.End) + } + endInt64 := endTime.UnixNano() + max = numeric.Int64ToFloat64(endInt64) + } + + return &min, &max, nil +} + +func (q *DateRangeStringQuery) Validate() error { + // either start or end must be specified + if q.Start == "" && q.End == "" { + return fmt.Errorf("date range query must specify at least one of start/end") + } + return nil +} + +func isDateTimeWithinRange(t time.Time) bool { + if t.Before(MinRFC3339CompatibleTime) || t.After(MaxRFC3339CompatibleTime) { + return false + } + return true +} diff --git a/search/query/date_range_test.go b/search/query/date_range_test.go new file mode 100644 index 0000000..bbf8280 --- /dev/null +++ b/search/query/date_range_test.go @@ -0,0 +1,132 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "encoding/json" + "testing" + "time" +) + +func TestBleveQueryTime(t *testing.T) { + testTimes := []time.Time{ + time.Now(), + {}, + } + + for i, testTime := range testTimes { + bqt := &BleveQueryTime{testTime} + + buf, err := json.Marshal(bqt) + if err != nil { + t.Errorf("expected no err") + } + + var bqt2 BleveQueryTime + err = json.Unmarshal(buf, &bqt2) + if err != nil { + t.Errorf("expected no unmarshal err, got: %v", err) + } + + if bqt.Time.Format(time.RFC3339) != bqt2.Time.Format(time.RFC3339) { + t.Errorf("test %d - expected same time, %#v != %#v", i, bqt.Time, bqt2.Time) + } + + if testTime.Format(time.RFC3339) != bqt2.Time.Format(time.RFC3339) { + t.Errorf("test %d - expected orig time, %#v != %#v", i, testTime, bqt2.Time) + } + } +} + +func TestValidateDatetimeRanges(t *testing.T) { + tests := []struct { + start string + end string + expect bool + }{ + { + start: "2019-03-22T13:25:00Z", + end: "2019-03-22T18:25:00Z", + expect: true, + }, + { + start: "2019-03-22T13:25:00Z", + end: "9999-03-22T13:25:00Z", + expect: false, + }, + { + start: "2019-03-22T13:25:00Z", + end: "2262-04-11T11:59:59Z", + expect: true, + }, + { + start: "2019-03-22T13:25:00Z", + end: "2262-04-12T00:00:00Z", + expect: false, + }, + { + start: "1950-03-22T12:23:23Z", + end: "1960-02-21T15:23:34Z", + expect: true, + }, + { + start: "0001-01-01T00:00:00Z", + end: "0001-01-01T00:00:00Z", + expect: false, + }, + { + start: "0001-01-01T00:00:00Z", + end: "2000-01-01T00:00:00Z", + expect: true, + }, + { + start: "1677-11-30T11:59:59Z", + end: "2262-04-11T11:59:59Z", + expect: false, + }, + { + start: "2262-04-12T00:00:00Z", + end: "2262-04-11T11:59:59Z", + expect: false, + }, + { + start: "1677-12-01T00:00:00Z", + end: "2262-04-12T00:00:00Z", + expect: false, + }, + { + start: "1677-12-01T00:00:00Z", + end: "1677-11-30T11:59:59Z", + expect: false, + }, + { + start: "1677-12-01T00:00:00Z", + end: "2262-04-11T11:59:59Z", + expect: true, + }, + } + + for _, test := range tests { + startTime, _ := time.Parse(time.RFC3339, test.start) + endTime, _ := time.Parse(time.RFC3339, test.end) + + dateRangeQuery := NewDateRangeQuery(startTime, endTime) + if (dateRangeQuery.Validate() == nil) != test.expect { + t.Errorf("unexpected results while validating date range query with"+ + " {start: %v, end: %v}, expected: %v", + test.start, test.end, test.expect) + } + } +} diff --git a/search/query/disjunction.go b/search/query/disjunction.go new file mode 100644 index 0000000..da46478 --- /dev/null +++ b/search/query/disjunction.go @@ -0,0 +1,134 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +type DisjunctionQuery struct { + Disjuncts []Query `json:"disjuncts"` + BoostVal *Boost `json:"boost,omitempty"` + Min float64 `json:"min"` + retrieveScoreBreakdown bool + queryStringMode bool +} + +func (q *DisjunctionQuery) RetrieveScoreBreakdown(b bool) { + q.retrieveScoreBreakdown = b +} + +// NewDisjunctionQuery creates a new compound Query. +// Result documents satisfy at least one Query. +func NewDisjunctionQuery(disjuncts []Query) *DisjunctionQuery { + return &DisjunctionQuery{ + Disjuncts: disjuncts, + } +} + +func (q *DisjunctionQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *DisjunctionQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *DisjunctionQuery) AddQuery(aq ...Query) { + q.Disjuncts = append(q.Disjuncts, aq...) +} + +func (q *DisjunctionQuery) SetMin(m float64) { + q.Min = m +} + +func (q *DisjunctionQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, + options search.SearcherOptions, +) (search.Searcher, error) { + ss := make([]search.Searcher, 0, len(q.Disjuncts)) + for _, disjunct := range q.Disjuncts { + sr, err := disjunct.Searcher(ctx, i, m, options) + if err != nil { + for _, searcher := range ss { + if searcher != nil { + _ = searcher.Close() + } + } + return nil, err + } + if sr != nil { + if _, ok := sr.(*searcher.MatchNoneSearcher); ok && q.queryStringMode { + // in query string mode, skip match none + continue + } + ss = append(ss, sr) + } + } + + if len(ss) < 1 { + return searcher.NewMatchNoneSearcher(i) + } + + nctx := context.WithValue(ctx, search.IncludeScoreBreakdownKey, q.retrieveScoreBreakdown) + + return searcher.NewDisjunctionSearcher(nctx, i, ss, q.Min, options) +} + +func (q *DisjunctionQuery) Validate() error { + if int(q.Min) > len(q.Disjuncts) { + return fmt.Errorf("disjunction query has fewer than the minimum number of clauses to satisfy") + } + for _, q := range q.Disjuncts { + if q, ok := q.(ValidatableQuery); ok { + err := q.Validate() + if err != nil { + return err + } + } + } + return nil +} + +func (q *DisjunctionQuery) UnmarshalJSON(data []byte) error { + tmp := struct { + Disjuncts []json.RawMessage `json:"disjuncts"` + Boost *Boost `json:"boost,omitempty"` + Min float64 `json:"min"` + }{} + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return err + } + q.Disjuncts = make([]Query, len(tmp.Disjuncts)) + for i, term := range tmp.Disjuncts { + query, err := ParseQuery(term) + if err != nil { + return err + } + q.Disjuncts[i] = query + } + q.BoostVal = tmp.Boost + q.Min = tmp.Min + return nil +} diff --git a/search/query/docid.go b/search/query/docid.go new file mode 100644 index 0000000..7116f39 --- /dev/null +++ b/search/query/docid.go @@ -0,0 +1,51 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +type DocIDQuery struct { + IDs []string `json:"ids"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewDocIDQuery creates a new Query object returning indexed documents among +// the specified set. Combine it with ConjunctionQuery to restrict the scope of +// other queries output. +func NewDocIDQuery(ids []string) *DocIDQuery { + return &DocIDQuery{ + IDs: ids, + } +} + +func (q *DocIDQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *DocIDQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *DocIDQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + return searcher.NewDocIDSearcher(ctx, i, q.IDs, q.BoostVal.Value(), options) +} diff --git a/search/query/fuzzy.go b/search/query/fuzzy.go new file mode 100644 index 0000000..72d7c0e --- /dev/null +++ b/search/query/fuzzy.go @@ -0,0 +1,134 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +type FuzzyQuery struct { + Term string `json:"term"` + Prefix int `json:"prefix_length"` + Fuzziness int `json:"fuzziness"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + autoFuzzy bool +} + +// NewFuzzyQuery creates a new Query which finds +// documents containing terms within a specific +// fuzziness of the specified term. +// The default fuzziness is 1. +// +// The current implementation uses Levenshtein edit +// distance as the fuzziness metric. +func NewFuzzyQuery(term string) *FuzzyQuery { + return &FuzzyQuery{ + Term: term, + Fuzziness: 1, + } +} + +func (q *FuzzyQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *FuzzyQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *FuzzyQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *FuzzyQuery) Field() string { + return q.FieldVal +} + +func (q *FuzzyQuery) SetFuzziness(f int) { + q.Fuzziness = f +} + +func (q *FuzzyQuery) SetAutoFuzziness(a bool) { + q.autoFuzzy = a +} + +func (q *FuzzyQuery) SetPrefix(p int) { + q.Prefix = p +} + +func (q *FuzzyQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + if q.autoFuzzy { + return searcher.NewAutoFuzzySearcher(ctx, i, q.Term, q.Prefix, field, q.BoostVal.Value(), options) + } + return searcher.NewFuzzySearcher(ctx, i, q.Term, q.Prefix, q.Fuzziness, field, q.BoostVal.Value(), options) +} + +func (q *FuzzyQuery) UnmarshalJSON(data []byte) error { + type Alias FuzzyQuery + aux := &struct { + Fuzziness interface{} `json:"fuzziness"` + *Alias + }{ + Alias: (*Alias)(q), + } + if err := util.UnmarshalJSON(data, &aux); err != nil { + return err + } + switch v := aux.Fuzziness.(type) { + case float64: + q.Fuzziness = int(v) + case string: + if v == "auto" { + q.autoFuzzy = true + } + } + return nil +} + +func (f *FuzzyQuery) MarshalJSON() ([]byte, error) { + var fuzzyValue interface{} + if f.autoFuzzy { + fuzzyValue = "auto" + } else { + fuzzyValue = f.Fuzziness + } + type fuzzyQuery struct { + Term string `json:"term"` + Prefix int `json:"prefix_length"` + Fuzziness interface{} `json:"fuzziness"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + } + aux := fuzzyQuery{ + Term: f.Term, + Prefix: f.Prefix, + Fuzziness: fuzzyValue, + FieldVal: f.FieldVal, + BoostVal: f.BoostVal, + } + return util.MarshalJSON(aux) +} diff --git a/search/query/geo_boundingbox.go b/search/query/geo_boundingbox.go new file mode 100644 index 0000000..1653e6e --- /dev/null +++ b/search/query/geo_boundingbox.go @@ -0,0 +1,119 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +type GeoBoundingBoxQuery struct { + TopLeft []float64 `json:"top_left,omitempty"` + BottomRight []float64 `json:"bottom_right,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +func NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightLat float64) *GeoBoundingBoxQuery { + return &GeoBoundingBoxQuery{ + TopLeft: []float64{topLeftLon, topLeftLat}, + BottomRight: []float64{bottomRightLon, bottomRightLat}, + } +} + +func (q *GeoBoundingBoxQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *GeoBoundingBoxQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *GeoBoundingBoxQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *GeoBoundingBoxQuery) Field() string { + return q.FieldVal +} + +func (q *GeoBoundingBoxQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + ctx = context.WithValue(ctx, search.QueryTypeKey, search.Geo) + + if q.BottomRight[0] < q.TopLeft[0] { + // cross date line, rewrite as two parts + + leftSearcher, err := searcher.NewGeoBoundingBoxSearcher(ctx, i, -180, q.BottomRight[1], q.BottomRight[0], q.TopLeft[1], field, q.BoostVal.Value(), options, true) + if err != nil { + return nil, err + } + rightSearcher, err := searcher.NewGeoBoundingBoxSearcher(ctx, i, q.TopLeft[0], q.BottomRight[1], 180, q.TopLeft[1], field, q.BoostVal.Value(), options, true) + if err != nil { + _ = leftSearcher.Close() + return nil, err + } + + return searcher.NewDisjunctionSearcher(ctx, i, []search.Searcher{leftSearcher, rightSearcher}, 0, options) + } + + return searcher.NewGeoBoundingBoxSearcher(ctx, i, q.TopLeft[0], q.BottomRight[1], q.BottomRight[0], q.TopLeft[1], field, q.BoostVal.Value(), options, true) +} + +func (q *GeoBoundingBoxQuery) Validate() error { + if q.TopLeft[1] < q.BottomRight[1] { + return fmt.Errorf("geo bounding box top left should be higher than bottom right") + } + return nil +} + +func (q *GeoBoundingBoxQuery) UnmarshalJSON(data []byte) error { + tmp := struct { + TopLeft interface{} `json:"top_left,omitempty"` + BottomRight interface{} `json:"bottom_right,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + }{} + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return err + } + // now use our generic point parsing code from the geo package + lon, lat, found := geo.ExtractGeoPoint(tmp.TopLeft) + if !found { + return fmt.Errorf("geo location top_left not in a valid format") + } + q.TopLeft = []float64{lon, lat} + lon, lat, found = geo.ExtractGeoPoint(tmp.BottomRight) + if !found { + return fmt.Errorf("geo location bottom_right not in a valid format") + } + q.BottomRight = []float64{lon, lat} + q.FieldVal = tmp.FieldVal + q.BoostVal = tmp.BoostVal + return nil +} diff --git a/search/query/geo_boundingpolygon.go b/search/query/geo_boundingpolygon.go new file mode 100644 index 0000000..7f81a7c --- /dev/null +++ b/search/query/geo_boundingpolygon.go @@ -0,0 +1,97 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +type GeoBoundingPolygonQuery struct { + Points []geo.Point `json:"polygon_points"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +func NewGeoBoundingPolygonQuery(points []geo.Point) *GeoBoundingPolygonQuery { + return &GeoBoundingPolygonQuery{ + Points: points} +} + +func (q *GeoBoundingPolygonQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *GeoBoundingPolygonQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *GeoBoundingPolygonQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *GeoBoundingPolygonQuery) Field() string { + return q.FieldVal +} + +func (q *GeoBoundingPolygonQuery) Searcher(ctx context.Context, i index.IndexReader, + m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + ctx = context.WithValue(ctx, search.QueryTypeKey, search.Geo) + + return searcher.NewGeoBoundedPolygonSearcher(ctx, i, q.Points, field, q.BoostVal.Value(), options) +} + +func (q *GeoBoundingPolygonQuery) Validate() error { + return nil +} + +func (q *GeoBoundingPolygonQuery) UnmarshalJSON(data []byte) error { + tmp := struct { + Points []interface{} `json:"polygon_points"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + }{} + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return err + } + + q.Points = make([]geo.Point, 0, len(tmp.Points)) + for _, i := range tmp.Points { + // now use our generic point parsing code from the geo package + lon, lat, found := geo.ExtractGeoPoint(i) + if !found { + return fmt.Errorf("geo polygon point: %v is not in a valid format", i) + } + q.Points = append(q.Points, geo.Point{Lon: lon, Lat: lat}) + } + + q.FieldVal = tmp.FieldVal + q.BoostVal = tmp.BoostVal + return nil +} diff --git a/search/query/geo_distance.go b/search/query/geo_distance.go new file mode 100644 index 0000000..2ca0964 --- /dev/null +++ b/search/query/geo_distance.go @@ -0,0 +1,103 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +type GeoDistanceQuery struct { + Location []float64 `json:"location,omitempty"` + Distance string `json:"distance,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +func NewGeoDistanceQuery(lon, lat float64, distance string) *GeoDistanceQuery { + return &GeoDistanceQuery{ + Location: []float64{lon, lat}, + Distance: distance, + } +} + +func (q *GeoDistanceQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *GeoDistanceQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *GeoDistanceQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *GeoDistanceQuery) Field() string { + return q.FieldVal +} + +func (q *GeoDistanceQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, + options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + ctx = context.WithValue(ctx, search.QueryTypeKey, search.Geo) + + dist, err := geo.ParseDistance(q.Distance) + if err != nil { + return nil, err + } + + return searcher.NewGeoPointDistanceSearcher(ctx, i, q.Location[0], q.Location[1], + dist, field, q.BoostVal.Value(), options) +} + +func (q *GeoDistanceQuery) Validate() error { + return nil +} + +func (q *GeoDistanceQuery) UnmarshalJSON(data []byte) error { + tmp := struct { + Location interface{} `json:"location,omitempty"` + Distance string `json:"distance,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + }{} + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return err + } + // now use our generic point parsing code from the geo package + lon, lat, found := geo.ExtractGeoPoint(tmp.Location) + if !found { + return fmt.Errorf("geo location not in a valid format") + } + q.Location = []float64{lon, lat} + q.Distance = tmp.Distance + q.FieldVal = tmp.FieldVal + q.BoostVal = tmp.BoostVal + return nil +} diff --git a/search/query/geo_shape.go b/search/query/geo_shape.go new file mode 100644 index 0000000..686f486 --- /dev/null +++ b/search/query/geo_shape.go @@ -0,0 +1,138 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "encoding/json" + + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +type Geometry struct { + Shape index.GeoJSON `json:"shape"` + Relation string `json:"relation"` +} + +type GeoShapeQuery struct { + Geometry Geometry `json:"geometry"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewGeoShapeQuery creates a geoshape query for the +// given shape type. This method can be used for +// creating geoshape queries for shape types like: point, +// linestring, polygon, multipoint, multilinestring, +// multipolygon and envelope. +func NewGeoShapeQuery(coordinates [][][][]float64, typ, + relation string) (*GeoShapeQuery, error) { + s, _, err := geo.NewGeoJsonShape(coordinates, typ) + if err != nil { + return nil, err + } + + return &GeoShapeQuery{Geometry: Geometry{Shape: s, + Relation: relation}}, nil +} + +// NewGeoShapeCircleQuery creates a geoshape query for the +// given center point and the radius. Radius formats supported: +// "5in" "5inch" "7yd" "7yards" "9ft" "9feet" "11km" "11kilometers" +// "3nm" "3nauticalmiles" "13mm" "13millimeters" "15cm" "15centimeters" +// "17mi" "17miles" "19m" "19meters" If the unit cannot be determined, +// the entire string is parsed and the unit of meters is assumed. +func NewGeoShapeCircleQuery(coordinates []float64, radius, + relation string) (*GeoShapeQuery, error) { + + s, _, err := geo.NewGeoCircleShape(coordinates, radius) + if err != nil { + return nil, err + } + + return &GeoShapeQuery{Geometry: Geometry{Shape: s, + Relation: relation}}, nil +} + +// NewGeometryCollectionQuery creates a geoshape query for the +// given geometrycollection coordinates and types. +func NewGeometryCollectionQuery(coordinates [][][][][]float64, types []string, + relation string) (*GeoShapeQuery, error) { + s, _, err := geo.NewGeometryCollection(coordinates, types) + if err != nil { + return nil, err + } + + return &GeoShapeQuery{Geometry: Geometry{Shape: s, + Relation: relation}}, nil +} + +func (q *GeoShapeQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *GeoShapeQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *GeoShapeQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *GeoShapeQuery) Field() string { + return q.FieldVal +} + +func (q *GeoShapeQuery) Searcher(ctx context.Context, i index.IndexReader, + m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + ctx = context.WithValue(ctx, search.QueryTypeKey, search.Geo) + + return searcher.NewGeoShapeSearcher(ctx, i, q.Geometry.Shape, q.Geometry.Relation, field, + q.BoostVal.Value(), options) +} + +func (q *GeoShapeQuery) Validate() error { + return nil +} + +func (q *Geometry) UnmarshalJSON(data []byte) error { + tmp := struct { + Shape json.RawMessage `json:"shape"` + Relation string `json:"relation"` + }{} + + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return err + } + + q.Shape, err = geo.ParseGeoJSONShape(tmp.Shape) + if err != nil { + return err + } + q.Relation = tmp.Relation + return nil +} diff --git a/search/query/geo_shape_v2.go b/search/query/geo_shape_v2.go new file mode 100644 index 0000000..11f89ce --- /dev/null +++ b/search/query/geo_shape_v2.go @@ -0,0 +1,107 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +type GeoShapeV2Query struct { + GeometryV2 Geometry `json:"geometry_v2,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +func NewGeoShapeV2Query(coordinates [][][][]float64, typ, + relation string) (*GeoShapeV2Query, error) { + s, _, err := geo.NewGeoJsonShape(coordinates, typ) + if err != nil { + return nil, err + } + + return &GeoShapeV2Query{GeometryV2: Geometry{Shape: s, + Relation: relation}}, nil +} + +func NewGeoShapeV2CircleQuery(center []float64, radius, + relation string) (*GeoShapeV2Query, error) { + s, _, err := geo.NewGeoCircleShape(center, radius) + if err != nil { + return nil, err + } + + return &GeoShapeV2Query{GeometryV2: Geometry{Shape: s, + Relation: relation}}, nil +} + +func NewGeoShapeV2GeometryCollectionQuery(coordinates [][][][][]float64, + types []string, relation string) (*GeoShapeV2Query, error) { + s, _, err := geo.NewGeometryCollection(coordinates, types) + if err != nil { + return nil, err + } + + return &GeoShapeV2Query{GeometryV2: Geometry{Shape: s, + Relation: relation}}, nil +} + +func (q *GeoShapeV2Query) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *GeoShapeV2Query) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *GeoShapeV2Query) Field() string { + return q.FieldVal +} + +func (q *GeoShapeV2Query) SetField(f string) { + q.FieldVal = f +} + +func (q *GeoShapeV2Query) Validate() error { + switch q.GeometryV2.Relation { + case "intersects", "contains", "within", "disjoint": + return nil + default: + return fmt.Errorf("invalid relation: %q for geoshape_v2 query, "+ + "valid relations are: intersects, contains, within, disjoint", + q.GeometryV2.Relation) + } +} + +func (q *GeoShapeV2Query) Searcher(ctx context.Context, + i index.IndexReader, m mapping.IndexMapping, + options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + ctx = context.WithValue(ctx, search.QueryTypeKey, search.Geo) + + return searcher.NewGeoShapeV2Searcher(ctx, i, q.GeometryV2.Shape, + q.GeometryV2.Relation, field, q.BoostVal.Value(), options) +} diff --git a/search/query/ip_range.go b/search/query/ip_range.go new file mode 100644 index 0000000..6c447c2 --- /dev/null +++ b/search/query/ip_range.go @@ -0,0 +1,85 @@ +// Copyright (c) 2021 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + "net" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +type IPRangeQuery struct { + CIDR string `json:"cidr,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +func NewIPRangeQuery(cidr string) *IPRangeQuery { + return &IPRangeQuery{ + CIDR: cidr, + } +} + +func (q *IPRangeQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *IPRangeQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *IPRangeQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *IPRangeQuery) Field() string { + return q.FieldVal +} + +func (q *IPRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + _, ipNet, err := net.ParseCIDR(q.CIDR) + if err != nil { + ip := net.ParseIP(q.CIDR) + if ip == nil { + return nil, err + } + // If we are searching for a specific ip rather than members of a network, just use a term search. + return searcher.NewTermSearcherBytes(ctx, i, ip.To16(), field, q.BoostVal.Value(), options) + } + return searcher.NewIPRangeSearcher(ctx, i, ipNet, field, q.BoostVal.Value(), options) +} + +func (q *IPRangeQuery) Validate() error { + _, _, err := net.ParseCIDR(q.CIDR) + if err == nil { + return nil + } + // We also allow search for a specific IP. + ip := net.ParseIP(q.CIDR) + if ip != nil { + return nil // we have a valid ip + } + return fmt.Errorf("IPRangeQuery must be for a network or ip address, %q", q.CIDR) +} diff --git a/search/query/knn.go b/search/query/knn.go new file mode 100644 index 0000000..5282a74 --- /dev/null +++ b/search/query/knn.go @@ -0,0 +1,101 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package query + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +type KNNQuery struct { + VectorField string `json:"field"` + Vector []float32 `json:"vector"` + K int64 `json:"k"` + BoostVal *Boost `json:"boost,omitempty"` + + // see KNNRequest.Params for description + Params json.RawMessage `json:"params"` + // elegibleSelector is used to filter out documents that are + // eligible for the KNN search from a pre-filter query. + elegibleSelector index.EligibleDocumentSelector +} + +func NewKNNQuery(vector []float32) *KNNQuery { + return &KNNQuery{Vector: vector} +} + +func (q *KNNQuery) Field() string { + return q.VectorField +} + +func (q *KNNQuery) SetK(k int64) { + q.K = k +} + +func (q *KNNQuery) SetField(field string) { + q.VectorField = field +} + +func (q *KNNQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *KNNQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *KNNQuery) SetParams(params json.RawMessage) { + q.Params = params +} + +func (q *KNNQuery) SetEligibleSelector(eligibleSelector index.EligibleDocumentSelector) { + q.elegibleSelector = eligibleSelector +} + +func (q *KNNQuery) Searcher(ctx context.Context, i index.IndexReader, + m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + fieldMapping := m.FieldMappingForPath(q.VectorField) + similarityMetric := fieldMapping.Similarity + if similarityMetric == "" { + similarityMetric = index.DefaultVectorSimilarityMetric + } + if q.K <= 0 || len(q.Vector) == 0 { + return nil, fmt.Errorf("k must be greater than 0 and vector must be non-empty") + } + // bivf-sq8 indexes only supports hamming distance for the primary + // binary index. Similarity here is used for the backing flat index, + // which is set to cosine similarity for recall reasons + if index.OptimizationRequiresBinaryIndex(fieldMapping.VectorIndexOptimizedFor) { + similarityMetric = index.CosineSimilarity + } + if similarityMetric == index.CosineSimilarity { + // normalize the vector + q.Vector = mapping.NormalizeVector(q.Vector) + } + + return searcher.NewKNNSearcher(ctx, i, m, options, q.VectorField, + q.Vector, q.K, q.BoostVal.Value(), similarityMetric, q.Params, + q.elegibleSelector) +} diff --git a/search/query/match.go b/search/query/match.go new file mode 100644 index 0000000..ba84d92 --- /dev/null +++ b/search/query/match.go @@ -0,0 +1,236 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +type MatchQuery struct { + Match string `json:"match"` + FieldVal string `json:"field,omitempty"` + Analyzer string `json:"analyzer,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + Prefix int `json:"prefix_length"` + Fuzziness int `json:"fuzziness"` + Operator MatchQueryOperator `json:"operator,omitempty"` + autoFuzzy bool +} + +type MatchQueryOperator int + +const ( + // Document must satisfy AT LEAST ONE of term searches. + MatchQueryOperatorOr = MatchQueryOperator(0) + // Document must satisfy ALL of term searches. + MatchQueryOperatorAnd = MatchQueryOperator(1) +) + +func (o MatchQueryOperator) MarshalJSON() ([]byte, error) { + switch o { + case MatchQueryOperatorOr: + return util.MarshalJSON("or") + case MatchQueryOperatorAnd: + return util.MarshalJSON("and") + default: + return nil, fmt.Errorf("cannot marshal match operator %d to JSON", o) + } +} + +func (o *MatchQueryOperator) UnmarshalJSON(data []byte) error { + var operatorString string + err := util.UnmarshalJSON(data, &operatorString) + if err != nil { + return err + } + + switch operatorString { + case "or": + *o = MatchQueryOperatorOr + return nil + case "and": + *o = MatchQueryOperatorAnd + return nil + default: + return fmt.Errorf("cannot unmarshal match operator '%v' from JSON", o) + } +} + +// NewMatchQuery creates a Query for matching text. +// An Analyzer is chosen based on the field. +// Input text is analyzed using this analyzer. +// Token terms resulting from this analysis are +// used to perform term searches. Result documents +// must satisfy at least one of these term searches. +func NewMatchQuery(match string) *MatchQuery { + return &MatchQuery{ + Match: match, + Operator: MatchQueryOperatorOr, + } +} + +func (q *MatchQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *MatchQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *MatchQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *MatchQuery) Field() string { + return q.FieldVal +} + +func (q *MatchQuery) SetFuzziness(f int) { + q.Fuzziness = f +} + +func (q *MatchQuery) SetAutoFuzziness(auto bool) { + q.autoFuzzy = auto +} + +func (q *MatchQuery) SetPrefix(p int) { + q.Prefix = p +} + +func (q *MatchQuery) SetOperator(operator MatchQueryOperator) { + q.Operator = operator +} + +func (q *MatchQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + analyzerName := "" + if q.Analyzer != "" { + analyzerName = q.Analyzer + } else { + analyzerName = m.AnalyzerNameForPath(field) + } + analyzer := m.AnalyzerNamed(analyzerName) + + if analyzer == nil { + return nil, fmt.Errorf("no analyzer named '%s' registered", q.Analyzer) + } + + tokens := analyzer.Analyze([]byte(q.Match)) + if len(tokens) > 0 { + + tqs := make([]Query, len(tokens)) + if q.Fuzziness != 0 || q.autoFuzzy { + for i, token := range tokens { + query := NewFuzzyQuery(string(token.Term)) + if q.autoFuzzy { + query.SetAutoFuzziness(true) + } else { + query.SetFuzziness(q.Fuzziness) + } + query.SetPrefix(q.Prefix) + query.SetField(field) + query.SetBoost(q.BoostVal.Value()) + tqs[i] = query + } + } else { + for i, token := range tokens { + tq := NewTermQuery(string(token.Term)) + tq.SetField(field) + tq.SetBoost(q.BoostVal.Value()) + tqs[i] = tq + } + } + + switch q.Operator { + case MatchQueryOperatorOr: + shouldQuery := NewDisjunctionQuery(tqs) + shouldQuery.SetMin(1) + shouldQuery.SetBoost(q.BoostVal.Value()) + return shouldQuery.Searcher(ctx, i, m, options) + + case MatchQueryOperatorAnd: + mustQuery := NewConjunctionQuery(tqs) + mustQuery.SetBoost(q.BoostVal.Value()) + return mustQuery.Searcher(ctx, i, m, options) + + default: + return nil, fmt.Errorf("unhandled operator %d", q.Operator) + } + } + noneQuery := NewMatchNoneQuery() + return noneQuery.Searcher(ctx, i, m, options) +} + +func (q *MatchQuery) UnmarshalJSON(data []byte) error { + type Alias MatchQuery + aux := &struct { + Fuzziness interface{} `json:"fuzziness"` + *Alias + }{ + Alias: (*Alias)(q), + } + if err := util.UnmarshalJSON(data, &aux); err != nil { + return err + } + switch v := aux.Fuzziness.(type) { + case float64: + q.Fuzziness = int(v) + case string: + if v == "auto" { + q.autoFuzzy = true + } + } + return nil +} + +func (f *MatchQuery) MarshalJSON() ([]byte, error) { + var fuzzyValue interface{} + if f.autoFuzzy { + fuzzyValue = "auto" + } else { + fuzzyValue = f.Fuzziness + } + type match struct { + Match string `json:"match"` + FieldVal string `json:"field,omitempty"` + Analyzer string `json:"analyzer,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + Prefix int `json:"prefix_length"` + Fuzziness interface{} `json:"fuzziness"` + Operator MatchQueryOperator `json:"operator,omitempty"` + } + aux := match{ + Match: f.Match, + FieldVal: f.FieldVal, + Analyzer: f.Analyzer, + BoostVal: f.BoostVal, + Prefix: f.Prefix, + Fuzziness: fuzzyValue, + Operator: f.Operator, + } + return util.MarshalJSON(aux) +} diff --git a/search/query/match_all.go b/search/query/match_all.go new file mode 100644 index 0000000..e88825a --- /dev/null +++ b/search/query/match_all.go @@ -0,0 +1,56 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "encoding/json" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +type MatchAllQuery struct { + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewMatchAllQuery creates a Query which will +// match all documents in the index. +func NewMatchAllQuery() *MatchAllQuery { + return &MatchAllQuery{} +} + +func (q *MatchAllQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *MatchAllQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *MatchAllQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + return searcher.NewMatchAllSearcher(ctx, i, q.BoostVal.Value(), options) +} + +func (q *MatchAllQuery) MarshalJSON() ([]byte, error) { + tmp := map[string]interface{}{ + "boost": q.BoostVal, + "match_all": map[string]interface{}{}, + } + return json.Marshal(tmp) +} diff --git a/search/query/match_none.go b/search/query/match_none.go new file mode 100644 index 0000000..cb65a72 --- /dev/null +++ b/search/query/match_none.go @@ -0,0 +1,56 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "encoding/json" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +type MatchNoneQuery struct { + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewMatchNoneQuery creates a Query which will not +// match any documents in the index. +func NewMatchNoneQuery() *MatchNoneQuery { + return &MatchNoneQuery{} +} + +func (q *MatchNoneQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *MatchNoneQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *MatchNoneQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + return searcher.NewMatchNoneSearcher(i) +} + +func (q *MatchNoneQuery) MarshalJSON() ([]byte, error) { + tmp := map[string]interface{}{ + "boost": q.BoostVal, + "match_none": map[string]interface{}{}, + } + return json.Marshal(tmp) +} diff --git a/search/query/match_phrase.go b/search/query/match_phrase.go new file mode 100644 index 0000000..12a8396 --- /dev/null +++ b/search/query/match_phrase.go @@ -0,0 +1,176 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +type MatchPhraseQuery struct { + MatchPhrase string `json:"match_phrase"` + FieldVal string `json:"field,omitempty"` + Analyzer string `json:"analyzer,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + Fuzziness int `json:"fuzziness"` + autoFuzzy bool +} + +// NewMatchPhraseQuery creates a new Query object +// for matching phrases in the index. +// An Analyzer is chosen based on the field. +// Input text is analyzed using this analyzer. +// Token terms resulting from this analysis are +// used to build a search phrase. Result documents +// must match this phrase. Queried field must have been indexed with +// IncludeTermVectors set to true. +func NewMatchPhraseQuery(matchPhrase string) *MatchPhraseQuery { + return &MatchPhraseQuery{ + MatchPhrase: matchPhrase, + } +} + +func (q *MatchPhraseQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *MatchPhraseQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *MatchPhraseQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *MatchPhraseQuery) SetFuzziness(f int) { + q.Fuzziness = f +} + +func (q *MatchPhraseQuery) SetAutoFuzziness(auto bool) { + q.autoFuzzy = auto +} + +func (q *MatchPhraseQuery) Field() string { + return q.FieldVal +} + +func (q *MatchPhraseQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + analyzerName := "" + if q.Analyzer != "" { + analyzerName = q.Analyzer + } else { + analyzerName = m.AnalyzerNameForPath(field) + } + analyzer := m.AnalyzerNamed(analyzerName) + if analyzer == nil { + return nil, fmt.Errorf("no analyzer named '%s' registered", q.Analyzer) + } + + tokens := analyzer.Analyze([]byte(q.MatchPhrase)) + if len(tokens) > 0 { + phrase := tokenStreamToPhrase(tokens) + phraseQuery := NewMultiPhraseQuery(phrase, field) + phraseQuery.SetBoost(q.BoostVal.Value()) + if q.autoFuzzy { + phraseQuery.SetAutoFuzziness(true) + } else { + phraseQuery.SetFuzziness(q.Fuzziness) + } + return phraseQuery.Searcher(ctx, i, m, options) + } + noneQuery := NewMatchNoneQuery() + return noneQuery.Searcher(ctx, i, m, options) +} + +func tokenStreamToPhrase(tokens analysis.TokenStream) [][]string { + firstPosition := int(^uint(0) >> 1) + lastPosition := 0 + for _, token := range tokens { + if token.Position < firstPosition { + firstPosition = token.Position + } + if token.Position > lastPosition { + lastPosition = token.Position + } + } + phraseLen := lastPosition - firstPosition + 1 + if phraseLen > 0 { + rv := make([][]string, phraseLen) + for _, token := range tokens { + pos := token.Position - firstPosition + rv[pos] = append(rv[pos], string(token.Term)) + } + return rv + } + return nil +} + +func (q *MatchPhraseQuery) UnmarshalJSON(data []byte) error { + type Alias MatchPhraseQuery + aux := &struct { + Fuzziness interface{} `json:"fuzziness"` + *Alias + }{ + Alias: (*Alias)(q), + } + if err := util.UnmarshalJSON(data, &aux); err != nil { + return err + } + switch v := aux.Fuzziness.(type) { + case float64: + q.Fuzziness = int(v) + case string: + if v == "auto" { + q.autoFuzzy = true + } + } + return nil +} + +func (f *MatchPhraseQuery) MarshalJSON() ([]byte, error) { + var fuzzyValue interface{} + if f.autoFuzzy { + fuzzyValue = "auto" + } else { + fuzzyValue = f.Fuzziness + } + type matchPhrase struct { + MatchPhrase string `json:"match_phrase"` + FieldVal string `json:"field,omitempty"` + Analyzer string `json:"analyzer,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + Fuzziness interface{} `json:"fuzziness"` + } + aux := matchPhrase{ + MatchPhrase: f.MatchPhrase, + FieldVal: f.FieldVal, + Analyzer: f.Analyzer, + BoostVal: f.BoostVal, + Fuzziness: fuzzyValue, + } + return util.MarshalJSON(aux) +} diff --git a/search/query/match_phrase_test.go b/search/query/match_phrase_test.go new file mode 100644 index 0000000..8ed4888 --- /dev/null +++ b/search/query/match_phrase_test.go @@ -0,0 +1,101 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" +) + +func TestTokenStreamToPhrase(t *testing.T) { + + tests := []struct { + tokens analysis.TokenStream + result [][]string + }{ + // empty token stream returns nil + { + tokens: analysis.TokenStream{}, + result: nil, + }, + // typical token + { + tokens: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("one"), + Position: 1, + }, + &analysis.Token{ + Term: []byte("two"), + Position: 2, + }, + }, + result: [][]string{{"one"}, {"two"}}, + }, + // token stream containing a gap (usually from stop words) + { + tokens: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("wag"), + Position: 1, + }, + &analysis.Token{ + Term: []byte("dog"), + Position: 3, + }, + }, + result: [][]string{{"wag"}, nil, {"dog"}}, + }, + // token stream containing multiple tokens at the same position + { + tokens: analysis.TokenStream{ + &analysis.Token{ + Term: []byte("nia"), + Position: 1, + }, + &analysis.Token{ + Term: []byte("onia"), + Position: 1, + }, + &analysis.Token{ + Term: []byte("donia"), + Position: 1, + }, + &analysis.Token{ + Term: []byte("imo"), + Position: 2, + }, + &analysis.Token{ + Term: []byte("nimo"), + Position: 2, + }, + &analysis.Token{ + Term: []byte("ónimo"), + Position: 2, + }, + }, + result: [][]string{{"nia", "onia", "donia"}, {"imo", "nimo", "ónimo"}}, + }, + } + + for i, test := range tests { + actual := tokenStreamToPhrase(test.tokens) + if !reflect.DeepEqual(actual, test.result) { + t.Fatalf("expected %#v got %#v for test %d", test.result, actual, i) + } + } +} diff --git a/search/query/multi_phrase.go b/search/query/multi_phrase.go new file mode 100644 index 0000000..aa2cc04 --- /dev/null +++ b/search/query/multi_phrase.go @@ -0,0 +1,130 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +type MultiPhraseQuery struct { + Terms [][]string `json:"terms"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + Fuzziness int `json:"fuzziness"` + autoFuzzy bool +} + +// NewMultiPhraseQuery creates a new Query for finding +// term phrases in the index. +// It is like PhraseQuery, but each position in the +// phrase may be satisfied by a list of terms +// as opposed to just one. +// At least one of the terms must exist in the correct +// order, at the correct index offsets, in the +// specified field. Queried field must have been indexed with +// IncludeTermVectors set to true. +func NewMultiPhraseQuery(terms [][]string, field string) *MultiPhraseQuery { + return &MultiPhraseQuery{ + Terms: terms, + FieldVal: field, + } +} + +func (q *MultiPhraseQuery) SetFuzziness(f int) { + q.Fuzziness = f +} + +func (q *MultiPhraseQuery) SetAutoFuzziness(auto bool) { + q.autoFuzzy = auto +} + +func (q *MultiPhraseQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *MultiPhraseQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *MultiPhraseQuery) Field() string { + return q.FieldVal +} + +func (q *MultiPhraseQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *MultiPhraseQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + return searcher.NewMultiPhraseSearcher(ctx, i, q.Terms, q.Fuzziness, q.autoFuzzy, q.FieldVal, q.BoostVal.Value(), options) +} + +func (q *MultiPhraseQuery) Validate() error { + if len(q.Terms) < 1 { + return fmt.Errorf("phrase query must contain at least one term") + } + return nil +} + +func (q *MultiPhraseQuery) UnmarshalJSON(data []byte) error { + type Alias MultiPhraseQuery + aux := &struct { + Fuzziness interface{} `json:"fuzziness"` + *Alias + }{ + Alias: (*Alias)(q), + } + if err := util.UnmarshalJSON(data, &aux); err != nil { + return err + } + switch v := aux.Fuzziness.(type) { + case float64: + q.Fuzziness = int(v) + case string: + if v == "auto" { + q.autoFuzzy = true + } + } + return nil +} + +func (f *MultiPhraseQuery) MarshalJSON() ([]byte, error) { + var fuzzyValue interface{} + if f.autoFuzzy { + fuzzyValue = "auto" + } else { + fuzzyValue = f.Fuzziness + } + type multiPhraseQuery struct { + Terms [][]string `json:"terms"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + Fuzziness interface{} `json:"fuzziness"` + } + aux := multiPhraseQuery{ + Terms: f.Terms, + FieldVal: f.FieldVal, + BoostVal: f.BoostVal, + Fuzziness: fuzzyValue, + } + return util.MarshalJSON(aux) +} diff --git a/search/query/numeric_range.go b/search/query/numeric_range.go new file mode 100644 index 0000000..205ceec --- /dev/null +++ b/search/query/numeric_range.go @@ -0,0 +1,89 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +type NumericRangeQuery struct { + Min *float64 `json:"min,omitempty"` + Max *float64 `json:"max,omitempty"` + InclusiveMin *bool `json:"inclusive_min,omitempty"` + InclusiveMax *bool `json:"inclusive_max,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewNumericRangeQuery creates a new Query for ranges +// of numeric values. +// Either, but not both endpoints can be nil. +// The minimum value is inclusive. +// The maximum value is exclusive. +func NewNumericRangeQuery(min, max *float64) *NumericRangeQuery { + return NewNumericRangeInclusiveQuery(min, max, nil, nil) +} + +// NewNumericRangeInclusiveQuery creates a new Query for ranges +// of numeric values. +// Either, but not both endpoints can be nil. +// Control endpoint inclusion with inclusiveMin, inclusiveMax. +func NewNumericRangeInclusiveQuery(min, max *float64, minInclusive, maxInclusive *bool) *NumericRangeQuery { + return &NumericRangeQuery{ + Min: min, + Max: max, + InclusiveMin: minInclusive, + InclusiveMax: maxInclusive, + } +} + +func (q *NumericRangeQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *NumericRangeQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *NumericRangeQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *NumericRangeQuery) Field() string { + return q.FieldVal +} + +func (q *NumericRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + ctx = context.WithValue(ctx, search.QueryTypeKey, search.Numeric) + return searcher.NewNumericRangeSearcher(ctx, i, q.Min, q.Max, q.InclusiveMin, q.InclusiveMax, field, q.BoostVal.Value(), options) +} + +func (q *NumericRangeQuery) Validate() error { + if q.Min == nil && q.Min == q.Max { + return fmt.Errorf("numeric range query must specify min or max") + } + return nil +} diff --git a/search/query/phrase.go b/search/query/phrase.go new file mode 100644 index 0000000..96bc1b7 --- /dev/null +++ b/search/query/phrase.go @@ -0,0 +1,127 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +type PhraseQuery struct { + Terms []string `json:"terms"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + Fuzziness int `json:"fuzziness"` + autoFuzzy bool +} + +// NewPhraseQuery creates a new Query for finding +// exact term phrases in the index. +// The provided terms must exist in the correct +// order, at the correct index offsets, in the +// specified field. Queried field must have been indexed with +// IncludeTermVectors set to true. +func NewPhraseQuery(terms []string, field string) *PhraseQuery { + return &PhraseQuery{ + Terms: terms, + FieldVal: field, + } +} + +func (q *PhraseQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *PhraseQuery) SetFuzziness(f int) { + q.Fuzziness = f +} + +func (q *PhraseQuery) SetAutoFuzziness(auto bool) { + q.autoFuzzy = auto +} + +func (q *PhraseQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *PhraseQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *PhraseQuery) Field() string { + return q.FieldVal +} + +func (q *PhraseQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + return searcher.NewPhraseSearcher(ctx, i, q.Terms, q.Fuzziness, q.autoFuzzy, q.FieldVal, q.BoostVal.Value(), options) +} + +func (q *PhraseQuery) Validate() error { + if len(q.Terms) < 1 { + return fmt.Errorf("phrase query must contain at least one term") + } + return nil +} + +func (q *PhraseQuery) UnmarshalJSON(data []byte) error { + type Alias PhraseQuery + aux := &struct { + Fuzziness interface{} `json:"fuzziness"` + *Alias + }{ + Alias: (*Alias)(q), + } + if err := util.UnmarshalJSON(data, &aux); err != nil { + return err + } + switch v := aux.Fuzziness.(type) { + case float64: + q.Fuzziness = int(v) + case string: + if v == "auto" { + q.autoFuzzy = true + } + } + return nil +} + +func (f *PhraseQuery) MarshalJSON() ([]byte, error) { + var fuzzyValue interface{} + if f.autoFuzzy { + fuzzyValue = "auto" + } else { + fuzzyValue = f.Fuzziness + } + type phraseQuery struct { + Terms []string `json:"terms"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + Fuzziness interface{} `json:"fuzziness"` + } + aux := phraseQuery{ + Terms: f.Terms, + FieldVal: f.FieldVal, + BoostVal: f.BoostVal, + Fuzziness: fuzzyValue, + } + return util.MarshalJSON(aux) +} diff --git a/search/query/prefix.go b/search/query/prefix.go new file mode 100644 index 0000000..debbbc1 --- /dev/null +++ b/search/query/prefix.go @@ -0,0 +1,64 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +type PrefixQuery struct { + Prefix string `json:"prefix"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewPrefixQuery creates a new Query which finds +// documents containing terms that start with the +// specified prefix. +func NewPrefixQuery(prefix string) *PrefixQuery { + return &PrefixQuery{ + Prefix: prefix, + } +} + +func (q *PrefixQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *PrefixQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *PrefixQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *PrefixQuery) Field() string { + return q.FieldVal +} + +func (q *PrefixQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + return searcher.NewTermPrefixSearcher(ctx, i, q.Prefix, field, q.BoostVal.Value(), options) +} diff --git a/search/query/query.go b/search/query/query.go new file mode 100644 index 0000000..711b9de --- /dev/null +++ b/search/query/query.go @@ -0,0 +1,818 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log" + "strings" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +var logger = log.New(io.Discard, "bleve mapping ", log.LstdFlags) + +// SetLog sets the logger used for logging +// by default log messages are sent to io.Discard +func SetLog(l *log.Logger) { + logger = l +} + +// A Query represents a description of the type +// and parameters for a query into the index. +type Query interface { + Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, + options search.SearcherOptions) (search.Searcher, error) +} + +// A BoostableQuery represents a Query which can be boosted +// relative to other queries. +type BoostableQuery interface { + Query + SetBoost(b float64) + Boost() float64 +} + +// A FieldableQuery represents a Query which can be restricted +// to a single field. +type FieldableQuery interface { + Query + SetField(f string) + Field() string +} + +// A ValidatableQuery represents a Query which can be validated +// prior to execution. +type ValidatableQuery interface { + Query + Validate() error +} + +// ParsePreSearchData deserializes a JSON representation of +// a PreSearchData object. +func ParsePreSearchData(input []byte) (map[string]interface{}, error) { + var rv map[string]interface{} + + var tmp map[string]json.RawMessage + err := util.UnmarshalJSON(input, &tmp) + if err != nil { + return nil, err + } + + for k, v := range tmp { + switch k { + case search.KnnPreSearchDataKey: + var value []*search.DocumentMatch + if v != nil { + err := util.UnmarshalJSON(v, &value) + if err != nil { + return nil, err + } + } + if rv == nil { + rv = make(map[string]interface{}) + } + rv[search.KnnPreSearchDataKey] = value + case search.SynonymPreSearchDataKey: + var value search.FieldTermSynonymMap + if v != nil { + err := util.UnmarshalJSON(v, &value) + if err != nil { + return nil, err + } + } + if rv == nil { + rv = make(map[string]interface{}) + } + rv[search.SynonymPreSearchDataKey] = value + case search.BM25PreSearchDataKey: + var value *search.BM25Stats + if v != nil { + err := util.UnmarshalJSON(v, &value) + if err != nil { + return nil, err + } + } + if rv == nil { + rv = make(map[string]interface{}) + } + rv[search.BM25PreSearchDataKey] = value + + } + } + return rv, nil +} + +// ParseQuery deserializes a JSON representation of +// a Query object. +func ParseQuery(input []byte) (Query, error) { + if len(input) == 0 { + // interpret as a match_none query + return NewMatchNoneQuery(), nil + } + + var tmp map[string]interface{} + err := util.UnmarshalJSON(input, &tmp) + if err != nil { + return nil, err + } + + if len(tmp) == 0 { + // interpret as a match_none query + return NewMatchNoneQuery(), nil + } + + _, hasFuzziness := tmp["fuzziness"] + _, isMatchQuery := tmp["match"] + _, isMatchPhraseQuery := tmp["match_phrase"] + _, hasTerms := tmp["terms"] + if hasFuzziness && !isMatchQuery && !isMatchPhraseQuery && !hasTerms { + var rv FuzzyQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + if isMatchQuery { + var rv MatchQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + if isMatchPhraseQuery { + var rv MatchPhraseQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + if hasTerms { + var rv PhraseQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + // now try multi-phrase + var rv2 MultiPhraseQuery + err = util.UnmarshalJSON(input, &rv2) + if err != nil { + return nil, err + } + return &rv2, nil + } + return &rv, nil + } + _, isTermQuery := tmp["term"] + if isTermQuery { + var rv TermQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasMust := tmp["must"] + _, hasShould := tmp["should"] + _, hasMustNot := tmp["must_not"] + _, hasFilter := tmp["filter"] + if hasMust || hasShould || hasMustNot || hasFilter { + var rv BooleanQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasConjuncts := tmp["conjuncts"] + if hasConjuncts { + var rv ConjunctionQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasDisjuncts := tmp["disjuncts"] + if hasDisjuncts { + var rv DisjunctionQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + + _, hasSyntaxQuery := tmp["query"] + if hasSyntaxQuery { + var rv QueryStringQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasMin := tmp["min"].(float64) + _, hasMax := tmp["max"].(float64) + if hasMin || hasMax { + var rv NumericRangeQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasMinStr := tmp["min"].(string) + _, hasMaxStr := tmp["max"].(string) + if hasMinStr || hasMaxStr { + var rv TermRangeQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasStart := tmp["start"] + _, hasEnd := tmp["end"] + if hasStart || hasEnd { + var rv DateRangeStringQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasPrefix := tmp["prefix"] + if hasPrefix { + var rv PrefixQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasRegexp := tmp["regexp"] + if hasRegexp { + var rv RegexpQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasWildcard := tmp["wildcard"] + if hasWildcard { + var rv WildcardQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasMatchAll := tmp["match_all"] + if hasMatchAll { + var rv MatchAllQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasMatchNone := tmp["match_none"] + if hasMatchNone { + var rv MatchNoneQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasCustomFilter := tmp["custom_filter"] + if hasCustomFilter { + if CustomFilterQueryParser == nil { + return nil, fmt.Errorf("custom filter query parser is not registered") + } + return CustomFilterQueryParser(input) + } + _, hasCustomScore := tmp["custom_score"] + if hasCustomScore { + if CustomScoreQueryParser == nil { + return nil, fmt.Errorf("custom score query parser is not registered") + } + return CustomScoreQueryParser(input) + } + _, hasDocIds := tmp["ids"] + if hasDocIds { + var rv DocIDQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasBool := tmp["bool"] + if hasBool { + var rv BoolFieldQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasTopLeft := tmp["top_left"] + _, hasBottomRight := tmp["bottom_right"] + if hasTopLeft && hasBottomRight { + var rv GeoBoundingBoxQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasDistance := tmp["distance"] + if hasDistance { + var rv GeoDistanceQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasPoints := tmp["polygon_points"] + if hasPoints { + var rv GeoBoundingPolygonQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + + _, hasGeo := tmp["geometry"] + if hasGeo { + var rv GeoShapeQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + + _, hasGeo = tmp["geometry_v2"] + if hasGeo { + var rv GeoShapeV2Query + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + + _, hasCIDR := tmp["cidr"] + if hasCIDR { + var rv IPRangeQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + + return nil, fmt.Errorf("unknown query type") +} + +// expandQuery traverses the input query tree and returns a new tree where +// query string queries have been expanded into base queries. Returned tree may +// reference queries from the input tree or new queries. +func expandQuery(m mapping.IndexMapping, query Query) (Query, error) { + var expand func(query Query) (Query, error) + var expandSlice func(queries []Query) ([]Query, error) = func(queries []Query) ([]Query, error) { + expanded := []Query{} + for _, q := range queries { + exp, err := expand(q) + if err != nil { + return nil, err + } + expanded = append(expanded, exp) + } + return expanded, nil + } + + expand = func(query Query) (Query, error) { + switch q := query.(type) { + case *QueryStringQuery: + parsed, err := parseQuerySyntax(q.Query) + if err != nil { + return nil, fmt.Errorf("could not parse '%s': %s", q.Query, err) + } + return expand(parsed) + case *ConjunctionQuery: + children, err := expandSlice(q.Conjuncts) + if err != nil { + return nil, err + } + q.Conjuncts = children + return q, nil + case *DisjunctionQuery: + children, err := expandSlice(q.Disjuncts) + if err != nil { + return nil, err + } + q.Disjuncts = children + return q, nil + case *BooleanQuery: + var err error + q.Must, err = expand(q.Must) + if err != nil { + return nil, err + } + q.Should, err = expand(q.Should) + if err != nil { + return nil, err + } + q.MustNot, err = expand(q.MustNot) + if err != nil { + return nil, err + } + q.Filter, err = expand(q.Filter) + if err != nil { + return nil, err + } + return q, nil + default: + return query, nil + } + } + return expand(query) +} + +// DumpQuery returns a string representation of the query tree, where query +// string queries have been expanded into base queries. The output format is +// meant for debugging purpose and may change in the future. +func DumpQuery(m mapping.IndexMapping, query Query) (string, error) { + q, err := expandQuery(m, query) + if err != nil { + return "", err + } + data, err := json.MarshalIndent(q, "", " ") + return string(data), err +} + +// ExtractFields returns a set of fields referenced by the query. +// The returned set may be nil if the query does not explicitly reference any field +// and the DefaultSearchField is unset in the index mapping. +func ExtractFields(q Query, m mapping.IndexMapping, fs search.FieldSet) (search.FieldSet, error) { + if q == nil || m == nil { + return fs, nil + } + var err error + switch q := q.(type) { + case FieldableQuery: + f := q.Field() + if f == "" { + f = m.DefaultSearchField() + } + if f != "" { + if fs == nil { + fs = search.NewFieldSet() + } + fs.AddField(f) + } + case *QueryStringQuery: + var expandedQuery Query + expandedQuery, err = expandQuery(m, q) + if err == nil { + fs, err = ExtractFields(expandedQuery, m, fs) + } + case *BooleanQuery: + for _, subq := range []Query{q.Must, q.Should, q.MustNot, q.Filter} { + fs, err = ExtractFields(subq, m, fs) + if err != nil { + break + } + } + case *ConjunctionQuery: + for _, subq := range q.Conjuncts { + fs, err = ExtractFields(subq, m, fs) + if err != nil { + break + } + } + case *DisjunctionQuery: + for _, subq := range q.Disjuncts { + fs, err = ExtractFields(subq, m, fs) + if err != nil { + break + } + } + case *DocIDQuery, *MatchAllQuery: + if fs == nil { + fs = search.NewFieldSet() + } + fs.AddField("_id") + } + return fs, err +} + +const ( + FuzzyMatchType = iota + RegexpMatchType + PrefixMatchType +) + +// ExtractSynonyms extracts synonyms from the query tree and returns a map of +// field-term pairs to their synonyms. The input query tree is traversed and +// for each term query, the synonyms are extracted from the synonym source +// associated with the field. The synonyms are then added to the provided map. +// The map is returned and may be nil if no synonyms were found. +func ExtractSynonyms(ctx context.Context, m mapping.SynonymMapping, r index.ThesaurusReader, + query Query, rv search.FieldTermSynonymMap, +) (search.FieldTermSynonymMap, error) { + if r == nil || m == nil || query == nil { + return rv, nil + } + var err error + resolveFieldAndSource := func(field string) (string, string) { + if field == "" { + field = m.DefaultSearchField() + } + return field, m.SynonymSourceForPath(field) + } + handleAnalyzer := func(analyzerName, field string) (analysis.Analyzer, error) { + if analyzerName == "" { + analyzerName = m.AnalyzerNameForPath(field) + } + analyzer := m.AnalyzerNamed(analyzerName) + if analyzer == nil { + return nil, fmt.Errorf("no analyzer named '%s' registered", analyzerName) + } + return analyzer, nil + } + switch q := query.(type) { + case *BooleanQuery: + rv, err = ExtractSynonyms(ctx, m, r, q.Must, rv) + if err != nil { + return nil, err + } + rv, err = ExtractSynonyms(ctx, m, r, q.Should, rv) + if err != nil { + return nil, err + } + rv, err = ExtractSynonyms(ctx, m, r, q.MustNot, rv) + if err != nil { + return nil, err + } + rv, err = ExtractSynonyms(ctx, m, r, q.Filter, rv) + if err != nil { + return nil, err + } + case *ConjunctionQuery: + for _, child := range q.Conjuncts { + rv, err = ExtractSynonyms(ctx, m, r, child, rv) + if err != nil { + return nil, err + } + } + case *DisjunctionQuery: + for _, child := range q.Disjuncts { + rv, err = ExtractSynonyms(ctx, m, r, child, rv) + if err != nil { + return nil, err + } + } + case *FuzzyQuery: + field, source := resolveFieldAndSource(q.FieldVal) + if source != "" { + fuzziness := q.Fuzziness + if q.autoFuzzy { + fuzziness = searcher.GetAutoFuzziness(q.Term) + } + rv, err = addSynonymsForTermWithMatchType(ctx, FuzzyMatchType, source, field, q.Term, fuzziness, q.Prefix, r, rv) + if err != nil { + return nil, err + } + } + case *MatchQuery, *MatchPhraseQuery: + var analyzerName, matchString, fieldVal string + var fuzziness, prefix int + var autoFuzzy bool + if mq, ok := q.(*MatchQuery); ok { + analyzerName, fieldVal, matchString, fuzziness, prefix, autoFuzzy = mq.Analyzer, mq.FieldVal, mq.Match, mq.Fuzziness, mq.Prefix, mq.autoFuzzy + } else if mpq, ok := q.(*MatchPhraseQuery); ok { + analyzerName, fieldVal, matchString, fuzziness, autoFuzzy = mpq.Analyzer, mpq.FieldVal, mpq.MatchPhrase, mpq.Fuzziness, mpq.autoFuzzy + } + field, source := resolveFieldAndSource(fieldVal) + if source != "" { + analyzer, err := handleAnalyzer(analyzerName, field) + if err != nil { + return nil, err + } + tokens := analyzer.Analyze([]byte(matchString)) + for _, token := range tokens { + if autoFuzzy { + fuzziness = searcher.GetAutoFuzziness(string(token.Term)) + } + rv, err = addSynonymsForTermWithMatchType(ctx, FuzzyMatchType, source, field, string(token.Term), fuzziness, prefix, r, rv) + if err != nil { + return nil, err + } + } + } + case *MultiPhraseQuery, *PhraseQuery: + var fieldVal string + var fuzziness int + var autoFuzzy bool + if mpq, ok := q.(*MultiPhraseQuery); ok { + fieldVal, fuzziness, autoFuzzy = mpq.FieldVal, mpq.Fuzziness, mpq.autoFuzzy + } else if pq, ok := q.(*PhraseQuery); ok { + fieldVal, fuzziness, autoFuzzy = pq.FieldVal, pq.Fuzziness, pq.autoFuzzy + } + field, source := resolveFieldAndSource(fieldVal) + if source != "" { + var terms []string + if mpq, ok := q.(*MultiPhraseQuery); ok { + for _, termGroup := range mpq.Terms { + terms = append(terms, termGroup...) + } + } else if pq, ok := q.(*PhraseQuery); ok { + terms = pq.Terms + } + for _, term := range terms { + if autoFuzzy { + fuzziness = searcher.GetAutoFuzziness(term) + } + rv, err = addSynonymsForTermWithMatchType(ctx, FuzzyMatchType, source, field, term, fuzziness, 0, r, rv) + if err != nil { + return nil, err + } + } + } + case *PrefixQuery: + field, source := resolveFieldAndSource(q.FieldVal) + if source != "" { + rv, err = addSynonymsForTermWithMatchType(ctx, PrefixMatchType, source, field, q.Prefix, 0, 0, r, rv) + if err != nil { + return nil, err + } + } + case *QueryStringQuery: + expanded, err := expandQuery(m, q) + if err != nil { + return nil, err + } + rv, err = ExtractSynonyms(ctx, m, r, expanded, rv) + if err != nil { + return nil, err + } + case *TermQuery: + field, source := resolveFieldAndSource(q.FieldVal) + if source != "" { + rv, err = addSynonymsForTerm(ctx, source, field, q.Term, r, rv) + if err != nil { + return nil, err + } + } + case *RegexpQuery: + field, source := resolveFieldAndSource(q.FieldVal) + if source != "" { + rv, err = addSynonymsForTermWithMatchType(ctx, RegexpMatchType, source, field, strings.TrimPrefix(q.Regexp, "^"), 0, 0, r, rv) + if err != nil { + return nil, err + } + } + case *WildcardQuery: + field, source := resolveFieldAndSource(q.FieldVal) + if source != "" { + rv, err = addSynonymsForTermWithMatchType(ctx, RegexpMatchType, source, field, wildcardRegexpReplacer.Replace(q.Wildcard), 0, 0, r, rv) + if err != nil { + return nil, err + } + } + } + return rv, nil +} + +// addFuzzySynonymsForTerm finds all terms that match the given term with the +// given fuzziness and adds their synonyms to the provided map. +func addSynonymsForTermWithMatchType(ctx context.Context, matchType int, src, field, term string, fuzziness, prefix int, + r index.ThesaurusReader, rv search.FieldTermSynonymMap, +) (search.FieldTermSynonymMap, error) { + // Determine the terms based on the match type (fuzzy, prefix, or regexp) + var thesKeys index.ThesaurusKeys + var err error + var terms []string + switch matchType { + case FuzzyMatchType: + // Ensure valid fuzziness + if fuzziness == 0 { + rv, err = addSynonymsForTerm(ctx, src, field, term, r, rv) + if err != nil { + return nil, err + } + return rv, nil + } + if fuzziness > searcher.MaxFuzziness { + return nil, fmt.Errorf("fuzziness exceeds max (%d)", searcher.MaxFuzziness) + } + if fuzziness < 0 { + return nil, fmt.Errorf("invalid fuzziness, negative") + } + // Handle fuzzy match + prefixTerm := "" + for i, r := range term { + if i < prefix { + prefixTerm += string(r) + } else { + break + } + } + thesKeys, err = r.ThesaurusKeysFuzzy(src, term, fuzziness, prefixTerm) + case RegexpMatchType: + // Handle regexp match + thesKeys, err = r.ThesaurusKeysRegexp(src, term) + case PrefixMatchType: + // Handle prefix match + thesKeys, err = r.ThesaurusKeysPrefix(src, []byte(term)) + default: + return nil, fmt.Errorf("invalid match type: %d", matchType) + } + if err != nil { + return nil, err + } + defer func() { + if cerr := thesKeys.Close(); cerr != nil && err == nil { + err = cerr + } + }() + // Collect the matching terms + terms = []string{} + tfd, err := thesKeys.Next() + for err == nil && tfd != nil { + terms = append(terms, tfd.Term) + tfd, err = thesKeys.Next() + } + if err != nil { + return nil, err + } + for _, synTerm := range terms { + rv, err = addSynonymsForTerm(ctx, src, field, synTerm, r, rv) + if err != nil { + return nil, err + } + } + return rv, nil +} + +func addSynonymsForTerm(ctx context.Context, src, field, term string, + r index.ThesaurusReader, rv search.FieldTermSynonymMap, +) (search.FieldTermSynonymMap, error) { + termReader, err := r.ThesaurusTermReader(ctx, src, []byte(term)) + if err != nil { + return nil, err + } + defer func() { + if cerr := termReader.Close(); cerr != nil && err == nil { + err = cerr + } + }() + var synonyms []string + synonym, err := termReader.Next() + for err == nil && synonym != "" { + synonyms = append(synonyms, synonym) + synonym, err = termReader.Next() + } + if err != nil { + return nil, err + } + if len(synonyms) > 0 { + if rv == nil { + rv = make(search.FieldTermSynonymMap) + } + if _, exists := rv[field]; !exists { + rv[field] = make(map[string][]string) + } + rv[field][term] = synonyms + } + return rv, nil +} diff --git a/search/query/query_string.go b/search/query/query_string.go new file mode 100644 index 0000000..42bb598 --- /dev/null +++ b/search/query/query_string.go @@ -0,0 +1,69 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +type QueryStringQuery struct { + Query string `json:"query"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewQueryStringQuery creates a new Query used for +// finding documents that satisfy a query string. The +// query string is a small query language for humans. +func NewQueryStringQuery(query string) *QueryStringQuery { + return &QueryStringQuery{ + Query: query, + } +} + +func (q *QueryStringQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *QueryStringQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *QueryStringQuery) Parse() (Query, error) { + return parseQuerySyntax(q.Query) +} + +func (q *QueryStringQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + newQuery, err := parseQuerySyntax(q.Query) + if err != nil { + return nil, err + } + return newQuery.Searcher(ctx, i, m, options) +} + +func (q *QueryStringQuery) Validate() error { + newQuery, err := parseQuerySyntax(q.Query) + if err != nil { + return err + } + if newQuery, ok := newQuery.(ValidatableQuery); ok { + return newQuery.Validate() + } + return nil +} diff --git a/search/query/query_string.y b/search/query/query_string.y new file mode 100644 index 0000000..aeec856 --- /dev/null +++ b/search/query/query_string.y @@ -0,0 +1,338 @@ +%{ +package query +import ( + "fmt" + "strconv" + "strings" + "time" +) + +func logDebugGrammar(format string, v ...interface{}) { + if debugParser { + logger.Printf(format, v...) + } +} +%} + +%union { +s string +n int +f float64 +q Query +pf *float64} + +%token tSTRING tPHRASE tPLUS tMINUS tCOLON tBOOST tNUMBER tSTRING tGREATER tLESS +tEQUAL tTILDE + +%type tSTRING +%type tPHRASE +%type tNUMBER +%type posOrNegNumber +%type fieldName +%type tTILDE +%type tBOOST +%type searchBase +%type searchSuffix +%type searchPrefix + +%% + +input: +searchParts { + logDebugGrammar("INPUT") +}; + +searchParts: +searchPart searchParts { + logDebugGrammar("SEARCH PARTS") +} +| +searchPart { + logDebugGrammar("SEARCH PART") +}; + +searchPart: +searchPrefix searchBase searchSuffix { + query := $2 + if $3 != nil { + if query, ok := query.(BoostableQuery); ok { + query.SetBoost(*$3) + } + } + switch($1) { + case queryShould: + yylex.(*lexerWrapper).query.AddShould(query) + case queryMust: + yylex.(*lexerWrapper).query.AddMust(query) + case queryMustNot: + yylex.(*lexerWrapper).query.AddMustNot(query) + } +}; + + +searchPrefix: +/* empty */ { + $$ = queryShould +} +| +tPLUS { + logDebugGrammar("PLUS") + $$ = queryMust +} +| +tMINUS { + logDebugGrammar("MINUS") + $$ = queryMustNot +}; + +searchBase: +tSTRING { + str := $1 + logDebugGrammar("STRING - %s", str) + var q FieldableQuery + if strings.HasPrefix(str, "/") && strings.HasSuffix(str, "/") { + q = NewRegexpQuery(str[1:len(str)-1]) + } else if strings.ContainsAny(str, "*?"){ + q = NewWildcardQuery(str) + } else { + q = NewMatchQuery(str) + } + $$ = q +} +| +tSTRING tTILDE { + str := $1 + fuzziness, err := strconv.ParseFloat($2, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid fuzziness value: %v", err)) + } + logDebugGrammar("FUZZY STRING - %s %f", str, fuzziness) + q := NewMatchQuery(str) + q.SetFuzziness(int(fuzziness)) + $$ = q +} +| +fieldName tCOLON tSTRING tTILDE { + field := $1 + str := $3 + fuzziness, err := strconv.ParseFloat($4, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid fuzziness value: %v", err)) + } + logDebugGrammar("FIELD - %s FUZZY STRING - %s %f", field, str, fuzziness) + q := NewMatchQuery(str) + q.SetFuzziness(int(fuzziness)) + q.SetField(field) + $$ = q +} +| +tNUMBER { + str := $1 + logDebugGrammar("STRING - %s", str) + q1 := NewMatchQuery(str) + val, err := strconv.ParseFloat($1, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + inclusive := true + q2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive) + q := NewDisjunctionQuery([]Query{q1,q2}) + q.queryStringMode = true + $$ = q +} +| +tPHRASE { + phrase := $1 + logDebugGrammar("PHRASE - %s", phrase) + q := NewMatchPhraseQuery(phrase) + $$ = q +} +| +fieldName tCOLON tSTRING { + field := $1 + str := $3 + logDebugGrammar("FIELD - %s STRING - %s", field, str) + var q FieldableQuery + if strings.HasPrefix(str, "/") && strings.HasSuffix(str, "/") { + q = NewRegexpQuery(str[1:len(str)-1]) + } else if strings.ContainsAny(str, "*?"){ + q = NewWildcardQuery(str) + } else { + q = NewMatchQuery(str) + } + q.SetField(field) + $$ = q +} +| +fieldName tCOLON posOrNegNumber { + field := $1 + str := $3 + logDebugGrammar("FIELD - %s STRING - %s", field, str) + q1 := NewMatchQuery(str) + q1.SetField(field) + val, err := strconv.ParseFloat($3, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + inclusive := true + q2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive) + q2.SetField(field) + q := NewDisjunctionQuery([]Query{q1,q2}) + q.queryStringMode = true + $$ = q +} +| +fieldName tCOLON tPHRASE { + field := $1 + phrase := $3 + logDebugGrammar("FIELD - %s PHRASE - %s", field, phrase) + q := NewMatchPhraseQuery(phrase) + q.SetField(field) + $$ = q +} +| +fieldName tCOLON tGREATER posOrNegNumber { + field := $1 + min, err := strconv.ParseFloat($4, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + minInclusive := false + logDebugGrammar("FIELD - GREATER THAN %f", min) + q := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil) + q.SetField(field) + $$ = q +} +| +fieldName tCOLON tGREATER tEQUAL posOrNegNumber { + field := $1 + min, err := strconv.ParseFloat($5, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + minInclusive := true + logDebugGrammar("FIELD - GREATER THAN OR EQUAL %f", min) + q := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil) + q.SetField(field) + $$ = q +} +| +fieldName tCOLON tLESS posOrNegNumber { + field := $1 + max, err := strconv.ParseFloat($4, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + maxInclusive := false + logDebugGrammar("FIELD - LESS THAN %f", max) + q := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive) + q.SetField(field) + $$ = q +} +| +fieldName tCOLON tLESS tEQUAL posOrNegNumber { + field := $1 + max, err := strconv.ParseFloat($5, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + maxInclusive := true + logDebugGrammar("FIELD - LESS THAN OR EQUAL %f", max) + q := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive) + q.SetField(field) + $$ = q +} +| +fieldName tCOLON tGREATER tPHRASE { + field := $1 + minInclusive := false + phrase := $4 + + logDebugGrammar("FIELD - GREATER THAN DATE %s", phrase) + minTime, err := queryTimeFromString(phrase) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) + } + q := NewDateRangeInclusiveQuery(minTime, time.Time{}, &minInclusive, nil) + q.SetField(field) + $$ = q +} +| +fieldName tCOLON tGREATER tEQUAL tPHRASE { + field := $1 + minInclusive := true + phrase := $5 + + logDebugGrammar("FIELD - GREATER THAN OR EQUAL DATE %s", phrase) + minTime, err := queryTimeFromString(phrase) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) + } + q := NewDateRangeInclusiveQuery(minTime, time.Time{}, &minInclusive, nil) + q.SetField(field) + $$ = q +} +| +fieldName tCOLON tLESS tPHRASE { + field := $1 + maxInclusive := false + phrase := $4 + + logDebugGrammar("FIELD - LESS THAN DATE %s", phrase) + maxTime, err := queryTimeFromString(phrase) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) + } + q := NewDateRangeInclusiveQuery(time.Time{}, maxTime, nil, &maxInclusive) + q.SetField(field) + $$ = q +} +| +fieldName tCOLON tLESS tEQUAL tPHRASE { + field := $1 + maxInclusive := true + phrase := $5 + + logDebugGrammar("FIELD - LESS THAN OR EQUAL DATE %s", phrase) + maxTime, err := queryTimeFromString(phrase) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) + } + q := NewDateRangeInclusiveQuery(time.Time{}, maxTime, nil, &maxInclusive) + q.SetField(field) + $$ = q +}; + +searchSuffix: +/* empty */ { + $$ = nil +} +| +tBOOST { + $$ = nil + boost, err := strconv.ParseFloat($1, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid boost value: %v", err)) + } else { + $$ = &boost + } + logDebugGrammar("BOOST %f", boost) +}; + +posOrNegNumber: +tNUMBER { + $$ = $1 +} +| +tMINUS tNUMBER { + $$ = "-" + $2 +}; + +fieldName: +tPHRASE { + $$ = $1 +} +| +tSTRING { + $$ = $1 +}; diff --git a/search/query/query_string.y.go b/search/query/query_string.y.go new file mode 100644 index 0000000..3a2abc1 --- /dev/null +++ b/search/query/query_string.y.go @@ -0,0 +1,833 @@ +// Code generated by goyacc -o query_string.y.go query_string.y. DO NOT EDIT. + +//line query_string.y:2 +package query + +import __yyfmt__ "fmt" + +//line query_string.y:2 +import ( + "fmt" + "strconv" + "strings" + "time" +) + +func logDebugGrammar(format string, v ...interface{}) { + if debugParser { + logger.Printf(format, v...) + } +} + +//line query_string.y:17 +type yySymType struct { + yys int + s string + n int + f float64 + q Query + pf *float64 +} + +const tSTRING = 57346 +const tPHRASE = 57347 +const tPLUS = 57348 +const tMINUS = 57349 +const tCOLON = 57350 +const tBOOST = 57351 +const tNUMBER = 57352 +const tGREATER = 57353 +const tLESS = 57354 +const tEQUAL = 57355 +const tTILDE = 57356 + +var yyToknames = [...]string{ + "$end", + "error", + "$unk", + "tSTRING", + "tPHRASE", + "tPLUS", + "tMINUS", + "tCOLON", + "tBOOST", + "tNUMBER", + "tGREATER", + "tLESS", + "tEQUAL", + "tTILDE", +} + +var yyStatenames = [...]string{} + +const yyEofCode = 1 +const yyErrCode = 2 +const yyInitialStackSize = 16 + +//line yacctab:1 +var yyExca = [...]int{ + -1, 1, + 1, -1, + -2, 0, + -1, 3, + 1, 3, + -2, 5, + -1, 9, + 8, 29, + -2, 8, + -1, 12, + 8, 28, + -2, 12, +} + +const yyPrivate = 57344 + +const yyLast = 43 + +var yyAct = [...]int{ + 18, 17, 19, 24, 23, 15, 31, 22, 20, 21, + 30, 27, 23, 23, 3, 22, 22, 14, 29, 26, + 16, 25, 28, 35, 33, 23, 23, 32, 22, 22, + 34, 9, 12, 1, 5, 6, 2, 11, 4, 13, + 7, 8, 10, +} + +var yyPact = [...]int{ + 28, -1000, -1000, 28, 27, -1000, -1000, -1000, 8, -9, + 12, -1000, -1000, -1000, -1000, -1000, -3, -11, -1000, -1000, + 6, 5, -1000, -4, -1000, -1000, 19, -1000, -1000, 18, + -1000, -1000, -1000, -1000, -1000, -1000, +} + +var yyPgo = [...]int{ + 0, 0, 42, 41, 39, 38, 33, 36, 14, +} + +var yyR1 = [...]int{ + 0, 6, 7, 7, 8, 5, 5, 5, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 4, 4, 1, 1, 2, 2, +} + +var yyR2 = [...]int{ + 0, 1, 2, 1, 3, 0, 1, 1, 1, 2, + 4, 1, 1, 3, 3, 3, 4, 5, 4, 5, + 4, 5, 4, 5, 0, 1, 1, 2, 1, 1, +} + +var yyChk = [...]int{ + -1000, -6, -7, -8, -5, 6, 7, -7, -3, 4, + -2, 10, 5, -4, 9, 14, 8, 4, -1, 5, + 11, 12, 10, 7, 14, -1, 13, 5, -1, 13, + 5, 10, -1, 5, -1, 5, +} + +var yyDef = [...]int{ + 5, -2, 1, -2, 0, 6, 7, 2, 24, -2, + 0, 11, -2, 4, 25, 9, 0, 13, 14, 15, + 0, 0, 26, 0, 10, 16, 0, 20, 18, 0, + 22, 27, 17, 21, 19, 23, +} + +var yyTok1 = [...]int{ + 1, +} + +var yyTok2 = [...]int{ + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, +} + +var yyTok3 = [...]int{ + 0, +} + +var yyErrorMessages = [...]struct { + state int + token int + msg string +}{} + +//line yaccpar:1 + +/* parser for yacc output */ + +var ( + yyDebug = 0 + yyErrorVerbose = false +) + +type yyLexer interface { + Lex(lval *yySymType) int + Error(s string) +} + +type yyParser interface { + Parse(yyLexer) int + Lookahead() int +} + +type yyParserImpl struct { + lval yySymType + stack [yyInitialStackSize]yySymType + char int +} + +func (p *yyParserImpl) Lookahead() int { + return p.char +} + +func yyNewParser() yyParser { + return &yyParserImpl{} +} + +const yyFlag = -1000 + +func yyTokname(c int) string { + if c >= 1 && c-1 < len(yyToknames) { + if yyToknames[c-1] != "" { + return yyToknames[c-1] + } + } + return __yyfmt__.Sprintf("tok-%v", c) +} + +func yyStatname(s int) string { + if s >= 0 && s < len(yyStatenames) { + if yyStatenames[s] != "" { + return yyStatenames[s] + } + } + return __yyfmt__.Sprintf("state-%v", s) +} + +func yyErrorMessage(state, lookAhead int) string { + const TOKSTART = 4 + + if !yyErrorVerbose { + return "syntax error" + } + + for _, e := range yyErrorMessages { + if e.state == state && e.token == lookAhead { + return "syntax error: " + e.msg + } + } + + res := "syntax error: unexpected " + yyTokname(lookAhead) + + // To match Bison, suggest at most four expected tokens. + expected := make([]int, 0, 4) + + // Look for shiftable tokens. + base := yyPact[state] + for tok := TOKSTART; tok-1 < len(yyToknames); tok++ { + if n := base + tok; n >= 0 && n < yyLast && yyChk[yyAct[n]] == tok { + if len(expected) == cap(expected) { + return res + } + expected = append(expected, tok) + } + } + + if yyDef[state] == -2 { + i := 0 + for yyExca[i] != -1 || yyExca[i+1] != state { + i += 2 + } + + // Look for tokens that we accept or reduce. + for i += 2; yyExca[i] >= 0; i += 2 { + tok := yyExca[i] + if tok < TOKSTART || yyExca[i+1] == 0 { + continue + } + if len(expected) == cap(expected) { + return res + } + expected = append(expected, tok) + } + + // If the default action is to accept or reduce, give up. + if yyExca[i+1] != 0 { + return res + } + } + + for i, tok := range expected { + if i == 0 { + res += ", expecting " + } else { + res += " or " + } + res += yyTokname(tok) + } + return res +} + +func yylex1(lex yyLexer, lval *yySymType) (char, token int) { + token = 0 + char = lex.Lex(lval) + if char <= 0 { + token = yyTok1[0] + goto out + } + if char < len(yyTok1) { + token = yyTok1[char] + goto out + } + if char >= yyPrivate { + if char < yyPrivate+len(yyTok2) { + token = yyTok2[char-yyPrivate] + goto out + } + } + for i := 0; i < len(yyTok3); i += 2 { + token = yyTok3[i+0] + if token == char { + token = yyTok3[i+1] + goto out + } + } + +out: + if token == 0 { + token = yyTok2[1] /* unknown char */ + } + if yyDebug >= 3 { + __yyfmt__.Printf("lex %s(%d)\n", yyTokname(token), uint(char)) + } + return char, token +} + +func yyParse(yylex yyLexer) int { + return yyNewParser().Parse(yylex) +} + +func (yyrcvr *yyParserImpl) Parse(yylex yyLexer) int { + var yyn int + var yyVAL yySymType + var yyDollar []yySymType + _ = yyDollar // silence set and not used + yyS := yyrcvr.stack[:] + + Nerrs := 0 /* number of errors */ + Errflag := 0 /* error recovery flag */ + yystate := 0 + yyrcvr.char = -1 + yytoken := -1 // yyrcvr.char translated into internal numbering + defer func() { + // Make sure we report no lookahead when not parsing. + yystate = -1 + yyrcvr.char = -1 + yytoken = -1 + }() + yyp := -1 + goto yystack + +ret0: + return 0 + +ret1: + return 1 + +yystack: + /* put a state and value onto the stack */ + if yyDebug >= 4 { + __yyfmt__.Printf("char %v in %v\n", yyTokname(yytoken), yyStatname(yystate)) + } + + yyp++ + if yyp >= len(yyS) { + nyys := make([]yySymType, len(yyS)*2) + copy(nyys, yyS) + yyS = nyys + } + yyS[yyp] = yyVAL + yyS[yyp].yys = yystate + +yynewstate: + yyn = yyPact[yystate] + if yyn <= yyFlag { + goto yydefault /* simple state */ + } + if yyrcvr.char < 0 { + yyrcvr.char, yytoken = yylex1(yylex, &yyrcvr.lval) + } + yyn += yytoken + if yyn < 0 || yyn >= yyLast { + goto yydefault + } + yyn = yyAct[yyn] + if yyChk[yyn] == yytoken { /* valid shift */ + yyrcvr.char = -1 + yytoken = -1 + yyVAL = yyrcvr.lval + yystate = yyn + if Errflag > 0 { + Errflag-- + } + goto yystack + } + +yydefault: + /* default state action */ + yyn = yyDef[yystate] + if yyn == -2 { + if yyrcvr.char < 0 { + yyrcvr.char, yytoken = yylex1(yylex, &yyrcvr.lval) + } + + /* look through exception table */ + xi := 0 + for { + if yyExca[xi+0] == -1 && yyExca[xi+1] == yystate { + break + } + xi += 2 + } + for xi += 2; ; xi += 2 { + yyn = yyExca[xi+0] + if yyn < 0 || yyn == yytoken { + break + } + } + yyn = yyExca[xi+1] + if yyn < 0 { + goto ret0 + } + } + if yyn == 0 { + /* error ... attempt to resume parsing */ + switch Errflag { + case 0: /* brand new error */ + yylex.Error(yyErrorMessage(yystate, yytoken)) + Nerrs++ + if yyDebug >= 1 { + __yyfmt__.Printf("%s", yyStatname(yystate)) + __yyfmt__.Printf(" saw %s\n", yyTokname(yytoken)) + } + fallthrough + + case 1, 2: /* incompletely recovered error ... try again */ + Errflag = 3 + + /* find a state where "error" is a legal shift action */ + for yyp >= 0 { + yyn = yyPact[yyS[yyp].yys] + yyErrCode + if yyn >= 0 && yyn < yyLast { + yystate = yyAct[yyn] /* simulate a shift of "error" */ + if yyChk[yystate] == yyErrCode { + goto yystack + } + } + + /* the current p has no shift on "error", pop stack */ + if yyDebug >= 2 { + __yyfmt__.Printf("error recovery pops state %d\n", yyS[yyp].yys) + } + yyp-- + } + /* there is no state on the stack with an error shift ... abort */ + goto ret1 + + case 3: /* no shift yet; clobber input char */ + if yyDebug >= 2 { + __yyfmt__.Printf("error recovery discards %s\n", yyTokname(yytoken)) + } + if yytoken == yyEofCode { + goto ret1 + } + yyrcvr.char = -1 + yytoken = -1 + goto yynewstate /* try again in the same state */ + } + } + + /* reduction by production yyn */ + if yyDebug >= 2 { + __yyfmt__.Printf("reduce %v in:\n\t%v\n", yyn, yyStatname(yystate)) + } + + yynt := yyn + yypt := yyp + _ = yypt // guard against "declared and not used" + + yyp -= yyR2[yyn] + // yyp is now the index of $0. Perform the default action. Iff the + // reduced production is ε, $1 is possibly out of range. + if yyp+1 >= len(yyS) { + nyys := make([]yySymType, len(yyS)*2) + copy(nyys, yyS) + yyS = nyys + } + yyVAL = yyS[yyp+1] + + /* consult goto table to find next state */ + yyn = yyR1[yyn] + yyg := yyPgo[yyn] + yyj := yyg + yyS[yyp].yys + 1 + + if yyj >= yyLast { + yystate = yyAct[yyg] + } else { + yystate = yyAct[yyj] + if yyChk[yystate] != -yyn { + yystate = yyAct[yyg] + } + } + // dummy call; replaced with literal code + switch yynt { + + case 1: + yyDollar = yyS[yypt-1 : yypt+1] +//line query_string.y:41 + { + logDebugGrammar("INPUT") + } + case 2: + yyDollar = yyS[yypt-2 : yypt+1] +//line query_string.y:46 + { + logDebugGrammar("SEARCH PARTS") + } + case 3: + yyDollar = yyS[yypt-1 : yypt+1] +//line query_string.y:50 + { + logDebugGrammar("SEARCH PART") + } + case 4: + yyDollar = yyS[yypt-3 : yypt+1] +//line query_string.y:55 + { + query := yyDollar[2].q + if yyDollar[3].pf != nil { + if query, ok := query.(BoostableQuery); ok { + query.SetBoost(*yyDollar[3].pf) + } + } + switch yyDollar[1].n { + case queryShould: + yylex.(*lexerWrapper).query.AddShould(query) + case queryMust: + yylex.(*lexerWrapper).query.AddMust(query) + case queryMustNot: + yylex.(*lexerWrapper).query.AddMustNot(query) + } + } + case 5: + yyDollar = yyS[yypt-0 : yypt+1] +//line query_string.y:74 + { + yyVAL.n = queryShould + } + case 6: + yyDollar = yyS[yypt-1 : yypt+1] +//line query_string.y:78 + { + logDebugGrammar("PLUS") + yyVAL.n = queryMust + } + case 7: + yyDollar = yyS[yypt-1 : yypt+1] +//line query_string.y:83 + { + logDebugGrammar("MINUS") + yyVAL.n = queryMustNot + } + case 8: + yyDollar = yyS[yypt-1 : yypt+1] +//line query_string.y:89 + { + str := yyDollar[1].s + logDebugGrammar("STRING - %s", str) + var q FieldableQuery + if strings.HasPrefix(str, "/") && strings.HasSuffix(str, "/") { + q = NewRegexpQuery(str[1 : len(str)-1]) + } else if strings.ContainsAny(str, "*?") { + q = NewWildcardQuery(str) + } else { + q = NewMatchQuery(str) + } + yyVAL.q = q + } + case 9: + yyDollar = yyS[yypt-2 : yypt+1] +//line query_string.y:103 + { + str := yyDollar[1].s + fuzziness, err := strconv.ParseFloat(yyDollar[2].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid fuzziness value: %v", err)) + } + logDebugGrammar("FUZZY STRING - %s %f", str, fuzziness) + q := NewMatchQuery(str) + q.SetFuzziness(int(fuzziness)) + yyVAL.q = q + } + case 10: + yyDollar = yyS[yypt-4 : yypt+1] +//line query_string.y:115 + { + field := yyDollar[1].s + str := yyDollar[3].s + fuzziness, err := strconv.ParseFloat(yyDollar[4].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid fuzziness value: %v", err)) + } + logDebugGrammar("FIELD - %s FUZZY STRING - %s %f", field, str, fuzziness) + q := NewMatchQuery(str) + q.SetFuzziness(int(fuzziness)) + q.SetField(field) + yyVAL.q = q + } + case 11: + yyDollar = yyS[yypt-1 : yypt+1] +//line query_string.y:129 + { + str := yyDollar[1].s + logDebugGrammar("STRING - %s", str) + q1 := NewMatchQuery(str) + val, err := strconv.ParseFloat(yyDollar[1].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + inclusive := true + q2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive) + q := NewDisjunctionQuery([]Query{q1, q2}) + q.queryStringMode = true + yyVAL.q = q + } + case 12: + yyDollar = yyS[yypt-1 : yypt+1] +//line query_string.y:144 + { + phrase := yyDollar[1].s + logDebugGrammar("PHRASE - %s", phrase) + q := NewMatchPhraseQuery(phrase) + yyVAL.q = q + } + case 13: + yyDollar = yyS[yypt-3 : yypt+1] +//line query_string.y:151 + { + field := yyDollar[1].s + str := yyDollar[3].s + logDebugGrammar("FIELD - %s STRING - %s", field, str) + var q FieldableQuery + if strings.HasPrefix(str, "/") && strings.HasSuffix(str, "/") { + q = NewRegexpQuery(str[1 : len(str)-1]) + } else if strings.ContainsAny(str, "*?") { + q = NewWildcardQuery(str) + } else { + q = NewMatchQuery(str) + } + q.SetField(field) + yyVAL.q = q + } + case 14: + yyDollar = yyS[yypt-3 : yypt+1] +//line query_string.y:167 + { + field := yyDollar[1].s + str := yyDollar[3].s + logDebugGrammar("FIELD - %s STRING - %s", field, str) + q1 := NewMatchQuery(str) + q1.SetField(field) + val, err := strconv.ParseFloat(yyDollar[3].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + inclusive := true + q2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive) + q2.SetField(field) + q := NewDisjunctionQuery([]Query{q1, q2}) + q.queryStringMode = true + yyVAL.q = q + } + case 15: + yyDollar = yyS[yypt-3 : yypt+1] +//line query_string.y:185 + { + field := yyDollar[1].s + phrase := yyDollar[3].s + logDebugGrammar("FIELD - %s PHRASE - %s", field, phrase) + q := NewMatchPhraseQuery(phrase) + q.SetField(field) + yyVAL.q = q + } + case 16: + yyDollar = yyS[yypt-4 : yypt+1] +//line query_string.y:194 + { + field := yyDollar[1].s + min, err := strconv.ParseFloat(yyDollar[4].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + minInclusive := false + logDebugGrammar("FIELD - GREATER THAN %f", min) + q := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil) + q.SetField(field) + yyVAL.q = q + } + case 17: + yyDollar = yyS[yypt-5 : yypt+1] +//line query_string.y:207 + { + field := yyDollar[1].s + min, err := strconv.ParseFloat(yyDollar[5].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + minInclusive := true + logDebugGrammar("FIELD - GREATER THAN OR EQUAL %f", min) + q := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil) + q.SetField(field) + yyVAL.q = q + } + case 18: + yyDollar = yyS[yypt-4 : yypt+1] +//line query_string.y:220 + { + field := yyDollar[1].s + max, err := strconv.ParseFloat(yyDollar[4].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + maxInclusive := false + logDebugGrammar("FIELD - LESS THAN %f", max) + q := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive) + q.SetField(field) + yyVAL.q = q + } + case 19: + yyDollar = yyS[yypt-5 : yypt+1] +//line query_string.y:233 + { + field := yyDollar[1].s + max, err := strconv.ParseFloat(yyDollar[5].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + maxInclusive := true + logDebugGrammar("FIELD - LESS THAN OR EQUAL %f", max) + q := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive) + q.SetField(field) + yyVAL.q = q + } + case 20: + yyDollar = yyS[yypt-4 : yypt+1] +//line query_string.y:246 + { + field := yyDollar[1].s + minInclusive := false + phrase := yyDollar[4].s + + logDebugGrammar("FIELD - GREATER THAN DATE %s", phrase) + minTime, err := queryTimeFromString(phrase) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) + } + q := NewDateRangeInclusiveQuery(minTime, time.Time{}, &minInclusive, nil) + q.SetField(field) + yyVAL.q = q + } + case 21: + yyDollar = yyS[yypt-5 : yypt+1] +//line query_string.y:261 + { + field := yyDollar[1].s + minInclusive := true + phrase := yyDollar[5].s + + logDebugGrammar("FIELD - GREATER THAN OR EQUAL DATE %s", phrase) + minTime, err := queryTimeFromString(phrase) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) + } + q := NewDateRangeInclusiveQuery(minTime, time.Time{}, &minInclusive, nil) + q.SetField(field) + yyVAL.q = q + } + case 22: + yyDollar = yyS[yypt-4 : yypt+1] +//line query_string.y:276 + { + field := yyDollar[1].s + maxInclusive := false + phrase := yyDollar[4].s + + logDebugGrammar("FIELD - LESS THAN DATE %s", phrase) + maxTime, err := queryTimeFromString(phrase) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) + } + q := NewDateRangeInclusiveQuery(time.Time{}, maxTime, nil, &maxInclusive) + q.SetField(field) + yyVAL.q = q + } + case 23: + yyDollar = yyS[yypt-5 : yypt+1] +//line query_string.y:291 + { + field := yyDollar[1].s + maxInclusive := true + phrase := yyDollar[5].s + + logDebugGrammar("FIELD - LESS THAN OR EQUAL DATE %s", phrase) + maxTime, err := queryTimeFromString(phrase) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) + } + q := NewDateRangeInclusiveQuery(time.Time{}, maxTime, nil, &maxInclusive) + q.SetField(field) + yyVAL.q = q + } + case 24: + yyDollar = yyS[yypt-0 : yypt+1] +//line query_string.y:307 + { + yyVAL.pf = nil + } + case 25: + yyDollar = yyS[yypt-1 : yypt+1] +//line query_string.y:311 + { + yyVAL.pf = nil + boost, err := strconv.ParseFloat(yyDollar[1].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid boost value: %v", err)) + } else { + yyVAL.pf = &boost + } + logDebugGrammar("BOOST %f", boost) + } + case 26: + yyDollar = yyS[yypt-1 : yypt+1] +//line query_string.y:323 + { + yyVAL.s = yyDollar[1].s + } + case 27: + yyDollar = yyS[yypt-2 : yypt+1] +//line query_string.y:327 + { + yyVAL.s = "-" + yyDollar[2].s + } + case 28: + yyDollar = yyS[yypt-1 : yypt+1] +//line query_string.y:332 + { + yyVAL.s = yyDollar[1].s + } + case 29: + yyDollar = yyS[yypt-1 : yypt+1] +//line query_string.y:336 + { + yyVAL.s = yyDollar[1].s + } + } + goto yystack /* stack new state and value */ +} diff --git a/search/query/query_string_lex.go b/search/query/query_string_lex.go new file mode 100644 index 0000000..b7a1283 --- /dev/null +++ b/search/query/query_string_lex.go @@ -0,0 +1,352 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "bufio" + "io" + "strings" + "sync" + "unicode" +) + +var queryStringLexPool = sync.Pool{ + New: func() interface{} { + return &queryStringLex{ + in: bufio.NewReader(strings.NewReader("")), + } + }, +} + +func getQueryStringLex(in io.Reader) *queryStringLex { + l := queryStringLexPool.Get().(*queryStringLex) + l.in.Reset(in) + l.currState = startState + l.currConsumed = true + l.buf = "" + l.inEscape = false + l.nextToken = nil + l.nextTokenType = 0 + l.seenDot = false + l.nextRune = 0 + l.nextRuneSize = 0 + l.atEOF = false + return l +} + +func putQueryStringLex(l *queryStringLex) { + l.in.Reset(strings.NewReader("")) + l.nextToken = nil + queryStringLexPool.Put(l) +} + +const reservedChars = "+-=&|>', '<', '=': + l.buf += string(next) + return singleCharOpState, true + case '^': + return inBoostState, true + case '~': + return inTildeState, true + } + + switch { + case !l.inEscape && next == '\\': + l.inEscape = true + return startState, true + case unicode.IsDigit(next): + l.buf += string(next) + return inNumOrStrState, true + case !unicode.IsSpace(next): + l.buf += string(next) + return inStrState, true + } + + // doesn't look like anything, just eat it and stay here + l.reset() + return startState, true +} + +func inPhraseState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + // unterminated phrase eats the phrase + if eof { + l.Error("unterminated quote") + return nil, false + } + + // only a non-escaped " ends the phrase + if !l.inEscape && next == '"' { + // end phrase + l.nextTokenType = tPHRASE + l.nextToken = &yySymType{ + s: l.buf, + } + logDebugTokens("PHRASE - '%s'", l.nextToken.s) + l.reset() + return startState, true + } else if !l.inEscape && next == '\\' { + l.inEscape = true + } else if l.inEscape { + // if in escape, end it + l.inEscape = false + l.buf += unescape(string(next)) + } else { + l.buf += string(next) + } + + return inPhraseState, true +} + +func singleCharOpState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + l.nextToken = &yySymType{} + + switch l.buf { + case "+": + l.nextTokenType = tPLUS + logDebugTokens("PLUS") + case "-": + l.nextTokenType = tMINUS + logDebugTokens("MINUS") + case ":": + l.nextTokenType = tCOLON + logDebugTokens("COLON") + case ">": + l.nextTokenType = tGREATER + logDebugTokens("GREATER") + case "<": + l.nextTokenType = tLESS + logDebugTokens("LESS") + case "=": + l.nextTokenType = tEQUAL + logDebugTokens("EQUAL") + } + + l.reset() + return startState, false +} + +func inBoostState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + + // only a non-escaped space ends the boost (or eof) + if eof || (!l.inEscape && next == ' ') { + // end boost + l.nextTokenType = tBOOST + if l.buf == "" { + l.buf = "1" + } + l.nextToken = &yySymType{ + s: l.buf, + } + logDebugTokens("BOOST - '%s'", l.nextToken.s) + l.reset() + return startState, true + } else if !l.inEscape && next == '\\' { + l.inEscape = true + } else if l.inEscape { + // if in escape, end it + l.inEscape = false + l.buf += unescape(string(next)) + } else { + l.buf += string(next) + } + + return inBoostState, true +} + +func inTildeState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + + // only a non-escaped space ends the tilde (or eof) + if eof || (!l.inEscape && next == ' ') { + // end tilde + l.nextTokenType = tTILDE + if l.buf == "" { + l.buf = "1" + } + l.nextToken = &yySymType{ + s: l.buf, + } + logDebugTokens("TILDE - '%s'", l.nextToken.s) + l.reset() + return startState, true + } else if !l.inEscape && next == '\\' { + l.inEscape = true + } else if l.inEscape { + // if in escape, end it + l.inEscape = false + l.buf += unescape(string(next)) + } else { + l.buf += string(next) + } + + return inTildeState, true +} + +func inNumOrStrState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + // end on non-escaped space, colon, tilde, boost (or eof) + if eof || (!l.inEscape && (next == ' ' || next == ':' || next == '^' || next == '~')) { + // end number + l.nextTokenType = tNUMBER + l.nextToken = &yySymType{ + s: l.buf, + } + logDebugTokens("NUMBER - '%s'", l.nextToken.s) + l.reset() + + consumed := true + if !eof && (next == ':' || next == '^' || next == '~') { + consumed = false + } + + return startState, consumed + } else if !l.inEscape && next == '\\' { + l.inEscape = true + return inNumOrStrState, true + } else if l.inEscape { + // if in escape, end it + l.inEscape = false + l.buf += unescape(string(next)) + // go directly to string, no successfully or unsuccessfully + // escaped string results in a valid number + return inStrState, true + } + + // see where to go + if !l.seenDot && next == '.' { + // stay in this state + l.seenDot = true + l.buf += string(next) + return inNumOrStrState, true + } else if unicode.IsDigit(next) { + l.buf += string(next) + return inNumOrStrState, true + } + + // doesn't look like an number, transition + l.buf += string(next) + return inStrState, true +} + +func inStrState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + // end on non-escaped space, colon, tilde, boost (or eof) + if eof || (!l.inEscape && (next == ' ' || next == ':' || next == '^' || next == '~')) { + // end string + l.nextTokenType = tSTRING + l.nextToken = &yySymType{ + s: l.buf, + } + logDebugTokens("STRING - '%s'", l.nextToken.s) + l.reset() + + consumed := true + if !eof && (next == ':' || next == '^' || next == '~') { + consumed = false + } + + return startState, consumed + } else if !l.inEscape && next == '\\' { + l.inEscape = true + } else if l.inEscape { + // if in escape, end it + l.inEscape = false + l.buf += unescape(string(next)) + } else { + l.buf += string(next) + } + + return inStrState, true +} + +func logDebugTokens(format string, v ...interface{}) { + if debugLexer { + logger.Printf(format, v...) + } +} diff --git a/search/query/query_string_lex_test.go b/search/query/query_string_lex_test.go new file mode 100644 index 0000000..34c359d --- /dev/null +++ b/search/query/query_string_lex_test.go @@ -0,0 +1,1232 @@ +package query + +import ( + "reflect" + "strings" + "testing" +) + +func TestLexer(t *testing.T) { + + tests := []struct { + input string + tokens []token + }{ + { + input: "test", + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "test", + }, + }, + }, + }, + { + input: "127.0.0.1", + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "127.0.0.1", + }, + }, + }, + }, + { + input: `"test phrase 1"`, + tokens: []token{ + { + typ: tPHRASE, + lval: yySymType{ + s: "test phrase 1", + }, + }, + }, + }, + { + input: "field:test", + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "field", + }, + }, + { + typ: tCOLON, + }, + { + typ: tSTRING, + lval: yySymType{ + s: "test", + }, + }, + }, + }, + { + input: "field:t-est", + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "field", + }, + }, + { + typ: tCOLON, + }, + { + typ: tSTRING, + lval: yySymType{ + s: "t-est", + }, + }, + }, + }, + { + input: "field:t+est", + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "field", + }, + }, + { + typ: tCOLON, + }, + { + typ: tSTRING, + lval: yySymType{ + s: "t+est", + }, + }, + }, + }, + { + input: "field:t>est", + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "field", + }, + }, + { + typ: tCOLON, + }, + { + typ: tSTRING, + lval: yySymType{ + s: "t>est", + }, + }, + }, + }, + { + input: "field:t5`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "field", + }, + }, + { + typ: tCOLON, + }, + { + typ: tGREATER, + }, + { + typ: tNUMBER, + lval: yySymType{ + s: "5", + }, + }, + }, + }, + { + input: `field:>=5`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "field", + }, + }, + { + typ: tCOLON, + }, + { + typ: tGREATER, + }, + { + typ: tEQUAL, + }, + { + typ: tNUMBER, + lval: yySymType{ + s: "5", + }, + }, + }, + }, + { + input: `field:<5`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "field", + }, + }, + { + typ: tCOLON, + }, + { + typ: tLESS, + }, + { + typ: tNUMBER, + lval: yySymType{ + s: "5", + }, + }, + }, + }, + { + input: `field:<=5`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "field", + }, + }, + { + typ: tCOLON, + }, + { + typ: tLESS, + }, + { + typ: tEQUAL, + }, + { + typ: tNUMBER, + lval: yySymType{ + s: "5", + }, + }, + }, + }, + { + input: "field:-5", + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "field", + }, + }, + { + typ: tCOLON, + }, + { + typ: tMINUS, + }, + { + typ: tNUMBER, + lval: yySymType{ + s: "5", + }, + }, + }, + }, + { + input: `field:>-5`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "field", + }, + }, + { + typ: tCOLON, + }, + { + typ: tGREATER, + }, + { + typ: tMINUS, + }, + { + typ: tNUMBER, + lval: yySymType{ + s: "5", + }, + }, + }, + }, + { + input: `field:>=-5`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "field", + }, + }, + { + typ: tCOLON, + }, + { + typ: tGREATER, + }, + { + typ: tEQUAL, + }, + { + typ: tMINUS, + }, + { + typ: tNUMBER, + lval: yySymType{ + s: "5", + }, + }, + }, + }, + { + input: `field:<-5`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "field", + }, + }, + { + typ: tCOLON, + }, + { + typ: tLESS, + }, + { + typ: tMINUS, + }, + { + typ: tNUMBER, + lval: yySymType{ + s: "5", + }, + }, + }, + }, + { + input: `field:<=-5`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "field", + }, + }, + { + typ: tCOLON, + }, + { + typ: tLESS, + }, + { + typ: tEQUAL, + }, + { + typ: tMINUS, + }, + { + typ: tNUMBER, + lval: yySymType{ + s: "5", + }, + }, + }, + }, + { + input: `field:>"2006-01-02T15:04:05Z"`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "field", + }, + }, + { + typ: tCOLON, + }, + { + typ: tGREATER, + }, + { + typ: tPHRASE, + lval: yySymType{ + s: "2006-01-02T15:04:05Z", + }, + }, + }, + }, + { + input: `field:>="2006-01-02T15:04:05Z"`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "field", + }, + }, + { + typ: tCOLON, + }, + { + typ: tGREATER, + }, + { + typ: tEQUAL, + }, + { + typ: tPHRASE, + lval: yySymType{ + s: "2006-01-02T15:04:05Z", + }, + }, + }, + }, + { + input: `field:<"2006-01-02T15:04:05Z"`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "field", + }, + }, + { + typ: tCOLON, + }, + { + typ: tLESS, + }, + { + typ: tPHRASE, + lval: yySymType{ + s: "2006-01-02T15:04:05Z", + }, + }, + }, + }, + { + input: `field:<="2006-01-02T15:04:05Z"`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "field", + }, + }, + { + typ: tCOLON, + }, + { + typ: tLESS, + }, + { + typ: tEQUAL, + }, + { + typ: tPHRASE, + lval: yySymType{ + s: "2006-01-02T15:04:05Z", + }, + }, + }, + }, + { + input: `/mar.*ty/`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: `/mar.*ty/`, + }, + }, + }, + }, + { + input: `name:/mar.*ty/`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "name", + }, + }, + { + typ: tCOLON, + }, + { + typ: tSTRING, + lval: yySymType{ + s: `/mar.*ty/`, + }, + }, + }, + }, + { + input: `mart*`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: `mart*`, + }, + }, + }, + }, + { + input: `name:mart*`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "name", + }, + }, + { + typ: tCOLON, + }, + { + typ: tSTRING, + lval: yySymType{ + s: `mart*`, + }, + }, + }, + }, + { + input: `name\:marty`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: `name:marty`, + }, + }, + }, + }, + { + input: `name:marty\:couchbase`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "name", + }, + }, + { + typ: tCOLON, + }, + { + typ: tSTRING, + lval: yySymType{ + s: `marty:couchbase`, + }, + }, + }, + }, + { + input: `marty\ couchbase`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: `marty couchbase`, + }, + }, + }, + }, + { + input: `\+marty`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: `+marty`, + }, + }, + }, + }, + { + input: `\-marty`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: `-marty`, + }, + }, + }, + }, + { + input: `"what does \"quote\" mean"`, + tokens: []token{ + { + typ: tPHRASE, + lval: yySymType{ + s: `what does "quote" mean`, + }, + }, + }, + }, + { + input: `can\ i\ escap\e`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: `can i escap\e`, + }, + }, + }, + }, + { + input: ` what`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: `what`, + }, + }, + }, + }, + { + input: `term^`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: `term`, + }, + }, + { + typ: tBOOST, + lval: yySymType{ + s: "1", + }, + }, + }, + }, + { + input: `3.0\:`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: `3.0:`, + }, + }, + }, + }, + { + input: `3.0\a`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: `3.0\a`, + }, + }, + }, + }, + { + input: `age:65^10`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "age", + }, + }, + { + typ: tCOLON, + }, + { + typ: tNUMBER, + lval: yySymType{ + s: "65", + }, + }, + { + typ: tBOOST, + lval: yySymType{ + s: "10", + }, + }, + }, + }, + { + input: `age:65^10 age:18^5`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "age", + }, + }, + { + typ: tCOLON, + }, + { + typ: tNUMBER, + lval: yySymType{ + s: "65", + }, + }, + { + typ: tBOOST, + lval: yySymType{ + s: "10", + }, + }, + { + typ: tSTRING, + lval: yySymType{ + s: "age", + }, + }, + { + typ: tCOLON, + }, + { + typ: tNUMBER, + lval: yySymType{ + s: "18", + }, + }, + { + typ: tBOOST, + lval: yySymType{ + s: "5", + }, + }, + }, + }, + { + input: `age:65~2`, + tokens: []token{ + { + typ: tSTRING, + lval: yySymType{ + s: "age", + }, + }, + { + typ: tCOLON, + }, + { + typ: tNUMBER, + lval: yySymType{ + s: "65", + }, + }, + { + typ: tTILDE, + lval: yySymType{ + s: "2", + }, + }, + }, + }, + { + input: `65:cat`, + tokens: []token{ + { + typ: tNUMBER, + lval: yySymType{ + s: "65", + }, + }, + { + typ: tCOLON, + }, + { + typ: tSTRING, + lval: yySymType{ + s: "cat", + }, + }, + }, + }, + } + + for _, test := range tests { + test := test + t.Run(test.input, func(t *testing.T) { + + r := strings.NewReader(test.input) + l := getQueryStringLex(r) + defer putQueryStringLex(l) + + var tokens []token + var lval yySymType + rv := l.Lex(&lval) + for rv > 0 { + //tokenTypes = append(tokenTypes, rv) + tokens = append(tokens, token{typ: rv, lval: lval}) + lval.s = "" + lval.n = 0 + rv = l.Lex(&lval) + } + + if !reflect.DeepEqual(tokens, test.tokens) { + t.Fatalf("\nexpected: %#v\n got: %#v\n", test.tokens, tokens) + } + }) + } +} + +type token struct { + typ int + lval yySymType +} diff --git a/search/query/query_string_parser.go b/search/query/query_string_parser.go new file mode 100644 index 0000000..d414c46 --- /dev/null +++ b/search/query/query_string_parser.go @@ -0,0 +1,87 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// as of Go 1.8 this requires the goyacc external tool +// available from golang.org/x/tools/cmd/goyacc + +//go:generate goyacc -o query_string.y.go query_string.y +//go:generate sed -i.tmp -e 1d query_string.y.go +//go:generate rm query_string.y.go.tmp + +// note: OSX sed and gnu sed handle the -i (in-place) option differently. +// using -i.tmp works on both, at the expense of having to remove +// the unsightly .tmp files + +package query + +import ( + "fmt" + "strings" +) + +var debugParser bool +var debugLexer bool + +func parseQuerySyntax(query string) (rq Query, err error) { + if query == "" { + return NewMatchNoneQuery(), nil + } + qsl := getQueryStringLex(strings.NewReader(query)) + defer putQueryStringLex(qsl) + lex := newLexerWrapper(qsl) + doParse(lex) + + if len(lex.errs) > 0 { + return nil, fmt.Errorf("%s", strings.Join(lex.errs, "\n")) + } + return lex.query, nil +} + +func doParse(lex *lexerWrapper) { + defer func() { + r := recover() + if r != nil { + lex.errs = append(lex.errs, fmt.Sprintf("parse error: %v", r)) + } + }() + + yyParse(lex) +} + +const ( + queryShould = iota + queryMust + queryMustNot +) + +type lexerWrapper struct { + lex yyLexer + errs []string + query *BooleanQuery +} + +func newLexerWrapper(lex yyLexer) *lexerWrapper { + return &lexerWrapper{ + lex: lex, + query: NewBooleanQueryForQueryString(nil, nil, nil), + } +} + +func (l *lexerWrapper) Lex(lval *yySymType) int { + return l.lex.Lex(lval) +} + +func (l *lexerWrapper) Error(s string) { + l.errs = append(l.errs, s) +} diff --git a/search/query/query_string_parser_test.go b/search/query/query_string_parser_test.go new file mode 100644 index 0000000..1eae236 --- /dev/null +++ b/search/query/query_string_parser_test.go @@ -0,0 +1,984 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "reflect" + "strings" + "testing" + "time" + + "github.com/blevesearch/bleve/v2/mapping" +) + +func TestQuerySyntaxParserValid(t *testing.T) { + thirtyThreePointOh := 33.0 + twoPointOh := 2.0 + fivePointOh := 5.0 + minusFivePointOh := -5.0 + theTruth := true + theFalsehood := false + theDate, err := time.Parse(time.RFC3339, "2006-01-02T15:04:05Z") + if err != nil { + t.Fatal(err) + } + tests := []struct { + input string + result Query + mapping mapping.IndexMapping + }{ + { + input: "test", + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + NewMatchQuery("test"), + }, + nil), + }, + { + input: "127.0.0.1", + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + NewMatchQuery("127.0.0.1"), + }, + nil), + }, + { + input: `"test phrase 1"`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + NewMatchPhraseQuery("test phrase 1"), + }, + nil), + }, + { + input: "field:test", + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewMatchQuery("test") + q.SetField("field") + return q + }(), + }, + nil), + }, + // - is allowed inside a term, just not the start + { + input: "field:t-est", + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewMatchQuery("t-est") + q.SetField("field") + return q + }(), + }, + nil), + }, + // + is allowed inside a term, just not the start + { + input: "field:t+est", + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewMatchQuery("t+est") + q.SetField("field") + return q + }(), + }, + nil), + }, + // > is allowed inside a term, just not the start + { + input: "field:t>est", + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewMatchQuery("t>est") + q.SetField("field") + return q + }(), + }, + nil), + }, + // < is allowed inside a term, just not the start + { + input: "field:t5`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewNumericRangeInclusiveQuery(&fivePointOh, nil, &theFalsehood, nil) + q.SetField("field") + return q + }(), + }, + nil), + }, + { + input: `field:>=5`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewNumericRangeInclusiveQuery(&fivePointOh, nil, &theTruth, nil) + q.SetField("field") + return q + }(), + }, + nil), + }, + { + input: `field:<5`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewNumericRangeInclusiveQuery(nil, &fivePointOh, nil, &theFalsehood) + q.SetField("field") + return q + }(), + }, + nil), + }, + { + input: `field:<=5`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewNumericRangeInclusiveQuery(nil, &fivePointOh, nil, &theTruth) + q.SetField("field") + return q + }(), + }, + nil), + }, + // new range tests with negative number + { + input: "field:-5", + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + qo := NewDisjunctionQuery( + []Query{ + func() Query { + q := NewMatchQuery("-5") + q.SetField("field") + return q + }(), + func() Query { + q := NewNumericRangeInclusiveQuery(&minusFivePointOh, &minusFivePointOh, &theTruth, &theTruth) + q.SetField("field") + return q + }(), + }) + qo.queryStringMode = true + return qo + }(), + }, + nil), + }, + { + input: `field:>-5`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewNumericRangeInclusiveQuery(&minusFivePointOh, nil, &theFalsehood, nil) + q.SetField("field") + return q + }(), + }, + nil), + }, + { + input: `field:>=-5`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewNumericRangeInclusiveQuery(&minusFivePointOh, nil, &theTruth, nil) + q.SetField("field") + return q + }(), + }, + nil), + }, + { + input: `field:<-5`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewNumericRangeInclusiveQuery(nil, &minusFivePointOh, nil, &theFalsehood) + q.SetField("field") + return q + }(), + }, + nil), + }, + { + input: `field:<=-5`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewNumericRangeInclusiveQuery(nil, &minusFivePointOh, nil, &theTruth) + q.SetField("field") + return q + }(), + }, + nil), + }, + { + input: `field:>"2006-01-02T15:04:05Z"`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewDateRangeInclusiveQuery(theDate, time.Time{}, &theFalsehood, nil) + q.SetField("field") + return q + }(), + }, + nil), + }, + { + input: `field:>="2006-01-02T15:04:05Z"`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewDateRangeInclusiveQuery(theDate, time.Time{}, &theTruth, nil) + q.SetField("field") + return q + }(), + }, + nil), + }, + { + input: `field:<"2006-01-02T15:04:05Z"`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewDateRangeInclusiveQuery(time.Time{}, theDate, nil, &theFalsehood) + q.SetField("field") + return q + }(), + }, + nil), + }, + { + input: `field:<="2006-01-02T15:04:05Z"`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewDateRangeInclusiveQuery(time.Time{}, theDate, nil, &theTruth) + q.SetField("field") + return q + }(), + }, + nil), + }, + { + input: `/mar.*ty/`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + NewRegexpQuery("mar.*ty"), + }, + nil), + }, + { + input: `name:/mar.*ty/`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewRegexpQuery("mar.*ty") + q.SetField("name") + return q + }(), + }, + nil), + }, + { + input: `mart*`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + NewWildcardQuery("mart*"), + }, + nil), + }, + { + input: `name:mart*`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewWildcardQuery("mart*") + q.SetField("name") + return q + }(), + }, + nil), + }, + + // tests for escaping + + // escape : as field delimiter + { + input: `name\:marty`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + NewMatchQuery("name:marty"), + }, + nil), + }, + // first colon delimiter, second escaped + { + input: `name:marty\:couchbase`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewMatchQuery("marty:couchbase") + q.SetField("name") + return q + }(), + }, + nil), + }, + // escape space, single argument to match query + { + input: `marty\ couchbase`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + NewMatchQuery("marty couchbase"), + }, + nil), + }, + // escape leading plus, not a must clause + { + input: `\+marty`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + NewMatchQuery("+marty"), + }, + nil), + }, + // escape leading minus, not a must not clause + { + input: `\-marty`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + NewMatchQuery("-marty"), + }, + nil), + }, + // escape quote inside of phrase + { + input: `"what does \"quote\" mean"`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + NewMatchPhraseQuery(`what does "quote" mean`), + }, + nil), + }, + // escaping an unsupported character retains backslash + { + input: `can\ i\ escap\e`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + NewMatchQuery(`can i escap\e`), + }, + nil), + }, + // leading spaces + { + input: ` what`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + NewMatchQuery(`what`), + }, + nil), + }, + // no boost value defaults to 1 + { + input: `term^`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewMatchQuery(`term`) + q.SetBoost(1.0) + return q + }(), + }, + nil), + }, + // weird lexer cases, something that starts like a number + // but contains escape and ends up as string + { + input: `3.0\:`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + NewMatchQuery(`3.0:`), + }, + nil), + }, + { + input: `3.0\a`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + NewMatchQuery(`3.0\a`), + }, + nil), + }, + + // field names as phrases + { + input: `"fie ld":test`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewMatchQuery("test") + q.SetField("fie ld") + return q + }(), + }, + nil), + }, + { + input: `"fie ld":"test"`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewMatchPhraseQuery("test") + q.SetField("fie ld") + return q + }(), + }, + nil), + }, + // exact match number with boost + { + input: `age:65^10`, + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + func() Query { + q := NewDisjunctionQuery([]Query{ + func() Query { + mq := NewMatchQuery("65") + mq.SetField("age") + return mq + }(), + func() Query { + val := float64(65) + inclusive := true + nq := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive) + nq.SetField("age") + return nq + }(), + }) + q.SetBoost(10) + q.queryStringMode = true + return q + }(), + }, + nil), + }, + } + + // turn on lexer debugging + // debugLexer = true + // debugParser = true + // logger = log.New(os.Stderr, "bleve ", log.LstdFlags) + + for _, test := range tests { + + q, err := parseQuerySyntax(test.input) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(q, test.result) { + t.Errorf("Expected %#v, got %#v: for %s", test.result, q, test.input) + } + } +} + +func TestQuerySyntaxParserInvalid(t *testing.T) { + tests := []struct { + input string + }{ + {"^"}, + {"^5"}, + {"field:-text"}, + {"field:+text"}, + {"field:>text"}, + {"field:>=text"}, + {"field:99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999`}, + {`field:>=99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999`}, + {`field:<99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999`}, + {`field:<=99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999`}, + } + + // turn on lexer debugging + // debugLexer = true + // logger = log.New(os.Stderr, "bleve", log.LstdFlags) + + for _, test := range tests { + _, err := parseQuerySyntax(test.input) + if err == nil { + t.Errorf("expected error, got nil for `%s`", test.input) + } + } +} + +func BenchmarkLexer(b *testing.B) { + for n := 0; n < b.N; n++ { + var tokenTypes []int + var tokens []yySymType + r := strings.NewReader(`+field4:"test phrase 1"`) + l := getQueryStringLex(r) + + var lval yySymType + rv := l.Lex(&lval) + + for rv > 0 { + tokenTypes = append(tokenTypes, rv) + tokens = append(tokens, lval) + + // use the slice to silence the compiler warning + _ = tokenTypes + _ = tokens + + lval.s = "" + lval.n = 0 + rv = l.Lex(&lval) + } + putQueryStringLex(l) + } +} + +func BenchmarkLexer_Parallel(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + var tokenTypes []int + var tokens []yySymType + r := strings.NewReader(`+field4:"test phrase 1"`) + l := getQueryStringLex(r) + + var lval yySymType + rv := l.Lex(&lval) + + for rv > 0 { + tokenTypes = append(tokenTypes, rv) + tokens = append(tokens, lval) + + // use the slice to silence the compiler warning + _ = tokenTypes + _ = tokens + + lval.s = "" + lval.n = 0 + rv = l.Lex(&lval) + } + putQueryStringLex(l) + } + }) +} diff --git a/search/query/query_test.go b/search/query/query_test.go new file mode 100644 index 0000000..fa020d7 --- /dev/null +++ b/search/query/query_test.go @@ -0,0 +1,1054 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "reflect" + "sort" + "strings" + "testing" + "time" + + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/mapping" +) + +var minNum = 5.1 +var maxNum = 7.1 +var minTerm = "bob" +var maxTerm = "cat" +var startDateStr = "2011-01-01T00:00:00Z" +var endDateStr = "2012-01-01T00:00:00Z" +var startDate time.Time +var endDate time.Time + +func init() { + var err error + startDate, err = time.Parse(time.RFC3339, startDateStr) + if err != nil { + panic(err) + } + endDate, err = time.Parse(time.RFC3339, endDateStr) + if err != nil { + panic(err) + } +} + +func TestParseQuery(t *testing.T) { + tests := []struct { + input []byte + output Query + err bool + }{ + { + input: []byte(`{"term":"water","field":"desc"}`), + output: func() Query { + q := NewTermQuery("water") + q.SetField("desc") + return q + }(), + }, + { + input: []byte(`{"match":"beer","field":"desc"}`), + output: func() Query { + q := NewMatchQuery("beer") + q.SetField("desc") + return q + }(), + }, + { + input: []byte(`{"match":"beer","field":"desc","operator":"or"}`), + output: func() Query { + q := NewMatchQuery("beer") + q.SetField("desc") + return q + }(), + }, + { + input: []byte(`{"match":"beer","field":"desc","operator":"and"}`), + output: func() Query { + q := NewMatchQuery("beer") + q.SetOperator(MatchQueryOperatorAnd) + q.SetField("desc") + return q + }(), + }, + { + input: []byte(`{"match":"beer","field":"desc","operator":"and"}`), + output: func() Query { + operator := MatchQueryOperatorAnd + q := NewMatchQuery("beer") + q.SetOperator(operator) + q.SetField("desc") + return q + }(), + }, + { + input: []byte(`{"match":"beer","field":"desc","operator":"or"}`), + output: func() Query { + q := NewMatchQuery("beer") + q.SetOperator(MatchQueryOperatorOr) + q.SetField("desc") + return q + }(), + }, + { + input: []byte(`{"match":"beer","field":"desc","operator":"or"}`), + output: func() Query { + operator := MatchQueryOperatorOr + q := NewMatchQuery("beer") + q.SetOperator(operator) + q.SetField("desc") + return q + }(), + }, + { + input: []byte(`{"match":"beer","field":"desc","operator":"does not exist"}`), + output: nil, + err: true, + }, + { + input: []byte(`{"match_phrase":"light beer","field":"desc"}`), + output: func() Query { + q := NewMatchPhraseQuery("light beer") + q.SetField("desc") + return q + }(), + }, + { + input: []byte(`{"must":{"conjuncts": [{"match":"beer","field":"desc"}]},"should":{"disjuncts": [{"match":"water","field":"desc"}],"min":1.0},"must_not":{"disjuncts": [{"match":"devon","field":"desc"}]}}`), + output: func() Query { + q := NewBooleanQuery( + []Query{func() Query { + q := NewMatchQuery("beer") + q.SetField("desc") + return q + }()}, + []Query{func() Query { + q := NewMatchQuery("water") + q.SetField("desc") + return q + }()}, + []Query{func() Query { + q := NewMatchQuery("devon") + q.SetField("desc") + return q + }()}) + q.SetMinShould(1) + return q + }(), + }, + { + input: []byte(`{"terms":["watered","down"],"field":"desc"}`), + output: NewPhraseQuery([]string{"watered", "down"}, "desc"), + }, + { + input: []byte(`{"query":"+beer \"light beer\" -devon"}`), + output: NewQueryStringQuery(`+beer "light beer" -devon`), + }, + { + input: []byte(`{"min":5.1,"max":7.1,"field":"desc"}`), + output: func() Query { + q := NewNumericRangeQuery(&minNum, &maxNum) + q.SetField("desc") + return q + }(), + }, + { + input: []byte(`{"min":"bob","max":"cat","field":"desc"}`), + output: func() Query { + q := NewTermRangeQuery(minTerm, maxTerm) + q.SetField("desc") + return q + }(), + }, + { + input: []byte(`{"start":"` + startDateStr + `","end":"` + endDateStr + `","field":"desc"}`), + output: func() Query { + q := NewDateRangeStringQuery(startDateStr, endDateStr) + q.SetField("desc") + return q + }(), + }, + { + input: []byte(`{"prefix":"budwei","field":"desc"}`), + output: func() Query { + q := NewPrefixQuery("budwei") + q.SetField("desc") + return q + }(), + }, + { + input: []byte(`{"match_all":{}}`), + output: NewMatchAllQuery(), + }, + { + input: []byte(`{"match_none":{}}`), + output: NewMatchNoneQuery(), + }, + { + input: []byte(`{"ids":["a","b","c"]}`), + output: NewDocIDQuery([]string{"a", "b", "c"}), + }, + { + input: []byte(`{"bool": true}`), + output: NewBoolFieldQuery(true), + }, + { + input: []byte(`{"field": "x", "cidr": "1.2.3.0/4"}`), + output: func() Query { + q := NewIPRangeQuery("1.2.3.0/4") + q.SetField("x") + return q + }(), + }, + { + input: []byte(`{"madeitup":"queryhere"}`), + output: nil, + err: true, + }, + } + + for i, test := range tests { + actual, err := ParseQuery(test.input) + if err != nil && test.err == false { + t.Errorf("error %v for %d", err, i) + } + + if !reflect.DeepEqual(test.output, actual) { + t.Errorf("expected: %#v, got: %#v for %s", test.output, actual, string(test.input)) + } + } +} + +func TestQueryValidate(t *testing.T) { + tests := []struct { + query Query + err bool + }{ + { + query: func() Query { + q := NewTermQuery("water") + q.SetField("desc") + return q + }(), + }, + { + query: func() Query { + q := NewMatchQuery("beer") + q.SetField("desc") + return q + }(), + }, + { + query: func() Query { + q := NewMatchPhraseQuery("light beer") + q.SetField("desc") + return q + }(), + }, + { + query: func() Query { + q := NewNumericRangeQuery(&minNum, &maxNum) + q.SetField("desc") + return q + }(), + }, + { + query: func() Query { + q := NewNumericRangeQuery(nil, nil) + q.SetField("desc") + return q + }(), + err: true, + }, + { + query: func() Query { + q := NewDateRangeQuery(startDate, endDate) + q.SetField("desc") + return q + }(), + }, + { + query: func() Query { + q := NewPrefixQuery("budwei") + q.SetField("desc") + return q + }(), + }, + { + query: NewQueryStringQuery(`+beer "light beer" -devon`), + }, + { + query: NewPhraseQuery([]string{"watered", "down"}, "desc"), + }, + { + query: NewPhraseQuery([]string{}, "field"), + err: true, + }, + { + query: func() Query { + q := NewMatchNoneQuery() + q.SetBoost(25) + return q + }(), + }, + { + query: func() Query { + q := NewMatchAllQuery() + q.SetBoost(25) + return q + }(), + }, + { + query: NewBooleanQuery( + []Query{func() Query { + q := NewMatchQuery("beer") + q.SetField("desc") + return q + }()}, + []Query{func() Query { + q := NewMatchQuery("water") + q.SetField("desc") + return q + }()}, + []Query{func() Query { + q := NewMatchQuery("devon") + q.SetField("desc") + return q + }()}), + }, + { + query: NewBooleanQuery( + nil, + nil, + []Query{func() Query { + q := NewMatchQuery("devon") + q.SetField("desc") + return q + }()}), + }, + { + query: NewBooleanQuery( + []Query{}, + []Query{}, + []Query{func() Query { + q := NewMatchQuery("devon") + q.SetField("desc") + return q + }()}), + }, + { + query: NewBooleanQuery( + nil, + nil, + nil), + err: true, + }, + { + query: NewBooleanQuery( + []Query{}, + []Query{}, + []Query{}), + err: true, + }, + { + query: func() Query { + q := NewBooleanQuery( + []Query{func() Query { + q := NewMatchQuery("beer") + q.SetField("desc") + return q + }()}, + []Query{func() Query { + q := NewMatchQuery("water") + q.SetField("desc") + return q + }()}, + []Query{func() Query { + q := NewMatchQuery("devon") + q.SetField("desc") + return q + }()}) + q.SetMinShould(2) + return q + }(), + err: true, + }, + { + query: func() Query { + q := NewDocIDQuery(nil) + q.SetBoost(25) + return q + }(), + }, + } + + for _, test := range tests { + if vq, ok := test.query.(ValidatableQuery); ok { + actual := vq.Validate() + if actual != nil && !test.err { + t.Errorf("expected no error: %#v got %#v", test.err, actual) + } else if actual == nil && test.err { + t.Errorf("expected error: %#v got %#v", test.err, actual) + } + } + } +} + +func TestDumpQuery(t *testing.T) { + mapping := mapping.NewIndexMapping() + q := NewQueryStringQuery("+water -light beer") + s, err := DumpQuery(mapping, q) + if err != nil { + t.Fatal(err) + } + s = strings.TrimSpace(s) + wanted := strings.TrimSpace(`{ + "must": { + "conjuncts": [ + { + "match": "water", + "prefix_length": 0, + "fuzziness": 0 + } + ] + }, + "should": { + "disjuncts": [ + { + "match": "beer", + "prefix_length": 0, + "fuzziness": 0 + } + ], + "min": 0 + }, + "must_not": { + "disjuncts": [ + { + "match": "light", + "prefix_length": 0, + "fuzziness": 0 + } + ], + "min": 0 + } +}`) + if wanted != s { + t.Fatalf("query:\n%s\ndiffers from expected:\n%s", s, wanted) + } +} + +func TestGeoShapeQuery(t *testing.T) { + tests := []struct { + input []byte + output Query + err bool + }{ + { + input: []byte(`{ + "field" : "region", + "geometry": { + "shape": { + "type": "polygon", + "coordinates": [[ + [ + 74.1357421875, + 30.600093873550072 + ], + [ + 67.0166015625, + 21.57571893245848 + ], + [ + 68.8623046875, + 9.145486056167277 + ], + [ + 83.1884765625, + 4.083452772038619 + ], + [ + 88.9892578125, + 22.67484735118852 + ], + [ + 74.1357421875, + 30.600093873550072 + ]]] + }, + "relation": "intersects" + }}`), + output: func() Query { + q, _ := NewGeoShapeQuery([][][][]float64{{{{74.1357421875, 30.600093873550072}, + {67.0166015625, 21.57571893245848}, {68.8623046875, 9.145486056167277}, + {83.1884765625, 4.083452772038619}, {88.9892578125, 22.67484735118852}, + {74.1357421875, 30.600093873550072}}}}, geo.PolygonType, "intersects") + q.SetField("region") + return q + }(), + }, + { + input: []byte(`{ + "field" : "region", + "geometry": { + "shape": { + "type": "multipolygon", + "coordinates": [ + [[ + [ + 77.58268117904663, + 12.980513152175025 + ], + [ + 77.58147954940794, + 12.977983107483992 + ], + [ + 77.58708000183104, + 12.97886130773254 + ], + [ + 77.58268117904663, + 12.980513152175025 + ] + ]], + [[ + [ + 77.5864577293396, + 12.97762764459667 + ], + [ + 77.58879661560059, + 12.975076660730531 + ], + [ + 77.59115695953369, + 12.979216768855913 + ], + [ + 77.5864577293396, + 12.97762764459667 + ] + ]] + ] + }, + "relation": "contains" + }}`), + output: func() Query { + q, _ := NewGeoShapeQuery([][][][]float64{ + {{{77.58268117904663, 12.980513152175025}, + {77.58147954940794, 12.977983107483992}, {77.58708000183104, 12.97886130773254}, + {77.58268117904663, 12.980513152175025}}}, + {{{77.5864577293396, 12.97762764459667}, {77.58879661560059, 12.975076660730531}, + {77.59115695953369, 12.979216768855913}, {77.5864577293396, 12.97762764459667}}}}, + geo.MultiPolygonType, "contains") + q.SetField("region") + return q + }(), + }, + { + input: []byte(`{ + "field" : "region", + "geometry": { + "shape": { + "type": "point", + "coordinates": [77.58268117904663, 12.980513152175025] + }, + "relation": "contains" + }}`), + output: func() Query { + q, _ := NewGeoShapeQuery([][][][]float64{ + {{{77.58268117904663, 12.980513152175025}}}}, + geo.PointType, "contains") + q.SetField("region") + return q + }(), + }, + { + input: []byte(`{ + "field" : "region", + "geometry": { + "shape": { + "type": "multipoint", + "coordinates": [[77.58268117904663, 12.980513152175025], + [77.5864577293396, 12.97762764459667]] + }, + "relation": "intersects" + }}`), + output: func() Query { + q, _ := NewGeoShapeQuery([][][][]float64{ + {{{77.58268117904663, 12.980513152175025}, + {77.5864577293396, 12.97762764459667}}}}, + geo.MultiPointType, "intersects") + q.SetField("region") + return q + }(), + }, + { + input: []byte(`{ + "field" : "region", + "geometry": { + "shape": { + "type": "linestring", + "coordinates": [[77.58268117904663, 12.980513152175025], + [77.5864577293396, 12.97762764459667]] + }, + "relation": "intersects" + }}`), + output: func() Query { + q, _ := NewGeoShapeQuery([][][][]float64{ + {{{77.58268117904663, 12.980513152175025}, + {77.5864577293396, 12.97762764459667}}}}, + geo.LineStringType, "intersects") + q.SetField("region") + return q + }(), + }, + { + input: []byte(`{ + "field" : "region", + "geometry": { + "shape": { + "type": "multilinestring", + "coordinates": [ + [[77.58268117904663, 12.980513152175025], + [77.5864577293396, 12.97762764459667]], + [[77.5864577293396,12.97762764459667], + [77.58879661560059, 12.975076660730531]]] + }, + "relation": "intersects" + }}`), + output: func() Query { + q, _ := NewGeoShapeQuery([][][][]float64{{ + {{77.58268117904663, 12.980513152175025}, + {77.5864577293396, 12.97762764459667}}, + {{77.5864577293396, 12.97762764459667}, + {77.58879661560059, 12.975076660730531}}}}, + geo.MultiLineStringType, "intersects") + + q.SetField("region") + return q + }(), + }, + { + input: []byte(`{ + "field" : "region", + "geometry": { + "shape": { + "type": "envelope", + "coordinates": [[77.58268117904663, 12.980513152175025], + [77.5864577293396, 12.97762764459667]] + }, + "relation": "within" + }}`), + output: func() Query { + q, _ := NewGeoShapeQuery([][][][]float64{{ + {{77.58268117904663, 12.980513152175025}, + {77.5864577293396, 12.97762764459667}}}}, + geo.EnvelopeType, "within") + + q.SetField("region") + return q + }(), + }, + { + input: []byte(`{ + "field" : "region", + "geometry": { + "shape": { + "type": "circle", + "coordinates": [77.58268117904663, 12.980513152175025], + "radius": "100m" + }, + "relation": "within" + }}`), + output: func() Query { + q, _ := NewGeoShapeCircleQuery([]float64{ + 77.58268117904663, 12.980513152175025}, + "100m", "within") + + q.SetField("region") + return q + }(), + }, + { + input: []byte(`{ + "field" : "region", + "geometry": { + "shape": { + "type": "geometrycollection", + "geometries": [ + { + "type": "point", + "coordinates": [ + 77.59158611297607, + 12.972002899506203 + ] + }, + { + "type": "linestring", + "coordinates": [ + [ + 77.58851766586304, + 12.973152950670608 + ], + [ + 77.58937597274779, + 12.972212000113458 + ] + ] + }, + { + "type": "polygon", + "coordinates": [ + [ + [ + 77.59055614471436, + 12.974721193688106 + ], + [ + 77.58954763412476, + 12.97350841995465 + ], + [ + 77.59141445159912, + 12.973382960265356 + ], + [ + 77.59055614471436, + 12.974721193688106 + ] + ] + ] + } + ] + }, + "relation": "contains" + }}`), + output: func() Query { + q, _ := NewGeometryCollectionQuery([][][][][]float64{ + {{{{77.59158611297607, 12.972002899506203}}}}, + {{{{77.58851766586304, 12.973152950670608}, {77.58937597274779, 12.972212000113458}}}}, + {{{{77.59055614471436, 12.974721193688106}, {77.58954763412476, 12.97350841995465}, + {77.59141445159912, 12.973382960265356}, {77.59055614471436, 12.974721193688106}}}}, + }, + []string{"point", "linestring", "polygon"}, "contains") + q.SetField("region") + return q + }(), + }, + } + + for i, test := range tests { + actual, err := ParseQuery(test.input) + if err != nil && test.err == false { + t.Errorf("error %v for %d", err, i) + } + + if !reflect.DeepEqual(test.output, actual) { + t.Errorf("expected: %#v, got: %#v for %s", test.output, actual, string(test.input)) + } + } +} + +func TestParseEmptyQuery(t *testing.T) { + var qBytes []byte + rv, err := ParseQuery(qBytes) + if err != nil { + t.Fatal(err) + } + expect := NewMatchNoneQuery() + if !reflect.DeepEqual(rv, expect) { + t.Errorf("[1] Expected %#v, got %#v", expect, rv) + } + + qBytes = []byte(`{}`) + rv, err = ParseQuery(qBytes) + if err != nil { + t.Fatal(err) + } + expect = NewMatchNoneQuery() + if !reflect.DeepEqual(rv, expect) { + t.Errorf("[2] Expected %#v, got %#v", expect, rv) + } +} + +func TestExtractFields(t *testing.T) { + testQueries := []struct { + query string + expFields []string + }{ + { + query: `{"term":"water","field":"desc"}`, + expFields: []string{"desc"}, + }, + { + query: `{ + "must": { + "conjuncts": [ + { + "match": "water", + "prefix_length": 0, + "fuzziness": 0 + } + ] + }, + "should": { + "disjuncts": [ + { + "match": "beer", + "prefix_length": 0, + "fuzziness": 0 + } + ], + "min": 0 + }, + "must_not": { + "disjuncts": [ + { + "match": "light", + "prefix_length": 0, + "fuzziness": 0 + } + ], + "min": 0 + } + }`, + expFields: []string{"_all"}, + }, + { + query: `{ + "must": { + "conjuncts": [ + { + "match": "water", + "prefix_length": 0, + "field": "desc", + "fuzziness": 0 + } + ] + }, + "should": { + "disjuncts": [ + { + "match": "beer", + "prefix_length": 0, + "field": "desc", + "fuzziness": 0 + } + ], + "min": 0 + }, + "must_not": { + "disjuncts": [ + { + "match": "light", + "prefix_length": 0, + "field": "genre", + "fuzziness": 0 + } + ], + "min": 0 + } + }`, + expFields: []string{"desc", "genre"}, + }, + { + query: ` + { + "conjuncts": [ + { + "conjuncts": [ + { + "conjuncts": [ + { + "conjuncts": [ + { + "field": "date", + "start": "2002-09-05T08:09:00Z", + "end": "2007-03-01T03:52:00Z", + "inclusive_start": true, + "inclusive_end": true + }, + { + "field": "number", + "min": 1260295, + "max": 3917314, + "inclusive_min": true, + "inclusive_max": true + } + ] + }, + { + "conjuncts": [ + { + "field": "date2", + "start": "2004-08-21T18:30:00Z", + "end": "2006-03-24T08:08:00Z", + "inclusive_start": true, + "inclusive_end": true + }, + { + "field": "number", + "min": 165449, + "max": 3847517, + "inclusive_min": true, + "inclusive_max": true + } + ] + } + ] + }, + { + "conjuncts": [ + { + "conjuncts": [ + { + "field": "date", + "start": "2004-09-02T22:15:00Z", + "end": "2008-06-22T15:06:00Z", + "inclusive_start": true, + "inclusive_end": true + }, + { + "field": "number2", + "min": 876843, + "max": 3363351, + "inclusive_min": true, + "inclusive_max": true + } + ] + }, + { + "conjuncts": [ + { + "field": "date", + "start": "2000-12-03T21:35:00Z", + "end": "2008-02-07T05:00:00Z", + "inclusive_start": true, + "inclusive_end": true + }, + { + "field": "number", + "min": 2021479, + "max": 4763404, + "inclusive_min": true, + "inclusive_max": true + } + ] + } + ] + } + ] + }, + { + "conjuncts": [ + { + "conjuncts": [ + { + "field": "date3", + "start": "2000-03-13T07:13:00Z", + "end": "2005-09-19T09:33:00Z", + "inclusive_start": true, + "inclusive_end": true + }, + { + "field": "number", + "min": 883125, + "max": 4817433, + "inclusive_min": true, + "inclusive_max": true + } + ] + }, + { + "conjuncts": [ + { + "field": "date", + "start": "2002-08-10T22:42:00Z", + "end": "2008-02-10T23:19:00Z", + "inclusive_start": true, + "inclusive_end": true + }, + { + "field": "number", + "min": 896115, + "max": 3897074, + "inclusive_min": true, + "inclusive_max": true + } + ] + } + ] + } + ] + }`, + expFields: []string{"date", "number", "date2", "number2", "date3"}, + }, + { + query: `{ + "query" : "hardworking people" + }`, + expFields: []string{"_all"}, + }, + { + query: `{ + "query" : "text:hardworking people" + }`, + expFields: []string{"text", "_all"}, + }, + { + query: `{ + "query" : "text:\"hardworking people\"" + }`, + expFields: []string{"text"}, + }, + { + query: `{ + "match_all": {} + }`, + expFields: []string{"_id"}, + }, + { + query: `{ + "ids": ["a", "b", "c"] + }`, + expFields: []string{"_id"}, + }, + } + + m := mapping.NewIndexMapping() + for i, test := range testQueries { + q, err := ParseQuery([]byte(test.query)) + if err != nil { + t.Fatal(err) + } + fields, err := ExtractFields(q, m, nil) + if err != nil { + t.Fatal(err) + } + var fieldsSlice []string + for k := range fields { + fieldsSlice = append(fieldsSlice, k) + } + sort.Strings(test.expFields) + sort.Strings(fieldsSlice) + if !reflect.DeepEqual(fieldsSlice, test.expFields) { + t.Errorf("Test %d: expected %v, got %v", i, test.expFields, fieldsSlice) + } + } +} diff --git a/search/query/regexp.go b/search/query/regexp.go new file mode 100644 index 0000000..189fd5f --- /dev/null +++ b/search/query/regexp.go @@ -0,0 +1,79 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "strings" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +type RegexpQuery struct { + Regexp string `json:"regexp"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewRegexpQuery creates a new Query which finds +// documents containing terms that match the +// specified regular expression. The regexp pattern +// SHOULD NOT include ^ or $ modifiers, the search +// will only match entire terms even without them. +func NewRegexpQuery(regexp string) *RegexpQuery { + return &RegexpQuery{ + Regexp: regexp, + } +} + +func (q *RegexpQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *RegexpQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *RegexpQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *RegexpQuery) Field() string { + return q.FieldVal +} + +func (q *RegexpQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + // require that pattern NOT be anchored to start and end of term. + // do not attempt to remove trailing $, its presence is not + // known to interfere with LiteralPrefix() the way ^ does + // and removing $ introduces possible ambiguities with escaped \$, \\$, etc + actualRegexp := q.Regexp + actualRegexp = strings.TrimPrefix(actualRegexp, "^") // remove leading ^ if it exists + + return searcher.NewRegexpStringSearcher(ctx, i, actualRegexp, field, q.BoostVal.Value(), options) +} + +func (q *RegexpQuery) Validate() error { + return nil // real validation delayed until searcher constructor +} diff --git a/search/query/term.go b/search/query/term.go new file mode 100644 index 0000000..5c6af39 --- /dev/null +++ b/search/query/term.go @@ -0,0 +1,63 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +type TermQuery struct { + Term string `json:"term"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewTermQuery creates a new Query for finding an +// exact term match in the index. +func NewTermQuery(term string) *TermQuery { + return &TermQuery{ + Term: term, + } +} + +func (q *TermQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *TermQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *TermQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *TermQuery) Field() string { + return q.FieldVal +} + +func (q *TermQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + return searcher.NewTermSearcher(ctx, i, q.Term, field, q.BoostVal.Value(), options) +} diff --git a/search/query/term_range.go b/search/query/term_range.go new file mode 100644 index 0000000..4dc3a34 --- /dev/null +++ b/search/query/term_range.go @@ -0,0 +1,96 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +type TermRangeQuery struct { + Min string `json:"min,omitempty"` + Max string `json:"max,omitempty"` + InclusiveMin *bool `json:"inclusive_min,omitempty"` + InclusiveMax *bool `json:"inclusive_max,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewTermRangeQuery creates a new Query for ranges +// of text term values. +// Either, but not both endpoints can be nil. +// The minimum value is inclusive. +// The maximum value is exclusive. +func NewTermRangeQuery(min, max string) *TermRangeQuery { + return NewTermRangeInclusiveQuery(min, max, nil, nil) +} + +// NewTermRangeInclusiveQuery creates a new Query for ranges +// of numeric values. +// Either, but not both endpoints can be nil. +// Control endpoint inclusion with inclusiveMin, inclusiveMax. +func NewTermRangeInclusiveQuery(min, max string, minInclusive, maxInclusive *bool) *TermRangeQuery { + return &TermRangeQuery{ + Min: min, + Max: max, + InclusiveMin: minInclusive, + InclusiveMax: maxInclusive, + } +} + +func (q *TermRangeQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *TermRangeQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *TermRangeQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *TermRangeQuery) Field() string { + return q.FieldVal +} + +func (q *TermRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + var minTerm []byte + if q.Min != "" { + minTerm = []byte(q.Min) + } + var maxTerm []byte + if q.Max != "" { + maxTerm = []byte(q.Max) + } + return searcher.NewTermRangeSearcher(ctx, i, minTerm, maxTerm, q.InclusiveMin, q.InclusiveMax, field, q.BoostVal.Value(), options) +} + +func (q *TermRangeQuery) Validate() error { + if q.Min == "" && q.Min == q.Max { + return fmt.Errorf("term range query must specify min or max") + } + return nil +} diff --git a/search/query/wildcard.go b/search/query/wildcard.go new file mode 100644 index 0000000..f04f3f2 --- /dev/null +++ b/search/query/wildcard.go @@ -0,0 +1,94 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "strings" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +var wildcardRegexpReplacer = strings.NewReplacer( + // characters in the wildcard that must + // be escaped in the regexp + "+", `\+`, + "(", `\(`, + ")", `\)`, + "^", `\^`, + "$", `\$`, + ".", `\.`, + "{", `\{`, + "}", `\}`, + "[", `\[`, + "]", `\]`, + `|`, `\|`, + `\`, `\\`, + // wildcard characters + "*", ".*", + "?", ".") + +type WildcardQuery struct { + Wildcard string `json:"wildcard"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewWildcardQuery creates a new Query which finds +// documents containing terms that match the +// specified wildcard. In the wildcard pattern '*' +// will match any sequence of 0 or more characters, +// and '?' will match any single character. +func NewWildcardQuery(wildcard string) *WildcardQuery { + return &WildcardQuery{ + Wildcard: wildcard, + } +} + +func (q *WildcardQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *WildcardQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *WildcardQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *WildcardQuery) Field() string { + return q.FieldVal +} + +func (q *WildcardQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + regexpString := wildcardRegexpReplacer.Replace(q.Wildcard) + + return searcher.NewRegexpStringSearcher(ctx, i, regexpString, field, + q.BoostVal.Value(), options) +} + +func (q *WildcardQuery) Validate() error { + return nil // real validation delayed until searcher constructor +} diff --git a/search/scorer/scorer_conjunction.go b/search/scorer/scorer_conjunction.go new file mode 100644 index 0000000..f3c81a7 --- /dev/null +++ b/search/scorer/scorer_conjunction.go @@ -0,0 +1,72 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorer + +import ( + "reflect" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" +) + +var reflectStaticSizeConjunctionQueryScorer int + +func init() { + var cqs ConjunctionQueryScorer + reflectStaticSizeConjunctionQueryScorer = int(reflect.TypeOf(cqs).Size()) +} + +type ConjunctionQueryScorer struct { + options search.SearcherOptions +} + +func (s *ConjunctionQueryScorer) Size() int { + return reflectStaticSizeConjunctionQueryScorer + size.SizeOfPtr +} + +func NewConjunctionQueryScorer(options search.SearcherOptions) *ConjunctionQueryScorer { + return &ConjunctionQueryScorer{ + options: options, + } +} + +func (s *ConjunctionQueryScorer) Score(ctx *search.SearchContext, constituents []*search.DocumentMatch) *search.DocumentMatch { + var sum float64 + var childrenExplanations []*search.Explanation + if s.options.Explain { + childrenExplanations = make([]*search.Explanation, len(constituents)) + } + + for i, docMatch := range constituents { + sum += docMatch.Score + if s.options.Explain { + childrenExplanations[i] = docMatch.Expl + } + } + newScore := sum + var newExpl *search.Explanation + if s.options.Explain { + newExpl = &search.Explanation{Value: sum, Message: "sum of:", Children: childrenExplanations} + } + + // reuse constituents[0] as the return value + rv := constituents[0] + rv.Score = newScore + rv.Expl = newExpl + rv.FieldTermLocations = search.MergeFieldTermLocations( + rv.FieldTermLocations, constituents[1:]) + + return rv +} diff --git a/search/scorer/scorer_constant.go b/search/scorer/scorer_constant.go new file mode 100644 index 0000000..36310ec --- /dev/null +++ b/search/scorer/scorer_constant.go @@ -0,0 +1,132 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorer + +import ( + "fmt" + "reflect" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeConstantScorer int + +func init() { + var cs ConstantScorer + reflectStaticSizeConstantScorer = int(reflect.TypeOf(cs).Size()) +} + +type ConstantScorer struct { + constant float64 + boost float64 + options search.SearcherOptions + queryNorm float64 + queryWeight float64 + queryWeightExplanation *search.Explanation + includeScore bool +} + +func (s *ConstantScorer) Size() int { + sizeInBytes := reflectStaticSizeConstantScorer + size.SizeOfPtr + + if s.queryWeightExplanation != nil { + sizeInBytes += s.queryWeightExplanation.Size() + } + + return sizeInBytes +} + +func NewConstantScorer(constant float64, boost float64, options search.SearcherOptions) *ConstantScorer { + rv := ConstantScorer{ + options: options, + queryWeight: 1.0, + constant: constant, + boost: boost, + includeScore: options.Score != "none", + } + + return &rv +} + +func (s *ConstantScorer) Weight() float64 { + sum := s.boost + return sum * sum +} + +func (s *ConstantScorer) SetQueryNorm(qnorm float64) { + s.queryNorm = qnorm + + // update the query weight + s.queryWeight = s.boost * s.queryNorm + + if s.options.Explain { + childrenExplanations := make([]*search.Explanation, 2) + childrenExplanations[0] = &search.Explanation{ + Value: s.boost, + Message: "boost", + } + childrenExplanations[1] = &search.Explanation{ + Value: s.queryNorm, + Message: "queryNorm", + } + s.queryWeightExplanation = &search.Explanation{ + Value: s.queryWeight, + Message: fmt.Sprintf("ConstantScore()^%f, product of:", s.boost), + Children: childrenExplanations, + } + } +} + +func (s *ConstantScorer) Score(ctx *search.SearchContext, id index.IndexInternalID) *search.DocumentMatch { + var scoreExplanation *search.Explanation + + rv := ctx.DocumentMatchPool.Get() + rv.IndexInternalID = index.NewIndexInternalIDFrom(rv.IndexInternalID, id) + + if s.includeScore { + score := s.constant + + if s.options.Explain { + scoreExplanation = &search.Explanation{ + Value: score, + Message: "ConstantScore()", + } + } + + // if the query weight isn't 1, multiply + if s.queryWeight != 1.0 { + score = score * s.queryWeight + if s.options.Explain { + childExplanations := make([]*search.Explanation, 2) + childExplanations[0] = s.queryWeightExplanation + childExplanations[1] = scoreExplanation + scoreExplanation = &search.Explanation{ + Value: score, + Message: fmt.Sprintf("weight(^%f), product of:", s.boost), + Children: childExplanations, + } + } + } + + rv.Score = score + if s.options.Explain { + rv.Expl = scoreExplanation + } + } + + return rv +} diff --git a/search/scorer/scorer_constant_test.go b/search/scorer/scorer_constant_test.go new file mode 100644 index 0000000..71f0869 --- /dev/null +++ b/search/scorer/scorer_constant_test.go @@ -0,0 +1,131 @@ +// Copyright (c) 2013 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorer + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func TestConstantScorer(t *testing.T) { + + scorer := NewConstantScorer(1, 1, search.SearcherOptions{Explain: true}) + + tests := []struct { + termMatch *index.TermFieldDoc + result *search.DocumentMatch + }{ + // test some simple math + { + termMatch: &index.TermFieldDoc{ + ID: index.IndexInternalID("one"), + Freq: 1, + Norm: 1.0, + Vectors: []*index.TermFieldVector{ + { + Field: "desc", + Pos: 1, + Start: 0, + End: 4, + }, + }, + }, + result: &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("one"), + Score: 1.0, + Expl: &search.Explanation{ + Value: 1.0, + Message: "ConstantScore()", + }, + Sort: []string{}, + }, + }, + } + + for _, test := range tests { + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(1, 0), + } + actual := scorer.Score(ctx, test.termMatch.ID) + + if !reflect.DeepEqual(actual, test.result) { + t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) + } + } + +} + +func TestConstantScorerWithQueryNorm(t *testing.T) { + + scorer := NewConstantScorer(1, 1, search.SearcherOptions{Explain: true}) + scorer.SetQueryNorm(2.0) + + tests := []struct { + termMatch *index.TermFieldDoc + result *search.DocumentMatch + }{ + { + termMatch: &index.TermFieldDoc{ + ID: index.IndexInternalID("one"), + Freq: 1, + Norm: 1.0, + }, + result: &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("one"), + Score: 2.0, + Sort: []string{}, + Expl: &search.Explanation{ + Value: 2.0, + Message: "weight(^1.000000), product of:", + Children: []*search.Explanation{ + { + Value: 2.0, + Message: "ConstantScore()^1.000000, product of:", + Children: []*search.Explanation{ + { + Value: 1, + Message: "boost", + }, + { + Value: 2, + Message: "queryNorm", + }, + }, + }, + { + Value: 1.0, + Message: "ConstantScore()", + }, + }, + }, + }, + }, + } + + for _, test := range tests { + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(1, 0), + } + actual := scorer.Score(ctx, test.termMatch.ID) + + if !reflect.DeepEqual(actual, test.result) { + t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) + } + } + +} diff --git a/search/scorer/scorer_disjunction.go b/search/scorer/scorer_disjunction.go new file mode 100644 index 0000000..7565970 --- /dev/null +++ b/search/scorer/scorer_disjunction.go @@ -0,0 +1,123 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorer + +import ( + "fmt" + "reflect" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" +) + +var reflectStaticSizeDisjunctionQueryScorer int + +func init() { + var dqs DisjunctionQueryScorer + reflectStaticSizeDisjunctionQueryScorer = int(reflect.TypeOf(dqs).Size()) +} + +type DisjunctionQueryScorer struct { + options search.SearcherOptions +} + +func (s *DisjunctionQueryScorer) Size() int { + return reflectStaticSizeDisjunctionQueryScorer + size.SizeOfPtr +} + +func NewDisjunctionQueryScorer(options search.SearcherOptions) *DisjunctionQueryScorer { + return &DisjunctionQueryScorer{ + options: options, + } +} + +func (s *DisjunctionQueryScorer) Score(ctx *search.SearchContext, constituents []*search.DocumentMatch, countMatch, countTotal int) *search.DocumentMatch { + var sum float64 + var childrenExplanations []*search.Explanation + if s.options.Explain { + childrenExplanations = make([]*search.Explanation, len(constituents)) + } + + for i, docMatch := range constituents { + sum += docMatch.Score + if s.options.Explain { + childrenExplanations[i] = docMatch.Expl + } + } + + var rawExpl *search.Explanation + if s.options.Explain { + rawExpl = &search.Explanation{Value: sum, Message: "sum of:", Children: childrenExplanations} + } + + coord := float64(countMatch) / float64(countTotal) + newScore := sum * coord + var newExpl *search.Explanation + if s.options.Explain { + ce := make([]*search.Explanation, 2) + ce[0] = rawExpl + ce[1] = &search.Explanation{Value: coord, Message: fmt.Sprintf("coord(%d/%d)", countMatch, countTotal)} + newExpl = &search.Explanation{Value: newScore, Message: "product of:", Children: ce, PartialMatch: countMatch != countTotal} + } + + // reuse constituents[0] as the return value + rv := constituents[0] + rv.Score = newScore + rv.Expl = newExpl + rv.FieldTermLocations = search.MergeFieldTermLocations( + rv.FieldTermLocations, constituents[1:]) + + return rv +} + +// This method is used only when disjunction searcher is used over multiple +// KNN searchers, where only the score breakdown and the optional explanation breakdown +// is required. The final score and explanation is set when we finalize the KNN hits. +func (s *DisjunctionQueryScorer) ScoreAndExplBreakdown(ctx *search.SearchContext, constituents []*search.DocumentMatch, + matchingIdxs []int, originalPositions []int, countTotal int) *search.DocumentMatch { + + rv := constituents[0] + if rv.ScoreBreakdown == nil { + rv.ScoreBreakdown = make(map[int]float64, len(constituents)) + } + var childrenExplanations []*search.Explanation + if s.options.Explain { + // since we want to notify which expl belongs to which matched searcher within the disjunction searcher + childrenExplanations = make([]*search.Explanation, countTotal) + } + + for i, docMatch := range constituents { + var index int + if originalPositions != nil { + // scorer used in disjunction slice searcher + index = originalPositions[matchingIdxs[i]] + } else { + // scorer used in disjunction heap searcher + index = matchingIdxs[i] + } + rv.ScoreBreakdown[index] = docMatch.Score + if s.options.Explain { + childrenExplanations[index] = docMatch.Expl + } + } + var explBreakdown *search.Explanation + if s.options.Explain { + explBreakdown = &search.Explanation{Children: childrenExplanations} + } + rv.Expl = explBreakdown + rv.FieldTermLocations = search.MergeFieldTermLocations( + rv.FieldTermLocations, constituents[1:]) + return rv +} diff --git a/search/scorer/scorer_knn.go b/search/scorer/scorer_knn.go new file mode 100644 index 0000000..06f50cd --- /dev/null +++ b/search/scorer/scorer_knn.go @@ -0,0 +1,157 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package scorer + +import ( + "fmt" + "math" + "reflect" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeKNNQueryScorer int + +func init() { + var sqs KNNQueryScorer + reflectStaticSizeKNNQueryScorer = int(reflect.TypeOf(sqs).Size()) +} + +type KNNQueryScorer struct { + queryVector []float32 + queryField string + queryWeight float64 + queryBoost float64 + queryNorm float64 + options search.SearcherOptions + similarityMetric string + queryWeightExplanation *search.Explanation +} + +func (s *KNNQueryScorer) Size() int { + sizeInBytes := reflectStaticSizeKNNQueryScorer + size.SizeOfPtr + + (len(s.queryVector) * size.SizeOfFloat32) + len(s.queryField) + + len(s.similarityMetric) + + if s.queryWeightExplanation != nil { + sizeInBytes += s.queryWeightExplanation.Size() + } + + return sizeInBytes +} + +func NewKNNQueryScorer(queryVector []float32, queryField string, queryBoost float64, + options search.SearcherOptions, + similarityMetric string) *KNNQueryScorer { + return &KNNQueryScorer{ + queryVector: queryVector, + queryField: queryField, + queryBoost: queryBoost, + queryWeight: 1.0, + options: options, + similarityMetric: similarityMetric, + } +} + +// Score used when the knnMatch.Score = 0 -> +// the query and indexed vector are exactly the same. +const maxKNNScore = math.MaxFloat32 + +func (sqs *KNNQueryScorer) Score(ctx *search.SearchContext, + knnMatch *index.VectorDoc) *search.DocumentMatch { + rv := ctx.DocumentMatchPool.Get() + var scoreExplanation *search.Explanation + score := knnMatch.Score + if sqs.similarityMetric == index.EuclideanDistance { + // in case of euclidean distance being the distance metric, + // an exact vector (perfect match), would return distance = 0 + if score == 0 { + score = maxKNNScore + } else { + // euclidean distances need to be inverted to work with + // tf-idf scoring + score = 1.0 / score + } + } + if sqs.options.Explain { + scoreExplanation = &search.Explanation{ + Value: score, + Message: fmt.Sprintf("fieldWeight(%s in doc %s), score of:", + sqs.queryField, knnMatch.ID), + Children: []*search.Explanation{ + { + Value: score, + Message: fmt.Sprintf("vector(field(%s:%s) with similarity_metric(%s)=%e", + sqs.queryField, knnMatch.ID, sqs.similarityMetric, score), + }, + }, + } + } + // if the query weight isn't 1, multiply + if sqs.queryWeight != 1.0 && score != maxKNNScore { + score = score * sqs.queryWeight + if sqs.options.Explain { + scoreExplanation = &search.Explanation{ + Value: score, + // Product of score * weight + // Avoid adding the query vector to the explanation since vectors + // can get quite large. + Message: fmt.Sprintf("weight(%s:query Vector^%f in %s), product of:", + sqs.queryField, sqs.queryBoost, knnMatch.ID), + Children: []*search.Explanation{sqs.queryWeightExplanation, scoreExplanation}, + } + } + } + rv.Score = score + if sqs.options.Explain { + rv.Expl = scoreExplanation + } + rv.IndexInternalID = index.NewIndexInternalIDFrom(rv.IndexInternalID, knnMatch.ID) + return rv +} + +func (sqs *KNNQueryScorer) Weight() float64 { + return 1.0 +} + +func (sqs *KNNQueryScorer) SetQueryNorm(qnorm float64) { + sqs.queryNorm = qnorm + + // update the query weight + sqs.queryWeight = sqs.queryBoost * sqs.queryNorm + + if sqs.options.Explain { + childrenExplanations := make([]*search.Explanation, 2) + childrenExplanations[0] = &search.Explanation{ + Value: sqs.queryBoost, + Message: "boost", + } + childrenExplanations[1] = &search.Explanation{ + Value: sqs.queryNorm, + Message: "queryNorm", + } + sqs.queryWeightExplanation = &search.Explanation{ + Value: sqs.queryWeight, + Message: fmt.Sprintf("queryWeight(%s:query Vector^%f), product of:", + sqs.queryField, sqs.queryBoost), + Children: childrenExplanations, + } + } +} diff --git a/search/scorer/scorer_knn_test.go b/search/scorer/scorer_knn_test.go new file mode 100644 index 0000000..d04edc3 --- /dev/null +++ b/search/scorer/scorer_knn_test.go @@ -0,0 +1,181 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package scorer + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func TestKNNScorerExplanation(t *testing.T) { + var queryVector []float32 + // arbitrary vector of dims: 64 + for i := 0; i < 64; i++ { + queryVector = append(queryVector, float32(i)) + } + + var resVector []float32 + // arbitrary res vector. + for i := 0; i < 64; i++ { + resVector = append(resVector, float32(i)) + } + + tests := []struct { + vectorMatch *index.VectorDoc + scorer *KNNQueryScorer + norm float64 + result *search.DocumentMatch + }{ + { + vectorMatch: &index.VectorDoc{ + ID: index.IndexInternalID("one"), + Score: 0.5, + Vector: resVector, + }, + norm: 1.0, + scorer: NewKNNQueryScorer(queryVector, "desc", 1.0, + search.SearcherOptions{Explain: true}, index.EuclideanDistance), + // Specifically testing EuclideanDistance since that involves score inversion. + result: &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("one"), + Score: 0.5, + Expl: &search.Explanation{ + Value: 1 / 0.5, + Message: "fieldWeight(desc in doc one), score of:", + Children: []*search.Explanation{ + { + Value: 1 / 0.5, + Message: "vector(field(desc:one) with similarity_metric(l2_norm)=2.000000e+00", + }, + }, + }, + }, + }, + { + vectorMatch: &index.VectorDoc{ + ID: index.IndexInternalID("one"), + Score: 0.0, + // Result vector is an exact match of an existing vector. + Vector: queryVector, + }, + norm: 1.0, + scorer: NewKNNQueryScorer(queryVector, "desc", 1.0, + search.SearcherOptions{Explain: true}, index.EuclideanDistance), + // Specifically testing EuclideanDistance with 0 score. + result: &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("one"), + Score: 0.0, + Expl: &search.Explanation{ + Value: maxKNNScore, + Message: "fieldWeight(desc in doc one), score of:", + Children: []*search.Explanation{ + { + Value: maxKNNScore, + Message: "vector(field(desc:one) with similarity_metric(l2_norm)=3.402823e+38", + }, + }, + }, + }, + }, + { + vectorMatch: &index.VectorDoc{ + ID: index.IndexInternalID("one"), + Score: 0.5, + Vector: resVector, + }, + norm: 1.0, + scorer: NewKNNQueryScorer(queryVector, "desc", 1.0, + search.SearcherOptions{Explain: true}, index.InnerProduct), + result: &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("one"), + Score: 0.5, + Expl: &search.Explanation{ + Value: 0.5, + Message: "fieldWeight(desc in doc one), score of:", + Children: []*search.Explanation{ + { + Value: 0.5, + Message: "vector(field(desc:one) with similarity_metric(dot_product)=5.000000e-01", + }, + }, + }, + }, + }, + { + vectorMatch: &index.VectorDoc{ + ID: index.IndexInternalID("one"), + Score: 0.25, + Vector: resVector, + }, + norm: 0.5, + scorer: NewKNNQueryScorer(queryVector, "desc", 1.0, + search.SearcherOptions{Explain: true}, index.InnerProduct), + result: &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("one"), + Score: 0.25, + Expl: &search.Explanation{ + Value: 0.125, + Message: "weight(desc:query Vector^1.000000 in one), product of:", + Children: []*search.Explanation{ + { + Value: 0.5, + Message: "queryWeight(desc:query Vector^1.000000), product of:", + Children: []*search.Explanation{ + { + Value: 1, + Message: "boost", + }, + { + Value: 0.5, + Message: "queryNorm", + }, + }, + }, + { + Value: 0.25, + Message: "fieldWeight(desc in doc one), score of:", + Children: []*search.Explanation{ + { + Value: 0.25, + Message: "vector(field(desc:one) with similarity_metric(dot_product)=2.500000e-01", + }, + }, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(1, 0), + } + test.scorer.SetQueryNorm(test.norm) + actual := test.scorer.Score(ctx, test.vectorMatch) + actual.Complete(nil) + + if !reflect.DeepEqual(actual.Expl, test.result.Expl) { + t.Errorf("expected %#v got %#v for %#v", test.result.Expl, + actual.Expl, test.vectorMatch) + } + } +} diff --git a/search/scorer/scorer_term.go b/search/scorer/scorer_term.go new file mode 100644 index 0000000..d7e77f9 --- /dev/null +++ b/search/scorer/scorer_term.go @@ -0,0 +1,276 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorer + +import ( + "fmt" + "math" + "reflect" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeTermQueryScorer int + +func init() { + var tqs TermQueryScorer + reflectStaticSizeTermQueryScorer = int(reflect.TypeOf(tqs).Size()) +} + +type TermQueryScorer struct { + queryTerm string + queryField string + queryBoost float64 + docTerm uint64 // number of documents containing the term + docTotal uint64 // total number of documents in the index + avgDocLength float64 + idf float64 + options search.SearcherOptions + idfExplanation *search.Explanation + includeScore bool + queryNorm float64 + queryWeight float64 + queryWeightExplanation *search.Explanation +} + +func (s *TermQueryScorer) Size() int { + sizeInBytes := reflectStaticSizeTermQueryScorer + size.SizeOfPtr + + len(s.queryTerm) + len(s.queryField) + + if s.idfExplanation != nil { + sizeInBytes += s.idfExplanation.Size() + } + + if s.queryWeightExplanation != nil { + sizeInBytes += s.queryWeightExplanation.Size() + } + + return sizeInBytes +} + +func (s *TermQueryScorer) computeIDF(avgDocLength float64, docTotal, docTerm uint64) float64 { + var rv float64 + if avgDocLength > 0 { + // avgDocLength is set only for bm25 scoring + rv = math.Log(1 + (float64(docTotal)-float64(docTerm)+0.5)/ + (float64(docTerm)+0.5)) + } else { + rv = 1.0 + math.Log(float64(docTotal)/ + float64(docTerm+1.0)) + } + + return rv +} + +// queryTerm - the specific term being scored by this scorer object +// queryField - the field in which the term is being searched +// queryBoost - the boost value for the query term +// docTotal - total number of documents in the index +// docTerm - number of documents containing the term +// avgDocLength - average document length in the index +// options - search options such as explain scoring, include the location of the term etc. +func NewTermQueryScorer(queryTerm []byte, queryField string, queryBoost float64, docTotal, + docTerm uint64, avgDocLength float64, options search.SearcherOptions) *TermQueryScorer { + + rv := TermQueryScorer{ + queryTerm: string(queryTerm), + queryField: queryField, + queryBoost: queryBoost, + docTerm: docTerm, + docTotal: docTotal, + avgDocLength: avgDocLength, + options: options, + queryWeight: 1.0, + includeScore: options.Score != "none", + } + + rv.idf = rv.computeIDF(avgDocLength, docTotal, docTerm) + if options.Explain { + rv.idfExplanation = &search.Explanation{ + Value: rv.idf, + Message: fmt.Sprintf("idf(docFreq=%d, maxDocs=%d)", docTerm, docTotal), + } + } + + return &rv +} + +func (s *TermQueryScorer) Weight() float64 { + sum := s.queryBoost * s.idf + return sum * sum +} + +func (s *TermQueryScorer) SetQueryNorm(qnorm float64) { + s.queryNorm = qnorm + + // update the query weight + s.queryWeight = s.queryBoost * s.idf * s.queryNorm + + if s.options.Explain { + childrenExplanations := make([]*search.Explanation, 3) + childrenExplanations[0] = &search.Explanation{ + Value: s.queryBoost, + Message: "boost", + } + childrenExplanations[1] = s.idfExplanation + childrenExplanations[2] = &search.Explanation{ + Value: s.queryNorm, + Message: "queryNorm", + } + s.queryWeightExplanation = &search.Explanation{ + Value: s.queryWeight, + Message: fmt.Sprintf("queryWeight(%s:%s^%f), product of:", s.queryField, s.queryTerm, s.queryBoost), + Children: childrenExplanations, + } + } +} + +func (s *TermQueryScorer) docScore(tf, norm float64) (score float64, model string) { + if s.avgDocLength > 0 { + // bm25 scoring + // using the posting's norm value to recompute the field length for the doc num + fieldLength := 1 / (norm * norm) + + score = s.idf * (tf * search.BM25_k1) / + (tf + search.BM25_k1*(1-search.BM25_b+(search.BM25_b*fieldLength/s.avgDocLength))) + model = index.BM25Scoring + } else { + // tf-idf scoring by default + score = tf * norm * s.idf + model = index.DefaultScoringModel + } + return score, model +} + +func (s *TermQueryScorer) scoreExplanation(tf float64, termMatch *index.TermFieldDoc) []*search.Explanation { + var rv []*search.Explanation + if s.avgDocLength > 0 { + fieldLength := 1 / (termMatch.Norm * termMatch.Norm) + fieldNormVal := 1 - search.BM25_b + (search.BM25_b * fieldLength / s.avgDocLength) + fieldNormalizeExplanation := &search.Explanation{ + Value: fieldNormVal, + Message: fmt.Sprintf("fieldNorm(field=%s), b=%f, fieldLength=%f, avgFieldLength=%f)", + s.queryField, search.BM25_b, fieldLength, s.avgDocLength), + } + + saturationExplanation := &search.Explanation{ + Value: search.BM25_k1 / (tf + search.BM25_k1*fieldNormVal), + Message: fmt.Sprintf("saturation(term:%s), k1=%f/(tf=%f + k1*fieldNorm=%f))", + termMatch.Term, search.BM25_k1, tf, fieldNormVal), + Children: []*search.Explanation{fieldNormalizeExplanation}, + } + + rv = make([]*search.Explanation, 3) + rv[0] = &search.Explanation{ + Value: tf, + Message: fmt.Sprintf("tf(termFreq(%s:%s)=%d", s.queryField, s.queryTerm, termMatch.Freq), + } + rv[1] = saturationExplanation + rv[2] = s.idfExplanation + } else { + rv = make([]*search.Explanation, 3) + rv[0] = &search.Explanation{ + Value: tf, + Message: fmt.Sprintf("tf(termFreq(%s:%s)=%d", s.queryField, s.queryTerm, termMatch.Freq), + } + rv[1] = &search.Explanation{ + Value: termMatch.Norm, + Message: fmt.Sprintf("fieldNorm(field=%s, doc=%s)", s.queryField, termMatch.ID), + } + rv[2] = s.idfExplanation + } + return rv +} + +func (s *TermQueryScorer) Score(ctx *search.SearchContext, termMatch *index.TermFieldDoc) *search.DocumentMatch { + rv := ctx.DocumentMatchPool.Get() + // perform any score computations only when needed + if s.includeScore || s.options.Explain { + var scoreExplanation *search.Explanation + var tf float64 + if termMatch.Freq < MaxSqrtCache { + tf = SqrtCache[int(termMatch.Freq)] + } else { + tf = math.Sqrt(float64(termMatch.Freq)) + } + + score, scoringModel := s.docScore(tf, termMatch.Norm) + if s.options.Explain { + childrenExplanations := s.scoreExplanation(tf, termMatch) + scoreExplanation = &search.Explanation{ + Value: score, + Message: fmt.Sprintf("fieldWeight(%s:%s in %s), as per %s model, "+ + "product of:", s.queryField, s.queryTerm, termMatch.ID, scoringModel), + Children: childrenExplanations, + } + } + + // if the query weight isn't 1, multiply + if s.queryWeight != 1.0 { + score = score * s.queryWeight + if s.options.Explain { + childExplanations := make([]*search.Explanation, 2) + childExplanations[0] = s.queryWeightExplanation + childExplanations[1] = scoreExplanation + scoreExplanation = &search.Explanation{ + Value: score, + Message: fmt.Sprintf("weight(%s:%s^%f in %s), product of:", s.queryField, s.queryTerm, s.queryBoost, termMatch.ID), + Children: childExplanations, + } + } + } + + if s.includeScore { + rv.Score = score + } + + if s.options.Explain { + rv.Expl = scoreExplanation + } + } + + rv.IndexInternalID = index.NewIndexInternalIDFrom(rv.IndexInternalID, termMatch.ID) + + if len(termMatch.Vectors) > 0 { + if cap(rv.FieldTermLocations) < len(termMatch.Vectors) { + rv.FieldTermLocations = make([]search.FieldTermLocation, 0, len(termMatch.Vectors)) + } + + for _, v := range termMatch.Vectors { + var ap search.ArrayPositions + if len(v.ArrayPositions) > 0 { + n := len(rv.FieldTermLocations) + if n < cap(rv.FieldTermLocations) { // reuse ap slice if available + ap = rv.FieldTermLocations[:n+1][n].Location.ArrayPositions[:0] + } + ap = append(ap, v.ArrayPositions...) + } + rv.FieldTermLocations = + append(rv.FieldTermLocations, search.FieldTermLocation{ + Field: v.Field, + Term: s.queryTerm, + Location: search.Location{ + Pos: v.Pos, + Start: v.Start, + End: v.End, + ArrayPositions: ap, + }, + }) + } + } + return rv +} diff --git a/search/scorer/scorer_term_test.go b/search/scorer/scorer_term_test.go new file mode 100644 index 0000000..0612861 --- /dev/null +++ b/search/scorer/scorer_term_test.go @@ -0,0 +1,260 @@ +// Copyright (c) 2013 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorer + +import ( + "math" + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func TestTermScorer(t *testing.T) { + + var docTotal uint64 = 100 + var docTerm uint64 = 9 + var queryTerm = []byte("beer") + var queryField = "desc" + var queryBoost = 1.0 + scorer := NewTermQueryScorer(queryTerm, queryField, queryBoost, docTotal, docTerm, 0, search.SearcherOptions{Explain: true}) + idf := 1.0 + math.Log(float64(docTotal)/float64(docTerm+1.0)) + + tests := []struct { + termMatch *index.TermFieldDoc + result *search.DocumentMatch + }{ + // test some simple math + { + termMatch: &index.TermFieldDoc{ + ID: index.IndexInternalID("one"), + Freq: 1, + Norm: 1.0, + Vectors: []*index.TermFieldVector{ + { + Field: "desc", + Pos: 1, + Start: 0, + End: 4, + }, + }, + }, + result: &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("one"), + Score: math.Sqrt(1.0) * idf, + Sort: []string{}, + Expl: &search.Explanation{ + Value: math.Sqrt(1.0) * idf, + Message: "fieldWeight(desc:beer in one), as per tf-idf model, product of:", + Children: []*search.Explanation{ + { + Value: 1, + Message: "tf(termFreq(desc:beer)=1", + }, + { + Value: 1, + Message: "fieldNorm(field=desc, doc=one)", + }, + { + Value: idf, + Message: "idf(docFreq=9, maxDocs=100)", + }, + }, + }, + Locations: search.FieldTermLocationMap{ + "desc": search.TermLocationMap{ + "beer": []*search.Location{ + { + Pos: 1, + Start: 0, + End: 4, + }, + }, + }, + }, + }, + }, + // test the same thing again (score should be cached this time) + { + termMatch: &index.TermFieldDoc{ + ID: index.IndexInternalID("one"), + Freq: 1, + Norm: 1.0, + }, + result: &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("one"), + Score: math.Sqrt(1.0) * idf, + Sort: []string{}, + Expl: &search.Explanation{ + Value: math.Sqrt(1.0) * idf, + Message: "fieldWeight(desc:beer in one), as per tf-idf model, product of:", + Children: []*search.Explanation{ + { + Value: 1, + Message: "tf(termFreq(desc:beer)=1", + }, + { + Value: 1, + Message: "fieldNorm(field=desc, doc=one)", + }, + { + Value: idf, + Message: "idf(docFreq=9, maxDocs=100)", + }, + }, + }, + }, + }, + // test a case where the sqrt isn't precalculated + { + termMatch: &index.TermFieldDoc{ + ID: index.IndexInternalID("one"), + Freq: 65, + Norm: 1.0, + }, + result: &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("one"), + Score: math.Sqrt(65) * idf, + Sort: []string{}, + Expl: &search.Explanation{ + Value: math.Sqrt(65) * idf, + Message: "fieldWeight(desc:beer in one), as per tf-idf model, product of:", + Children: []*search.Explanation{ + { + Value: math.Sqrt(65), + Message: "tf(termFreq(desc:beer)=65", + }, + { + Value: 1, + Message: "fieldNorm(field=desc, doc=one)", + }, + { + Value: idf, + Message: "idf(docFreq=9, maxDocs=100)", + }, + }, + }, + }, + }, + } + + for _, test := range tests { + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(1, 0), + } + actual := scorer.Score(ctx, test.termMatch) + actual.Complete(nil) + if len(actual.FieldTermLocations) == 0 { + actual.FieldTermLocations = nil + } + + if !reflect.DeepEqual(actual, test.result) { + t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) + } + } + +} + +func TestTermScorerWithQueryNorm(t *testing.T) { + + var docTotal uint64 = 100 + var docTerm uint64 = 9 + var queryTerm = []byte("beer") + var queryField = "desc" + var queryBoost = 3.0 + scorer := NewTermQueryScorer(queryTerm, queryField, queryBoost, docTotal, docTerm, 0, search.SearcherOptions{Explain: true}) + idf := 1.0 + math.Log(float64(docTotal)/float64(docTerm+1.0)) + + scorer.SetQueryNorm(2.0) + + expectedQueryWeight := 3 * idf * 3 * idf + actualQueryWeight := scorer.Weight() + if expectedQueryWeight != actualQueryWeight { + t.Errorf("expected query weight %f, got %f", expectedQueryWeight, actualQueryWeight) + } + + tests := []struct { + termMatch *index.TermFieldDoc + result *search.DocumentMatch + }{ + { + termMatch: &index.TermFieldDoc{ + ID: index.IndexInternalID("one"), + Freq: 1, + Norm: 1.0, + }, + result: &search.DocumentMatch{ + IndexInternalID: index.IndexInternalID("one"), + Score: math.Sqrt(1.0) * idf * 3.0 * idf * 2.0, + Sort: []string{}, + Expl: &search.Explanation{ + Value: math.Sqrt(1.0) * idf * 3.0 * idf * 2.0, + Message: "weight(desc:beer^3.000000 in one), product of:", + Children: []*search.Explanation{ + { + Value: 2.0 * idf * 3.0, + Message: "queryWeight(desc:beer^3.000000), product of:", + Children: []*search.Explanation{ + { + Value: 3, + Message: "boost", + }, + { + Value: idf, + Message: "idf(docFreq=9, maxDocs=100)", + }, + { + Value: 2, + Message: "queryNorm", + }, + }, + }, + { + Value: math.Sqrt(1.0) * idf, + Message: "fieldWeight(desc:beer in one), as per tf-idf model, product of:", + Children: []*search.Explanation{ + { + Value: 1, + Message: "tf(termFreq(desc:beer)=1", + }, + { + Value: 1, + Message: "fieldNorm(field=desc, doc=one)", + }, + { + Value: idf, + Message: "idf(docFreq=9, maxDocs=100)", + }, + }, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(1, 0), + } + actual := scorer.Score(ctx, test.termMatch) + + if !reflect.DeepEqual(actual, test.result) { + t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) + } + } + +} diff --git a/search/scorer/sqrt_cache.go b/search/scorer/sqrt_cache.go new file mode 100644 index 0000000..e26d33d --- /dev/null +++ b/search/scorer/sqrt_cache.go @@ -0,0 +1,30 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorer + +import ( + "math" +) + +var SqrtCache []float64 + +const MaxSqrtCache = 64 + +func init() { + SqrtCache = make([]float64, MaxSqrtCache) + for i := 0; i < MaxSqrtCache; i++ { + SqrtCache[i] = math.Sqrt(float64(i)) + } +} diff --git a/search/search.go b/search/search.go new file mode 100644 index 0000000..8088c11 --- /dev/null +++ b/search/search.go @@ -0,0 +1,448 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import ( + "fmt" + "reflect" + "slices" + "sort" + + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var ( + reflectStaticSizeDocumentMatch int + reflectStaticSizeSearchContext int + reflectStaticSizeLocation int +) + +func init() { + var dm DocumentMatch + reflectStaticSizeDocumentMatch = int(reflect.TypeOf(dm).Size()) + var sc SearchContext + reflectStaticSizeSearchContext = int(reflect.TypeOf(sc).Size()) + var l Location + reflectStaticSizeLocation = int(reflect.TypeOf(l).Size()) +} + +type ArrayPositions []uint64 + +func (ap ArrayPositions) Equals(other ArrayPositions) bool { + return slices.Equal(ap, other) +} + +func (ap ArrayPositions) Compare(other ArrayPositions) int { + for i, p := range ap { + if i >= len(other) { + return 1 + } + if p < other[i] { + return -1 + } + if p > other[i] { + return 1 + } + } + if len(ap) < len(other) { + return -1 + } + return 0 +} + +type Location struct { + // Pos is the position of the term within the field, starting at 1 + Pos uint64 `json:"pos"` + + // Start and End are the byte offsets of the term in the field + Start uint64 `json:"start"` + End uint64 `json:"end"` + + // ArrayPositions contains the positions of the term within any elements. + ArrayPositions ArrayPositions `json:"array_positions"` +} + +func (l *Location) Size() int { + return reflectStaticSizeLocation + size.SizeOfPtr + + len(l.ArrayPositions)*size.SizeOfUint64 +} + +type Locations []*Location + +func (p Locations) Len() int { return len(p) } +func (p Locations) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +func (p Locations) Less(i, j int) bool { + c := p[i].ArrayPositions.Compare(p[j].ArrayPositions) + if c < 0 { + return true + } + if c > 0 { + return false + } + return p[i].Pos < p[j].Pos +} + +func (p Locations) Dedupe() Locations { // destructive! + if len(p) <= 1 { + return p + } + + sort.Sort(p) + + slow := 0 + + for _, pfast := range p { + pslow := p[slow] + if pslow.Pos == pfast.Pos && + pslow.Start == pfast.Start && + pslow.End == pfast.End && + pslow.ArrayPositions.Equals(pfast.ArrayPositions) { + continue // duplicate, so only move fast ahead + } + + slow++ + + p[slow] = pfast + } + + return p[:slow+1] +} + +type TermLocationMap map[string]Locations + +func (t TermLocationMap) AddLocation(term string, location *Location) { + t[term] = append(t[term], location) +} + +type FieldTermLocationMap map[string]TermLocationMap + +type FieldTermLocation struct { + Field string + Term string + Location Location +} + +type FieldFragmentMap map[string][]string + +type DocumentMatch struct { + Index string `json:"index,omitempty"` + ID string `json:"id"` + IndexInternalID index.IndexInternalID `json:"-"` + Score float64 `json:"score"` + Expl *Explanation `json:"explanation,omitempty"` + Locations FieldTermLocationMap `json:"locations,omitempty"` + Fragments FieldFragmentMap `json:"fragments,omitempty"` + Sort []string `json:"sort,omitempty"` + DecodedSort []string `json:"decoded_sort,omitempty"` + + // Fields contains the values for document fields listed in + // SearchRequest.Fields. Text fields are returned as strings, numeric + // fields as float64s and date fields as strings. + Fields map[string]interface{} `json:"fields,omitempty"` + + // used to maintain natural index order + HitNumber uint64 `json:"-"` + + // used to temporarily hold field term location information during + // search processing in an efficient, recycle-friendly manner, to + // be later incorporated into the Locations map when search + // results are completed + FieldTermLocations []FieldTermLocation `json:"-"` + + // used to indicate the sub-scores that combined to form the + // final score for this document match. This is only populated + // when the search request's query is a DisjunctionQuery. + // The map key is the index of the sub-query + // in the DisjunctionQuery. The map value is the + // sub-score for that sub-query. + ScoreBreakdown map[int]float64 `json:"score_breakdown,omitempty"` + + // internal variable used in PreSearch phase of search in alias + // to indicate the name of the index that this match came from. + // used in knn search. + // it is a stack of index names, the top of the stack is the name + // of the index that this match came from + // of the current alias view, used in alias of aliases scenario + IndexNames []string `json:"index_names,omitempty"` + + // Descendants holds the IDs of any child/descendant document that contributed + // to this root DocumentMatch. + Descendants []index.IndexInternalID `json:"-"` +} + +func (dm *DocumentMatch) AddFieldValue(name string, value interface{}) { + if dm.Fields == nil { + dm.Fields = make(map[string]interface{}) + } + existingVal, ok := dm.Fields[name] + if !ok { + dm.Fields[name] = value + return + } + + valSlice, ok := existingVal.([]interface{}) + if ok { + // already a slice, append to it + valSlice = append(valSlice, value) + } else { + // create a slice + valSlice = []interface{}{existingVal, value} + } + dm.Fields[name] = valSlice +} + +func (dm *DocumentMatch) AddFragments(field string, fragments []string) { + if dm.Fragments == nil { + dm.Fragments = make(FieldFragmentMap) + } +OUTER: + for _, newFrag := range fragments { + for _, existingFrag := range dm.Fragments[field] { + if existingFrag == newFrag { + continue OUTER // no duplicates allowed + } + } + dm.Fragments[field] = append(dm.Fragments[field], newFrag) + } +} + +// Reset allows an already allocated DocumentMatch to be reused +func (dm *DocumentMatch) Reset() *DocumentMatch { + // Save backing arrays for reuse — these are restored below. + indexInternalID := dm.IndexInternalID + sortBuf := dm.Sort + decodedSort := dm.DecodedSort + ftls := dm.FieldTermLocations + for i := range ftls { // recycle the ArrayPositions of each location + ftls[i].Location.ArrayPositions = ftls[i].Location.ArrayPositions[:0] + } + scoreBreakdown := dm.ScoreBreakdown + // clear out the score breakdown map + if scoreBreakdown != nil { + clear(scoreBreakdown) + } + // remember the Descendants backing array + descendants := dm.Descendants + for i := range descendants { // recycle each IndexInternalID + descendants[i] = descendants[i][:0] + } + // Zero only the fields that are not restored below. + dm.Index = "" + dm.ID = "" + dm.Score = 0 + dm.Expl = nil + dm.Locations = nil + dm.Fragments = nil + dm.Fields = nil + dm.HitNumber = 0 + dm.IndexNames = nil + // Restore reusable allocations. + dm.IndexInternalID = indexInternalID[:0] + dm.Sort = sortBuf[:0] + dm.DecodedSort = decodedSort[:0] + dm.FieldTermLocations = ftls[:0] + dm.Descendants = descendants[:0] + dm.ScoreBreakdown = scoreBreakdown + return dm +} + +func (dm *DocumentMatch) Size() int { + sizeInBytes := reflectStaticSizeDocumentMatch + size.SizeOfPtr + + len(dm.Index) + + len(dm.ID) + + len(dm.IndexInternalID) + + if dm.Expl != nil { + sizeInBytes += dm.Expl.Size() + } + + for k, v := range dm.Locations { + sizeInBytes += size.SizeOfString + len(k) + for k1, v1 := range v { + sizeInBytes += size.SizeOfString + len(k1) + + size.SizeOfSlice + for _, entry := range v1 { + sizeInBytes += entry.Size() + } + } + } + + for k, v := range dm.Fragments { + sizeInBytes += size.SizeOfString + len(k) + + size.SizeOfSlice + + for _, entry := range v { + sizeInBytes += size.SizeOfString + len(entry) + } + } + + for _, entry := range dm.Sort { + sizeInBytes += size.SizeOfString + len(entry) + } + + for _, entry := range dm.DecodedSort { + sizeInBytes += size.SizeOfString + len(entry) + } + + for k := range dm.Fields { + sizeInBytes += size.SizeOfString + len(k) + + size.SizeOfPtr + } + + return sizeInBytes +} + +// Complete performs final preparation & transformation of the +// DocumentMatch at the end of search processing, also allowing the +// caller to provide an optional preallocated locations slice +func (dm *DocumentMatch) Complete(prealloc []Location) []Location { + // transform the FieldTermLocations slice into the Locations map + nlocs := len(dm.FieldTermLocations) + if nlocs > 0 { + if cap(prealloc) < nlocs { + prealloc = make([]Location, nlocs) + } + prealloc = prealloc[:nlocs] + + var lastField string + var tlm TermLocationMap + var needsDedupe bool + + for i, ftl := range dm.FieldTermLocations { + if i == 0 || lastField != ftl.Field { + lastField = ftl.Field + + if dm.Locations == nil { + dm.Locations = make(FieldTermLocationMap) + } + + tlm = dm.Locations[ftl.Field] + if tlm == nil { + tlm = make(TermLocationMap) + dm.Locations[ftl.Field] = tlm + } + } + + loc := &prealloc[i] + *loc = ftl.Location + + if len(loc.ArrayPositions) > 0 { // copy + loc.ArrayPositions = append(ArrayPositions(nil), loc.ArrayPositions...) + } + + locs := tlm[ftl.Term] + + // if the loc is before or at the last location, then there + // might be duplicates that need to be deduplicated + if !needsDedupe && len(locs) > 0 { + last := locs[len(locs)-1] + cmp := loc.ArrayPositions.Compare(last.ArrayPositions) + if cmp < 0 || (cmp == 0 && loc.Pos <= last.Pos) { + needsDedupe = true + } + } + + tlm[ftl.Term] = append(locs, loc) + + dm.FieldTermLocations[i] = FieldTermLocation{ // recycle + Location: Location{ + ArrayPositions: ftl.Location.ArrayPositions[:0], + }, + } + } + + if needsDedupe { + for _, tlm := range dm.Locations { + for term, locs := range tlm { + tlm[term] = locs.Dedupe() + } + } + } + } + + dm.FieldTermLocations = dm.FieldTermLocations[:0] // recycle + + return prealloc +} + +func (dm *DocumentMatch) String() string { + return fmt.Sprintf("[%s-%f]", dm.ID, dm.Score) +} + +type DocumentMatchCollection []*DocumentMatch + +func (c DocumentMatchCollection) Len() int { return len(c) } +func (c DocumentMatchCollection) Swap(i, j int) { c[i], c[j] = c[j], c[i] } +func (c DocumentMatchCollection) Less(i, j int) bool { return c[i].Score > c[j].Score } + +type Searcher interface { + Next(ctx *SearchContext) (*DocumentMatch, error) + Advance(ctx *SearchContext, ID index.IndexInternalID) (*DocumentMatch, error) + Close() error + Weight() float64 + SetQueryNorm(float64) + Count() uint64 + Min() int + Size() int + + DocumentMatchPoolSize() int +} + +type SearcherOptions struct { + Explain bool + IncludeTermVectors bool + Score string +} + +// SearchContext represents the context around a single search +type SearchContext struct { + DocumentMatchPool *DocumentMatchPool + Collector Collector + IndexReader index.IndexReader +} + +func (sc *SearchContext) Size() int { + sizeInBytes := reflectStaticSizeSearchContext + size.SizeOfPtr + + reflectStaticSizeDocumentMatchPool + size.SizeOfPtr + + if sc.DocumentMatchPool != nil { + for _, entry := range sc.DocumentMatchPool.avail { + if entry != nil { + sizeInBytes += entry.Size() + } + } + } + + return sizeInBytes +} + +// A NestedDocumentMatch is like a DocumentMatch but used for nested documents +// and does not have score or locations, or a score and is mainly used to +// hold field values and fragments, to be embedded in the parent DocumentMatch +type NestedDocumentMatch struct { + Fields map[string]interface{} `json:"fields,omitempty"` + Fragments FieldFragmentMap `json:"fragments,omitempty"` +} + +// NewNestedDocumentMatch creates a new NestedDocumentMatch instance +// with the given fields and fragments +func NewNestedDocumentMatch(fields map[string]interface{}, fragments FieldFragmentMap) *NestedDocumentMatch { + return &NestedDocumentMatch{ + Fields: fields, + Fragments: fragments, + } +} diff --git a/search/search_test.go b/search/search_test.go new file mode 100644 index 0000000..af81391 --- /dev/null +++ b/search/search_test.go @@ -0,0 +1,94 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestArrayPositionsCompare(t *testing.T) { + tests := []struct { + a []uint64 + b []uint64 + expect int + }{ + {nil, nil, 0}, + {[]uint64{}, []uint64{}, 0}, + {[]uint64{1}, []uint64{}, 1}, + {[]uint64{1}, []uint64{1}, 0}, + {[]uint64{}, []uint64{1}, -1}, + {[]uint64{0}, []uint64{1}, -1}, + {[]uint64{1}, []uint64{0}, 1}, + {[]uint64{1}, []uint64{1, 2}, -1}, + {[]uint64{1, 2}, []uint64{1}, 1}, + {[]uint64{1, 2}, []uint64{1, 2}, 0}, + {[]uint64{1, 2}, []uint64{1, 200}, -1}, + {[]uint64{1, 2}, []uint64{100, 2}, -1}, + {[]uint64{1, 2}, []uint64{1, 2, 3}, -1}, + } + + for _, test := range tests { + res := ArrayPositions(test.a).Compare(test.b) + if res != test.expect { + t.Errorf("test: %+v, res: %v", test, res) + } + } +} + +func TestLocationsDedupe(t *testing.T) { + a := &Location{} + b := &Location{Pos: 1} + c := &Location{Pos: 2} + + tests := []struct { + input Locations + expect Locations + }{ + {Locations{}, Locations{}}, + {Locations{a}, Locations{a}}, + {Locations{a, b, c}, Locations{a, b, c}}, + {Locations{a, a}, Locations{a}}, + {Locations{a, a, a}, Locations{a}}, + {Locations{a, b}, Locations{a, b}}, + {Locations{b, a}, Locations{a, b}}, + {Locations{c, b, a, c, b, a, c, b, a}, Locations{a, b, c}}, + } + + for testi, test := range tests { + res := test.input.Dedupe() + if !reflect.DeepEqual(res, test.expect) { + t.Errorf("testi: %d, test: %+v, res: %+v", testi, test, res) + } + } +} + +func TestMarshallingHighTerm(t *testing.T) { + highTermBytes, err := json.Marshal(HighTerm) + if err != nil { + t.Fatal(err) + } + + var unmarshalledHighTerm string + err = json.Unmarshal(highTermBytes, &unmarshalledHighTerm) + if err != nil { + t.Fatal(err) + } + + if unmarshalledHighTerm != HighTerm { + t.Fatalf("unexpected %x != %x", unmarshalledHighTerm, HighTerm) + } +} diff --git a/search/searcher/base_test.go b/search/searcher/base_test.go new file mode 100644 index 0000000..6f80bf6 --- /dev/null +++ b/search/searcher/base_test.go @@ -0,0 +1,117 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "math" + "regexp" + + "github.com/blevesearch/bleve/v2/analysis" + regexpTokenizer "github.com/blevesearch/bleve/v2/analysis/tokenizer/regexp" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/index/upsidedown" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap" + index "github.com/blevesearch/bleve_index_api" +) + +var twoDocIndex index.Index + +func init() { + twoDocIndex = initTwoDocUpsideDown() +} + +func initTwoDocUpsideDown() index.Index { + analysisQueue := index.NewAnalysisQueue(1) + twoDocIndex, err := upsidedown.NewUpsideDownCouch( + gtreap.Name, + map[string]interface{}{ + "path": "", + }, analysisQueue) + if err != nil { + panic(err) + } + initTwoDocs(twoDocIndex) + return twoDocIndex +} + +func initTwoDocScorch(dir string) index.Index { + analysisQueue := index.NewAnalysisQueue(1) + twoDocIndex, err := scorch.NewScorch( + scorch.Name, + map[string]interface{}{ + "path": dir, + }, analysisQueue) + if err != nil { + panic(err) + } + initTwoDocs(twoDocIndex) + return twoDocIndex +} + +func initTwoDocs(twoDocIndex index.Index) { + err := twoDocIndex.Open() + if err != nil { + panic(err) + } + batch := index.NewBatch() + for _, doc := range twoDocIndexDocs { + batch.Update(doc) + } + err = twoDocIndex.Batch(batch) + if err != nil { + panic(err) + } +} + +// create a simpler analyzer which will support these tests +var testAnalyzer = &analysis.DefaultAnalyzer{ + Tokenizer: regexpTokenizer.NewRegexpTokenizer(regexp.MustCompile(`\w+`)), +} + +// sets up some mock data used in many tests in this package +var twoDocIndexDescIndexingOptions = document.DefaultTextIndexingOptions | index.IncludeTermVectors + +var twoDocIndexDocs = []*document.Document{ + // must have 4/4 beer + document.NewDocument("1"). + AddField(document.NewTextField("name", []uint64{}, []byte("marty"))). + AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("beer beer beer beer"), twoDocIndexDescIndexingOptions, testAnalyzer)). + AddField(document.NewTextFieldWithAnalyzer("street", []uint64{}, []byte("couchbase way"), testAnalyzer)), + // must have 1/4 beer + document.NewDocument("2"). + AddField(document.NewTextField("name", []uint64{}, []byte("steve"))). + AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("angst beer couch database"), twoDocIndexDescIndexingOptions, testAnalyzer)). + AddField(document.NewTextFieldWithAnalyzer("street", []uint64{}, []byte("couchbase way"), testAnalyzer)). + AddField(document.NewTextFieldWithAnalyzer("title", []uint64{}, []byte("mister"), testAnalyzer)), + // must have 1/4 beer + document.NewDocument("3"). + AddField(document.NewTextField("name", []uint64{}, []byte("dustin"))). + AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("apple beer column dank"), twoDocIndexDescIndexingOptions, testAnalyzer)). + AddField(document.NewTextFieldWithAnalyzer("title", []uint64{}, []byte("mister"), testAnalyzer)), + // must have 65/65 beer + document.NewDocument("4"). + AddField(document.NewTextField("name", []uint64{}, []byte("ravi"))). + AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer"), twoDocIndexDescIndexingOptions, testAnalyzer)), + // must have 0/x beer + document.NewDocument("5"). + AddField(document.NewTextField("name", []uint64{}, []byte("bobert"))). + AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("water"), twoDocIndexDescIndexingOptions, testAnalyzer)). + AddField(document.NewTextFieldWithAnalyzer("title", []uint64{}, []byte("mister"), testAnalyzer)), +} + +func scoresCloseEnough(a, b float64) bool { + return math.Abs(a-b) < 0.001 +} diff --git a/search/searcher/geoshape_contains_test.go b/search/searcher/geoshape_contains_test.go new file mode 100644 index 0000000..e75bd37 --- /dev/null +++ b/search/searcher/geoshape_contains_test.go @@ -0,0 +1,1006 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/document" + index "github.com/blevesearch/bleve_index_api" +) + +var ( + leftRectEdgeMultiPoint [][]float64 = [][]float64{{-1, 0.2}, {-0.9, 0.1}} + leftRectWithHole [][][]float64 = [][][]float64{ + {{-1, 0}, {0, 0}, {0, 1}, {-1, 1}, {-1, 0}}, + {{-0.75, 0.25}, {-0.75, -0.75}, {-0.25, 0.75}, {-0.25, 0.25}, {-0.74, 0.25}}, + } + leftRectEdgePoint []float64 = []float64{-1, 0.2} + leftRectMultiPoint [][]float64 = [][]float64{{0.5, 0.5}, {-0.9, 0.1}} +) + +func testCaseSetup(t *testing.T, docShapeName, docShapeType string, docShapeVertices [][][][]float64, + i index.Index, +) (index.IndexReader, func() error, error) { + doc := document.NewDocument(docShapeName) + addGeoShapeFieldV1V2(doc, "geometry", []uint64{}, + docShapeVertices, docShapeType, document.DefaultGeoShapeIndexingOptions) + err := i.Update(doc) + if err != nil { + return nil, nil, err + } + indexReader, err := i.Reader() + if err != nil { + t.Fatal(err) + } + + closeFn := func() error { + err = i.Delete(doc.ID()) + if err != nil { + return err + } + err = indexReader.Close() + if err != nil { + return err + } + return nil + } + + return indexReader, closeFn, nil +} + +func TestPointPolygonContains(t *testing.T) { + tests := []struct { + QueryShape []float64 + DocShapeVertices [][][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: rightRectPoint, + DocShapeVertices: rightRect, + DocShapeName: "polygon1", + Expected: []string{"polygon1"}, + Desc: "point inside polygon", + QueryType: "contains", + }, + { + QueryShape: leftRectPoint, + DocShapeVertices: nil, + DocShapeName: "", + Expected: nil, + Desc: "empty polygon", + QueryType: "contains", + }, + } + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "polygon", [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePointRelationQuery(test.QueryType, false, indexReader, [][]float64{test.QueryShape}, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for point: %+v", test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestLinestringPolygonContains(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices [][][]float64 + DocShapeName string + Desc string + Expected []string + QueryType string + }{ + { + QueryShape: [][]float64{{1, 2}, {3, 5}}, + DocShapeVertices: [][][]float64{{{1, 2}, {3, 5}, {2, 7}, {1, 2}}}, + DocShapeName: "polygon1", + Desc: "linestring coinciding with edge of the polygon", + Expected: []string{"polygon1"}, + QueryType: "contains", + }, + { + QueryShape: [][]float64{{1, 0}, {0, 1}}, + DocShapeVertices: rightRect, + DocShapeName: "polygon1", + Desc: "diagonal of a square", + Expected: []string{"polygon1"}, + QueryType: "contains", + }, + { + QueryShape: [][]float64{{0.2, 0.2}, {0.8, 0.8}}, + DocShapeVertices: rightRect, + DocShapeName: "polygon1", + Desc: "linestring within polygon", + Expected: []string{"polygon1"}, + QueryType: "contains", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "polygon", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeLinestringQueryWithRelation(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for linestring: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestEnvelopePointContains(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices []float64 + DocShapeName string + Desc string + Expected []string + QueryType string + }{ + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: []float64{0.5, 0.5}, + DocShapeName: "point1", + Desc: "point completely within bounded rectangle", + Expected: nil, // will always be nil since point can't contain envelope + QueryType: "contains", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "point", + [][][][]float64{{{test.DocShapeVertices}}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeEnvelopeRelationQuery(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for Envelope: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestEnvelopeLinestringContains(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices [][]float64 + DocShapeName string + Desc string + Expected []string + QueryType string + }{ + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: [][]float64{{0.5, 0.5}, {10, 10}}, + DocShapeName: "linestring1", + Desc: "linestring partially within bounded rectangle", + Expected: nil, // will always be nil since linestring can't contain envelope + QueryType: "contains", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "linestring", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeEnvelopeRelationQuery(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for Envelope: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestEnvelopePolygonContains(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices [][][]float64 + DocShapeName string + Desc string + Expected []string + QueryType string + }{ + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: [][][]float64{{{0.5, 0.5}, {1, 0.5}, {1, 1}, {0.5, 1}, {0.5, 0.5}}}, + DocShapeName: "polygon1", + Desc: "polygon completely within bounded rectangle", + Expected: nil, + QueryType: "contains", + }, + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: [][][]float64{{{10.5, 10.5}, {11.5, 10.5}, {11.5, 11.5}, {10.5, 11.5}, {10.5, 10.5}}}, + DocShapeName: "polygon1", + Desc: "polygon completely outside bounded rectangle", + Expected: nil, + QueryType: "contains", + }, + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: rightRect, + DocShapeName: "polygon1", + Desc: "polygon coincident with bounded rectangle", + Expected: []string{"polygon1"}, + QueryType: "contains", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "polygon", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeEnvelopeRelationQuery(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for Envelope: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPolygonPointContains(t *testing.T) { + tests := []struct { + QueryShape [][][]float64 + DocShapeVertices []float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: rightRect, + DocShapeVertices: rightRectPoint, + DocShapeName: "point1", + Expected: nil, // nil since point is a non-closed shape + Desc: "point inside polygon", + QueryType: "contains", + }, + { + QueryShape: leftRect, + DocShapeVertices: leftRectEdgePoint, + DocShapeName: "point1", + Expected: nil, + Desc: "point on edge of polygon", + QueryType: "contains", + }, + { + QueryShape: leftRectWithHole, + DocShapeVertices: leftRectPoint, + DocShapeName: "point1", + Expected: nil, + Desc: "point in polygon's hole", + QueryType: "contains", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "point", + [][][][]float64{{{test.DocShapeVertices}}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePolygonQueryWithRelation(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for polygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPolygonLinestringContains(t *testing.T) { + tests := []struct { + QueryShape [][][]float64 + DocShapeVertices [][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: rightRect, + DocShapeVertices: [][]float64{{0, 1}, {1, 0}}, + DocShapeName: "linestring1", + Expected: nil, // nil since linestring is a non-closed shape + Desc: "diagonal of a square", + QueryType: "contains", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "linestring", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePolygonQueryWithRelation(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for polygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPolygonEnvelopeContains(t *testing.T) { + tests := []struct { + QueryShape [][][]float64 + DocShapeVertices [][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: [][][]float64{{{0.5, 0.5}, {1, 0.5}, {1, 1}, {0.5, 1}, {0.5, 0.5}}}, + DocShapeVertices: [][]float64{{0, 1}, {1, 0}}, + DocShapeName: "envelope1", + Expected: nil, + Desc: "polygon contained inside envelope with edge overlaps", // this fails since + // contains doesn't include edges or vertices + QueryType: "contains", + }, + { + QueryShape: [][][]float64{{{0.25, 0.25}, {0.5, 0.25}, {0.5, 0.5}, {0.25, 0.25}, {0.25, 0.25}}}, + DocShapeVertices: [][]float64{{0, 1}, {1, 0}}, + DocShapeName: "envelope1", + Expected: []string{"envelope1"}, + Desc: "polygon contained completely inside envelope", + QueryType: "contains", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "envelope", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePolygonQueryWithRelation(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for polygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestMultiPointPolygonContains(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices [][][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: leftRectEdgeMultiPoint, + DocShapeVertices: leftRectWithHole, + DocShapeName: "polygon1", + Expected: []string{"polygon1"}, + Desc: "multi point inside polygon with hole", + QueryType: "contains", + }, + { + QueryShape: [][]float64{{1, 0.5}}, + DocShapeVertices: rightRect, + DocShapeName: "polygon1", + Expected: nil, + Desc: "multi point on polygon edge", + QueryType: "contains", + }, + { + QueryShape: [][]float64{{0.3, 0.3}, {0.5, 0.5}}, + DocShapeVertices: [][][]float64{ + {{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}, + {{0.2, 0.2}, {0.4, 0.2}, {0.4, 0.4}, {0.2, 0.4}, {0.2, 0.2}}, + }, + DocShapeName: "polygon1", + Expected: nil, // returns nil since one of the points is within the hole + Desc: "multi point inside polygon and hole", + QueryType: "contains", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "polygon", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePointRelationQuery(test.QueryType, + true, indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for multipoint: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestMultiPointLinestringContains(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices [][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: leftRectEdgeMultiPoint, + DocShapeVertices: [][]float64{{-1, 0.2}, {-0.9, 0.1}}, + DocShapeName: "linestring1", + Expected: []string{"linestring1"}, + Desc: "multi point overlaps with all linestring end points", + QueryType: "contains", + }, + { + QueryShape: [][]float64{{-1, 0.2}, {-0.9, 0.1}, {0.5, 0.5}}, + DocShapeVertices: [][]float64{{-1, 0.2}, {-0.9, 0.1}}, + DocShapeName: "linestring1", + Expected: nil, + Desc: "multi point overlaps with some linestring end points", + QueryType: "contains", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "linestring", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePointRelationQuery(test.QueryType, + true, indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for multipoint: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestMultiPointContains(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices [][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: leftRectEdgeMultiPoint, + DocShapeVertices: [][]float64{{-1, 0.2}, {-0.9, 0.1}}, + DocShapeName: "multipoint1", + Expected: []string{"multipoint1"}, + Desc: "multi point overlaps with all multi points", + QueryType: "contains", + }, + { + QueryShape: [][]float64{{-1, 0.2}, {-0.9, 0.1}, {0.5, 0.5}}, + DocShapeVertices: [][]float64{{-1, 0.2}, {-0.9, 0.1}}, + DocShapeName: "multipoint1", + Expected: nil, + Desc: "multi point overlaps with some multi points", + QueryType: "contains", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "multipoint", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePointRelationQuery(test.QueryType, + true, indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for multipoint: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPolygonContains(t *testing.T) { + tests := []struct { + QueryShape [][][]float64 + DocShapeVertices [][][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: leftRect, + DocShapeVertices: rightRect, + DocShapeName: "polygon1", + Expected: nil, + Desc: "polygons sharing an edge", + QueryType: "contains", + }, + { + QueryShape: rightRect, + DocShapeVertices: rightRect, + DocShapeName: "polygon1", + Expected: []string{"polygon1"}, + Desc: "coincident polygons", + QueryType: "contains", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "polygon", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePolygonQueryWithRelation(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for polygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPolygonMultiPointContains(t *testing.T) { + tests := []struct { + QueryShape [][][]float64 + DocShapeVertices [][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: leftRect, + DocShapeVertices: leftRectEdgeMultiPoint, + DocShapeName: "multipoint1", + Expected: nil, // nil since multipoint is a non-closed shape + Desc: "multiple points on polygon edge", + QueryType: "contains", + }, + { + QueryShape: leftRect, + DocShapeVertices: leftRectMultiPoint, + DocShapeName: "multipoint1", + Expected: nil, + Desc: "multiple points, both outside and inside polygon", + QueryType: "contains", + }, + { + QueryShape: leftRectWithHole, + DocShapeVertices: leftRectMultiPoint, + DocShapeName: "multipoint1", + Expected: nil, + Desc: "multiple points in polygon hole", + QueryType: "contains", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "multipoint", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePolygonQueryWithRelation(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for polygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestMultiPolygonPolygonContains(t *testing.T) { + tests := []struct { + QueryShape [][][][]float64 + DocShapeVertices [][][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: [][][][]float64{leftRect}, + DocShapeVertices: leftRect, + DocShapeName: "polygon1", + Expected: []string{"polygon1"}, + Desc: "coincident polygons", + QueryType: "contains", + }, + { + QueryShape: [][][][]float64{{{{2, 2}, {-2, 2}, {-2, -2}, {2, -2}}}}, + DocShapeVertices: leftRect, + DocShapeName: "polygon1", + Expected: nil, + Desc: "polygon larger than polygons in query shape", + QueryType: "contains", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "polygon", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeMultiPolygonQueryWithRelation(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for polygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestMultiLinestringMultiPolygonContains(t *testing.T) { + tests := []struct { + QueryShape [][][]float64 + DocShapeVertices [][][][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: [][][]float64{{{0.2, 1}, {0.8, 1}}, {{1, 0.2}, {1, 0.8}}}, + DocShapeVertices: [][][][]float64{rightRect}, + DocShapeName: "multipolygon1", + Expected: nil, // contains doesn't include edges or vertices + Desc: "linestrings on edge of polygon", + QueryType: "contains", + }, + { + QueryShape: [][][]float64{{{0.2, 0.2}, {0.8, 0.8}}, {{0.8, 0.2}, {0.2, 0.8}}}, + DocShapeVertices: [][][][]float64{{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}}, + DocShapeName: "multipolygon1", + Expected: []string{"multipolygon1"}, + Desc: "linestrings within polygon", + QueryType: "contains", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "multipolygon", test.DocShapeVertices, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeMultiLinestringQueryWithRelation(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for multilinestring: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestGeometryCollectionPolygonContains(t *testing.T) { + tests := []struct { + QueryShape [][][][][]float64 + QueryShapeTypes []string + DocShapeVertices [][][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: [][][][][]float64{{{{{0, 1}, {1, 0}}}}}, + QueryShapeTypes: []string{"linestring"}, + DocShapeVertices: rightRect, + DocShapeName: "polygon1", + Expected: []string{"polygon1"}, + Desc: "linestring on edge of polygon", + QueryType: "contains", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "polygon", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeGeometryCollectionRelationQuery(test.QueryType, + indexReader, test.QueryShape, test.QueryShapeTypes, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for polygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestGeometryCollectionMultiPolygonContains(t *testing.T) { + tests := []struct { + QueryShape [][][][][]float64 + QueryShapeTypes []string + DocShapeVertices [][][][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: [][][][][]float64{{{{{1, 1}}}}}, + QueryShapeTypes: []string{"point"}, + DocShapeVertices: [][][][]float64{rightRect, leftRect}, + DocShapeName: "multipolygon1", + Expected: []string{"multipolygon1"}, + Desc: "point on vertex of one of the polygons", + QueryType: "contains", + }, + { + // WIP - Adding a point (-0.5,-0.5) + QueryShape: [][][][][]float64{{{{{0.2, 0.4}, {0.2, 0.2}, {0.4, 0.2}, {0.4, 0.4}}}}}, + QueryShapeTypes: []string{"polygon"}, + DocShapeVertices: [][][][]float64{rightRect, leftRect}, + DocShapeName: "multipolygon1", + Expected: []string{"multipolygon1"}, + Desc: "polygon contained completely within multipolygons", + QueryType: "contains", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "multipolygon", test.DocShapeVertices, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeGeometryCollectionRelationQuery(test.QueryType, + indexReader, test.QueryShape, test.QueryShapeTypes, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for multipolygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} diff --git a/search/searcher/geoshape_intersects_test.go b/search/searcher/geoshape_intersects_test.go new file mode 100644 index 0000000..b03f03e --- /dev/null +++ b/search/searcher/geoshape_intersects_test.go @@ -0,0 +1,1785 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap" + index "github.com/blevesearch/bleve_index_api" +) + +func setupIndex(t *testing.T) index.Index { + analysisQueue := index.NewAnalysisQueue(1) + i, err := scorch.NewScorch( + gtreap.Name, + map[string]interface{}{ + "path": "", + }, + analysisQueue) + if err != nil { + t.Fatal(err) + } + err = i.Open() + if err != nil { + t.Fatal(err) + } + + return i +} + +func TestPointIntersects(t *testing.T) { + tests := []struct { + QueryShape []float64 + DocShapeVertices []float64 + DocShapeName string + Desc string + Expected []string + }{ + { + QueryShape: []float64{2.0, 2.0}, + DocShapeVertices: []float64{2.0, 2.0}, + DocShapeName: "point1", + Desc: "coincident points", + Expected: []string{"point1"}, + }, + { + QueryShape: []float64{2.0, 2.0}, + DocShapeVertices: []float64{2.0, 2.1}, + DocShapeName: "point2", + Desc: "non coincident points", + Expected: nil, + }, + } + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "point", + [][][][]float64{{{test.DocShapeVertices}}}, i) + if err != nil { + t.Error(err.Error()) + } + + // indexing and searching independently for each case. + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePointRelationQuery("intersects", + false, indexReader, [][]float64{test.QueryShape}, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for point: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPointMultiPointIntersects(t *testing.T) { + tests := []struct { + QueryShape []float64 + DocShapeVertices [][]float64 + DocShapeName string + Desc string + Expected []string + }{ + { + QueryShape: []float64{2.0, 2.0}, + DocShapeVertices: [][]float64{{2.0, 2.0}, {3.0, 2.0}}, + DocShapeName: "point1", + Desc: "point coincides with one point in multipoint", + Expected: []string{"point1"}, + }, + { + QueryShape: []float64{2.0, 2.0}, + DocShapeVertices: [][]float64{{2.0, 2.1}, {3.0, 3.1}}, + DocShapeName: "point2", + Desc: "non coincident points", + Expected: nil, + }, + } + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "point", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + // indexing and searching independently for each case. + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePointRelationQuery("intersects", + false, indexReader, [][]float64{test.QueryShape}, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for point: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPointLinestringIntersects(t *testing.T) { + tests := []struct { + QueryShape []float64 + DocShapeVertices [][]float64 + DocShapeName string + Desc string + Expected []string + }{ + { + QueryShape: []float64{4.0, 4.0}, + DocShapeVertices: [][]float64{{2.0, 2.0}, {3.0, 3.0}, {4.0, 4.0}}, + DocShapeName: "linestring1", + Desc: "point at the vertex of linestring", + Expected: []string{"linestring1"}, + }, + { + QueryShape: []float64{1.5, 1.5001714}, + DocShapeVertices: [][]float64{{0.0, 0.0}, {1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}}, + DocShapeName: "linestring1", + Desc: "point along linestring", + Expected: nil, // nil since point is said to intersect only when it matches any + // of the endpoints of the linestring + }, + { + QueryShape: []float64{1.5, 1.6001714}, + DocShapeVertices: [][]float64{{0.0, 0.0}, {1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}}, + DocShapeName: "linestring1", + Desc: "point outside linestring", + Expected: nil, + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "linestring", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePointRelationQuery("intersects", + false, indexReader, [][]float64{test.QueryShape}, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for point: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPointMultiLinestringIntersects(t *testing.T) { + tests := []struct { + QueryShape []float64 + DocShapeVertices [][][]float64 + DocShapeName string + Desc string + Expected []string + }{ + { + QueryShape: []float64{3.0, 3.0}, + DocShapeVertices: [][][]float64{{{2.0, 2.0}, {3.0, 3.0}, {4.0, 4.0}}}, + DocShapeName: "linestring1", + Desc: "point at the vertex of linestring", + Expected: []string{"linestring1"}, + }, + { + QueryShape: []float64{1.5, 1.5001714}, + DocShapeVertices: [][][]float64{{{0.0, 0.0}, {1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}}}, + DocShapeName: "linestring1", + Desc: "point along a linestring", + Expected: nil, // nil since point is said to intersect only when it matches any + // of the endpoints of any of the linestrings + }, + { + QueryShape: []float64{1.5, 1.6001714}, + DocShapeVertices: [][][]float64{{{0.0, 0.0}, {1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}}, {{1, 1.1}, {2, 2.1}, {3, 3.4}}}, + DocShapeName: "linestring1", + Desc: "point outside all linestrings", + Expected: nil, + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "multilinestring", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePointRelationQuery("intersects", + false, indexReader, [][]float64{test.QueryShape}, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for point: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPointPolygonIntersects(t *testing.T) { + tests := []struct { + QueryShape []float64 + DocShapeVertices [][][]float64 + DocShapeName string + Desc string + Expected []string + }{ + { + QueryShape: []float64{3.0, 3.0}, + DocShapeVertices: [][][]float64{{{2.0, 2.0}, {3.0, 3.0}, {1.0, 3.0}, {2.0, 2.0}}}, + DocShapeName: "polygon1", + Desc: "point on polygon vertex", + Expected: []string{"polygon1"}, + }, + { + QueryShape: []float64{1.5, 1.500714}, + DocShapeVertices: [][][]float64{{{1.0, 1.0}, {2.0, 2.0}, {0.0, 2.0}, {1.0, 1.0}}}, + DocShapeName: "polygon1", + Desc: "point on polygon edge", + Expected: []string{"polygon1"}, + }, + { + QueryShape: []float64{1.5, 1.9}, + DocShapeVertices: [][][]float64{{{1.0, 1.0}, {2.0, 2.0}, {0.0, 2.0}, {1.0, 1.0}}}, + DocShapeName: "polygon1", + Desc: "point inside polygon", + Expected: []string{"polygon1"}, + }, + { + QueryShape: []float64{0.3, 0.3}, + DocShapeVertices: [][][]float64{ + {{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}}, + {{0.2, 0.2}, {0.2, 0.4}, {0.4, 0.4}, {0.4, 0.2}, {0.2, 0.2}}, + }, + DocShapeName: "polygon1", + Desc: "point inside hole inside polygon", + Expected: nil, + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "polygon", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePointRelationQuery("intersects", + false, indexReader, [][]float64{test.QueryShape}, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for point: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPointMultiPolygonIntersects(t *testing.T) { + tests := []struct { + QueryShape []float64 + DocShapeVertices [][][][]float64 + DocShapeName string + Desc string + Expected []string + }{ + { + QueryShape: []float64{3.0, 3.0}, + DocShapeVertices: [][][][]float64{{{{2.0, 2.0}, {3.0, 3.0}, {1.0, 3.0}, {2.0, 2.0}}}}, + DocShapeName: "multipolygon1", + Desc: "point on a polygon vertex", + Expected: []string{"multipolygon1"}, + }, + { + QueryShape: []float64{1.5, 1.500714}, + DocShapeVertices: [][][][]float64{{{{1.0, 1.0}, {2.0, 2.0}, {0.0, 2.0}, {1.0, 1.0}}}}, + DocShapeName: "multipolygon1", + Desc: "point on polygon edge", + Expected: []string{"multipolygon1"}, + }, + { + QueryShape: []float64{1.5, 1.9}, + DocShapeVertices: [][][][]float64{ + {{{1.0, 1.0}, {2.0, 2.0}, {0.0, 2.0}, {1.0, 1.0}}}, + {{{1.5, 1.9}, {2.5, 2.9}, {0.5, 2.9}, {1.5, 1.9}}}, + }, + DocShapeName: "multipolygon1", + Desc: "point inside a polygon and on vertex of another polygon", + Expected: []string{"multipolygon1"}, + }, + { + QueryShape: []float64{0.3, 0.3}, + DocShapeVertices: [][][][]float64{{ + {{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}}, + {{0.2, 0.2}, {0.2, 0.4}, {0.4, 0.4}, {0.4, 0.2}, {0.2, 0.2}}, + }}, + DocShapeName: "multipolygon1", + Desc: "point inside hole inside one of the polygons", + Expected: nil, + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "multipolygon", test.DocShapeVertices, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePointRelationQuery("intersects", + false, indexReader, [][]float64{test.QueryShape}, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for point: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestEnvelopePointIntersects(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices []float64 + DocShapeName string + Desc string + Expected []string + QueryType string + }{ + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: rightRectPoint, + DocShapeName: "point1", + Desc: "point on vertex of bounded rectangle", + Expected: []string{"point1"}, + QueryType: "intersects", + }, + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: []float64{10, 10}, + DocShapeName: "point1", + Desc: "point outside bounded rectangle", + Expected: nil, + QueryType: "intersects", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "point", + [][][][]float64{{{test.DocShapeVertices}}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeEnvelopeRelationQuery(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for Envelope: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestEnvelopeLinestringIntersect(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices [][]float64 + DocShapeName string + Desc string + Expected []string + QueryType string + }{ + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: [][]float64{{0.25, 0.25}, {0.5, 0.5}}, + DocShapeName: "linestring1", + Desc: "linestring completely in bounded rectangle", + Expected: []string{"linestring1"}, + QueryType: "intersects", + }, + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: [][]float64{{2.5, 2.5}, {4.5, 4.5}}, + DocShapeName: "linestring1", + Desc: "linestring outside bounded rectangle", + Expected: nil, + QueryType: "intersects", + }, + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: [][]float64{{0.25, 0.25}, {4.5, 4.5}}, + DocShapeName: "linestring1", + Desc: "linestring partially in bounded rectangle", + Expected: []string{"linestring1"}, + QueryType: "intersects", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "linestring", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeEnvelopeRelationQuery(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for Envelope: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestEnvelopePolygonIntersect(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices [][][]float64 + DocShapeName string + Desc string + Expected []string + QueryType string + }{ + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: [][][]float64{{{0.5, 0.5}, {1.5, 0.5}, {1.5, 1.5}, {0.5, 1.5}, {0.5, 0.5}}}, + DocShapeName: "polygon1", + Desc: "polygon intersects bounded rectangle", + Expected: []string{"polygon1"}, + QueryType: "intersects", + }, + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: [][][]float64{{{10.5, 10.5}, {11.5, 10.5}, {11.5, 11.5}, {10.5, 11.5}, {10.5, 10.5}}}, + DocShapeName: "polygon1", + Desc: "polygon completely outside bounded rectangle", + Expected: nil, + QueryType: "intersects", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "polygon", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeEnvelopeRelationQuery(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for Envelope: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestMultiPointIntersects(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices [][]float64 + DocShapeName string + Desc string + Expected []string + }{ + { + QueryShape: [][]float64{{3.0, 3.0}, {4.0, 4.0}}, + DocShapeVertices: [][]float64{{4.0, 4.0}}, + DocShapeName: "multipoint1", + Desc: "single coincident multipoint", + Expected: []string{"multipoint1"}, + }, + } + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "multipoint", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePointRelationQuery("intersects", + true, indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for multipoint: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestLinestringIntersects(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices [][]float64 + DocShapeName string + Desc string + Expected []string + }{ + { + QueryShape: [][]float64{{3.0, 2.0}, {4.0, 2.0}}, + DocShapeVertices: [][]float64{{3.0, 2.0}, {4.0, 2.0}}, + DocShapeName: "linestring1", + Desc: "coincident linestrings", + Expected: []string{"linestring1"}, + }, + { + QueryShape: [][]float64{{1.0, 1.0}, {1.5, 1.5}, {2.0, 2.0}}, + DocShapeVertices: [][]float64{{2.0, 2.0}, {4.0, 3.0}}, + DocShapeName: "linestring1", + Desc: "linestrings intersecting at the ends", + Expected: []string{"linestring1"}, + }, + { + QueryShape: [][]float64{{1.0, 1.0}, {3.0, 3.0}}, + DocShapeVertices: [][]float64{{1.5499860, 1.5501575}, {4.0, 6.0}}, + DocShapeName: "linestring1", + Desc: "subline not at vertex", + Expected: []string{"linestring1"}, + }, + { + QueryShape: [][]float64{{1.0, 1.0}, {2.0, 2.0}}, + DocShapeVertices: [][]float64{{1.5499860, 1.5501575}, {1.5, 1.5001714}}, + DocShapeName: "linestring1", + Desc: "subline inside linestring", + Expected: []string{"linestring1"}, + }, + { + QueryShape: [][]float64{{1.0, 1.0}, {1.5, 1.5}, {2.0, 2.0}}, + DocShapeVertices: [][]float64{{1.0, 2.0}, {2.0, 1.0}}, + DocShapeName: "linestring1", + Desc: "linestrings intersecting at some edge", + Expected: []string{"linestring1"}, + }, + { + QueryShape: [][]float64{{1.0, 1.0}, {1.5, 1.5}, {2.0, 2.0}}, + DocShapeVertices: [][]float64{{1.0, 2.0}, {1.0, 4.0}}, + DocShapeName: "linestring1", + Desc: "non intersecting linestrings", + Expected: nil, + }, + { + QueryShape: [][]float64{{59.32, 0.52}, {68.99, -7.36}, {75.49, -12.21}}, + DocShapeVertices: [][]float64{{71.98, 0}, {67.58, -6.57}, {63.19, -12.72}}, + DocShapeName: "linestring1", + Desc: "linestrings with more than 2 points intersecting at some edges", + Expected: []string{"linestring1"}, + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "linestring", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeLinestringQueryWithRelation("intersects", + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for polygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestLinestringPolygonIntersects(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices [][][]float64 + DocShapeName string + Desc string + Expected []string + }{ + { + QueryShape: [][]float64{{1.0, 1.0}, {1.5, 1.5}, {2.0, 2.0}}, + DocShapeVertices: [][][]float64{{{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}}}, + DocShapeName: "polygon1", + Desc: "linestring intersects polygon at a vertex", + Expected: []string{"polygon1"}, + }, + { + QueryShape: [][]float64{{0.2, 0.2}, {0.4, 0.4}}, + DocShapeVertices: [][][]float64{{{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}}}, + DocShapeName: "polygon1", + Desc: "linestring within polygon", + Expected: []string{"polygon1"}, + }, + { + QueryShape: [][]float64{{-0.5, 0.5}, {0.5, 0.5}}, + DocShapeVertices: [][][]float64{{{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}}}, + DocShapeName: "polygon1", + Desc: "linestring intersects polygon at an edge", + Expected: []string{"polygon1"}, + }, + { + QueryShape: [][]float64{{-0.5, 0.5}, {1.5, 0.5}}, + DocShapeVertices: [][][]float64{{{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}}}, + DocShapeName: "polygon1", + Desc: "linestring intersects polygon as a whole", + Expected: []string{"polygon1"}, + }, + { + QueryShape: [][]float64{{-0.5, 0.5}, {-1.5, -1.5}}, + DocShapeVertices: [][][]float64{{{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}}}, + DocShapeName: "polygon1", + Desc: "linestring does not intersect polygon", + Expected: nil, + }, + { + QueryShape: [][]float64{{0.3, 0.3}, {0.35, 0.35}}, + DocShapeVertices: [][][]float64{ + {{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}}, + {{0.2, 0.2}, {0.2, 0.4}, {0.4, 0.4}, {0.4, 0.2}, {0.2, 0.2}}, + }, + DocShapeName: "polygon1", + Desc: "linestring does not intersect polygon when contained in the hole", + Expected: nil, + }, + { + QueryShape: [][]float64{{0.3, 0.3}, {0.5, 0.5}}, + DocShapeVertices: [][][]float64{ + {{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}}, + {{0.2, 0.2}, {0.2, 0.4}, {0.4, 0.4}, {0.4, 0.2}, {0.2, 0.2}}, + }, + DocShapeName: "polygon1", + Desc: "linestring intersects polygon in the hole", + Expected: []string{"polygon1"}, + }, + { + QueryShape: [][]float64{{0.4, 0.3}, {0.6, 0.3}}, + DocShapeVertices: [][][]float64{ + {{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}}, + {{0.3, 0.3}, {0.4, 0.2}, {0.5, 0.3}, {0.4, 0.4}, {0.3, 0.3}}, + {{0.5, 0.3}, {0.6, 0.2}, {0.7, 0.3}, {0.6, 0.4}, {0.5, 0.3}}, + }, + DocShapeName: "polygon1", + Desc: "linestring intersects polygon through touching holes", + Expected: []string{"polygon1"}, + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "polygon", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeLinestringQueryWithRelation("intersects", + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for linestring: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestLinestringPointIntersects(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices []float64 + DocShapeName string + Desc string + Expected []string + }{ + { + QueryShape: [][]float64{{179, 0}, {-179, 0}}, + DocShapeVertices: []float64{179.1, 0}, + DocShapeName: "point1", + Desc: "point across longitudinal boundary of linestring", + Expected: []string{"point1"}, + }, + { + QueryShape: [][]float64{{-179, 0}, {179, 0}}, + DocShapeVertices: []float64{179.1, 0}, + DocShapeName: "point1", + Desc: "point across longitudinal boundary of reversed linestring", + Expected: []string{"point1"}, + }, + { + QueryShape: [][]float64{{179, 0}, {-179, 0}}, + DocShapeVertices: []float64{170, 0}, + DocShapeName: "point1", + Desc: "point does not intersect linestring", + Expected: nil, + }, + { + QueryShape: [][]float64{{-179, 0}, {179, 0}}, + DocShapeVertices: []float64{170, 0}, + DocShapeName: "point1", + Desc: "point does not intersect reversed linestring", + Expected: nil, + }, + { + QueryShape: [][]float64{{-179, 0}, {179, 0}, {178, 0}}, + DocShapeVertices: []float64{178, 0}, + DocShapeName: "point1", + Desc: "point intersects linestring at end vertex", + Expected: []string{"point1"}, + }, + { + QueryShape: [][]float64{{-179, 0}, {179, 0}, {178, 0}, {180, 0}}, + DocShapeVertices: []float64{178, 0}, + DocShapeName: "point1", + Desc: "point intersects linestring with more than two points", + Expected: []string{"point1"}, + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "point", + [][][][]float64{{{test.DocShapeVertices}}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeLinestringQueryWithRelation("intersects", + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for linestring: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestMultiLinestringIntersects(t *testing.T) { + tests := []struct { + QueryShape [][][]float64 + DocShapeVertices [][][]float64 + DocShapeName string + Desc string + Expected []string + }{ + { + QueryShape: [][][]float64{{{1.0, 1.0}, {1.1, 1.1}, {2.0, 2.0}, {2.1, 2.1}}}, + DocShapeVertices: [][][]float64{{{0.0, 0.5132}, {-1.1, -1.1}, {1.5, 1.512}, {2.1, 2.1}}}, + DocShapeName: "multilinestring1", + Desc: "intersecting multilinestrings", + Expected: []string{"multilinestring1"}, + }, + { + QueryShape: [][][]float64{{{1.0, 1.0}, {1.1, 1.1}, {2.0, 2.0}, {2.1, 2.1}}}, + DocShapeVertices: [][][]float64{{{10.1, 100.5}, {11.5, 102.5}}}, + DocShapeName: "multilinestring1", + Desc: "non-intersecting multilinestrings", + Expected: nil, + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "multilinestring", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeMultiLinestringQueryWithRelation("intersects", + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for multilinestring: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestMultiLinestringMultiPointIntersects(t *testing.T) { + tests := []struct { + QueryShape [][][]float64 + DocShapeVertices [][]float64 + DocShapeName string + Desc string + Expected []string + }{ + { + QueryShape: [][][]float64{{{2.0, 2.0}, {2.1, 2.1}}, {{3.0, 3.0}, {3.1, 3.1}}}, + DocShapeVertices: [][]float64{{5.0, 6.0}, {67, 67}, {3.1, 3.1}}, + DocShapeName: "multipoint1", + Desc: "multilinestring intersects one of the multipoints", + Expected: []string{"multipoint1"}, + }, + { + QueryShape: [][][]float64{{{2.0, 2.0}, {2.1, 2.1}}, {{3.0, 3.0}, {3.1, 3.1}}}, + DocShapeVertices: [][]float64{{56.0, 56.0}, {66, 66}}, + DocShapeName: "multipoint1", + Desc: "multilinestring does not intersect any of the multipoints", + Expected: nil, + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "multipoint", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeMultiLinestringQueryWithRelation("intersects", + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for polygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPolygonIntersects(t *testing.T) { + tests := []struct { + QueryShape [][][]float64 + DocShapeVertices [][][]float64 + DocShapeName string + Desc string + Expected []string + }{ + { + QueryShape: [][][]float64{{ + {1.0, 1.0}, + {2.0, 1.0}, + {2.0, 2.0}, + {1.0, 2.0}, + {1.0, 1.0}, + }}, + DocShapeVertices: [][][]float64{{ + {1.0, 1.0}, + {2.0, 1.0}, + {2.0, 2.0}, + {1.0, 2.0}, + {1.0, 1.0}, + }}, + DocShapeName: "polygon1", + Desc: "coincident polygons", + Expected: []string{"polygon1"}, + }, + { + QueryShape: [][][]float64{{ + {1.0, 1.0}, + {2.0, 1.0}, + {2.0, 2.0}, + {1.0, 2.0}, + {1.0, 1.0}, + }}, + DocShapeVertices: [][][]float64{{ + {1.2, 1.2}, + {2.0, 1.0}, + {2.0, 2.0}, + {1.0, 2.0}, + {1.2, 1.2}, + }}, + DocShapeName: "polygon1", + Desc: "polygon and a window polygon", + Expected: []string{"polygon1"}, + }, + { + QueryShape: [][][]float64{{ + {1.0, 1.0}, + {2.0, 1.0}, + {2.0, 2.0}, + {1.0, 2.0}, + {1.0, 1.0}, + }}, + DocShapeVertices: [][][]float64{{ + {1.1, 1.1}, + {1.2, 1.1}, + {1.2, 1.2}, + {1.1, 1.2}, + {1.1, 1.1}, + }}, + DocShapeName: "polygon1", + Desc: "nested polygons", + Expected: []string{"polygon1"}, + }, + { + QueryShape: [][][]float64{{ + {1.0, 1.0}, + {2.0, 1.0}, + {2.0, 2.0}, + {1.0, 2.0}, + {1.0, 1.0}, + }}, + DocShapeVertices: [][][]float64{{ + {0.0, 1.0}, + {2.0, 1.0}, + {2.0, 2.0}, + {0.0, 2.0}, + {0.0, 1.0}, + }}, + DocShapeName: "polygon1", + Desc: "intersecting polygons", + Expected: []string{"polygon1"}, + }, + { + QueryShape: [][][]float64{{{0, 0}, {5, 0}, {5, 5}, {0, 5}, {0, 0}}, { + {1, 4}, + {4, 4}, + {4, 1}, + {1, 1}, + {1, 4}, + }}, + DocShapeVertices: [][][]float64{{{2, 2}, {3, 2}, {3, 3}, {2, 3}, {2, 2}}}, + DocShapeName: "polygon1", + Desc: "polygon inside hole of a larger polygon", + Expected: nil, + }, + { + QueryShape: [][][]float64{{ + {1.0, 1.0}, + {2.0, 1.0}, + {2.0, 2.0}, + {1.0, 2.0}, + {1.0, 1.0}, + }}, + DocShapeVertices: [][][]float64{{ + {3.0, 3.0}, + {4.0, 3.0}, + {4.0, 4.0}, + {3.0, 4.0}, + {3.0, 3.0}, + }}, + DocShapeName: "polygon1", + Desc: "disjoint polygons", + Expected: nil, + }, + } + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "polygon", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePolygonQueryWithRelation("intersects", + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for polygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPolygonLinestringIntersects(t *testing.T) { + tests := []struct { + QueryShape [][][]float64 + DocShapeVertices [][]float64 + DocShapeName string + Desc string + Expected []string + }{ + { + QueryShape: [][][]float64{{{150, 85}, {160, 85}, {-20, 85}, {-30, 85}, {150, 85}}}, + DocShapeVertices: [][]float64{{150, 85}, {160, 85}}, + DocShapeName: "linestring1", + Desc: "polygon intersects line along edge", + Expected: []string{"linestring1"}, + }, + { + QueryShape: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + DocShapeVertices: [][]float64{{150, 85}, {160, 85}}, + DocShapeName: "linestring1", + Desc: "polygon not intersecting line", + Expected: nil, + }, + { + QueryShape: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + DocShapeVertices: [][]float64{{0.2, 0.2}, {0.4, 0.4}}, + DocShapeName: "linestring1", + Desc: "polygon completely encloses line", + Expected: []string{"linestring1"}, + }, + { + QueryShape: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + DocShapeVertices: [][]float64{{-0.5, 0.5}, {1.5, 0.5}}, + DocShapeName: "linestring1", + Desc: "line cuts through entire polygon", + Expected: []string{"linestring1"}, + }, + { + QueryShape: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + DocShapeVertices: [][]float64{{-0.439, -0.318}, {0.4339, 0.335}}, + DocShapeName: "linestring1", + Desc: "line partially cuts through polygon", + Expected: []string{"linestring1"}, + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "linestring", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePolygonQueryWithRelation("intersects", + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for polygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPolygonMultiLinestringIntersects(t *testing.T) { + tests := []struct { + QueryShape [][][]float64 + DocShapeVertices [][][]float64 + DocShapeName string + Desc string + Expected []string + }{ + { + QueryShape: [][][]float64{{{150, 85}, {160, 85}, {-20, 85}, {-30, 85}, {150, 85}}}, + DocShapeVertices: [][][]float64{{{150, 85}, {160, 85}}, {{0, 1}, {5, 10}}}, + DocShapeName: "multilinestring1", + Desc: "polygon intersects one line along edge", + Expected: []string{"multilinestring1"}, + }, + { + QueryShape: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + DocShapeVertices: [][][]float64{{{150, 85}, {160, 85}}}, + DocShapeName: "multilinestring1", + Desc: "polygon not intersecting any line", + Expected: nil, + }, + { + QueryShape: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + DocShapeVertices: [][][]float64{{{0.2, 0.2}, {0.4, 0.4}}}, + DocShapeName: "multilinestring1", + Desc: "polygon completely encloses line", + Expected: []string{"multilinestring1"}, + }, + { + QueryShape: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + DocShapeVertices: [][][]float64{{{-0.5, 0.5}, {1.5, 0.5}}}, + DocShapeName: "multilinestring1", + Desc: "line cuts through entire polygon", + Expected: []string{"multilinestring1"}, + }, + { + QueryShape: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + DocShapeVertices: [][][]float64{{{-0.439, -0.318}, {0.4339, 0.335}}}, + DocShapeName: "multilinestring1", + Desc: "line partially cuts through polygon", + Expected: []string{"multilinestring1"}, + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "multilinestring", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePolygonQueryWithRelation("intersects", + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for polygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPolygonPointIntersects(t *testing.T) { + tests := []struct { + QueryShape [][][]float64 + DocShapeVertices []float64 + DocShapeName string + Desc string + Expected []string + }{ + { + QueryShape: [][][]float64{{{150, 85}, {160, 85}, {-20, 85}, {-30, 85}, {150, 85}}}, + DocShapeVertices: []float64{150, 88}, + DocShapeName: "point1", + Desc: "polygon intersects point in latitudinal boundary", + Expected: []string{"point1"}, + }, + { + QueryShape: [][][]float64{{{150, 85}, {160, 85}, {-20, 85}, {-30, 85}, {150, 85}}}, + DocShapeVertices: []float64{170, 88}, + DocShapeName: "point1", + Desc: "polygon does not intersects point outside latitudinal boundary", + Expected: nil, + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "point", + [][][][]float64{{{test.DocShapeVertices}}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePolygonQueryWithRelation("intersects", + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for polygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestMultiPolygonIntersects(t *testing.T) { + tests := []struct { + QueryShape [][][][]float64 + DocShapeVertices [][][][]float64 + DocShapeName string + Desc string + Expected []string + }{ + { + QueryShape: [][][][]float64{{{ + {15, 5}, + {40, 10}, + {10, 20}, + {5, 10}, + {15, 5}, + }}, {{{30, 20}, {45, 40}, {10, 40}, {30, 20}}}}, + DocShapeVertices: [][][][]float64{{{ + {0.0, 0.0}, + {1.0, 0.0}, + {1.0, 1.0}, + {0.0, 1.0}, + {0.0, 0.0}, + }, {{30, 20}, {45, 40}, {10, 40}, {30, 20}}}}, + DocShapeName: "multipolygon1", + Desc: "intersecting multi polygons", + Expected: []string{"multipolygon1"}, + }, + { + QueryShape: [][][][]float64{{{ + {15, 5}, + {40, 10}, + {10, 20}, + {5, 10}, + {15, 5}, + }}, {{{30, 20}, {45, 40}, {10, 40}, {30, 20}}}}, + DocShapeVertices: [][][][]float64{{{ + {0.0, 0.0}, + {1.0, 0.0}, + {1.0, 1.0}, + {0.0, 1.0}, + {0.0, 0.0}, + }}}, + DocShapeName: "multipolygon1", + Desc: "non intersecting multi polygons", + Expected: nil, + }, + } + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "multipolygon", + test.DocShapeVertices, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeMultiPolygonQueryWithRelation("intersects", + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for multipolygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestMultiPolygonMultiPointIntersects(t *testing.T) { + tests := []struct { + QueryShape [][][][]float64 + DocShapeVertices [][]float64 + DocShapeName string + Desc string + Expected []string + }{ + { + QueryShape: [][][][]float64{ + {{{30, 20}, {45, 40}, {10, 40}, {30, 20}}}, + {{{15, 5}, {40, 10}, {10, 20}, {5, 10}, {15, 5}}}, + }, + DocShapeVertices: [][]float64{{30, 20}, {30, 30}}, + DocShapeName: "multipoint1", + Desc: "multipolygon intersects multipoint at the vertex", + Expected: []string{"multipoint1"}, + }, + { + QueryShape: [][][][]float64{ + {{{15, 5}, {40, 10}, {10, 20}, {5, 10}, {15, 5}}}, + {{{30, 20}, {45, 50}, {10, 50}, {30, 20}}}, + }, + DocShapeVertices: [][]float64{{30, -20}, {-30, 30}, {45, 66}}, + DocShapeName: "multipoint1", + Desc: "multipolygon does not intersect multipoint", + Expected: nil, + }, + } + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "multipoint", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeMultiPolygonQueryWithRelation("intersects", + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for multipolygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestMultiPolygonMultiLinestringIntersects(t *testing.T) { + tests := []struct { + QueryShape [][][][]float64 + DocShapeVertices [][][]float64 + DocShapeName string + Desc string + Expected []string + }{ + { + QueryShape: [][][][]float64{{{{15, 5}, {40, 10}, {10, 20}, {5, 10}, {15, 5}}}, {{{30, 20}, {45, 40}, {10, 40}, {30, 20}}}}, + DocShapeVertices: [][][]float64{{{65, 40}, {60, 40}}, {{45, 40}, {10, 40}, {30, 20}}}, + DocShapeName: "multilinestring1", + Desc: "multipolygon intersects multilinestring", + Expected: []string{"multilinestring1"}, + }, + { + QueryShape: [][][][]float64{{{{15, 5}, {40, 10}, {10, 20}, {5, 10}, {15, 5}}}, {{{30, 20}, {45, 40}, {10, 40}, {30, 20}}}}, + DocShapeVertices: [][][]float64{{{45, 41}, {60, 80}}, {{-45, -40}, {-10, -40}}}, + DocShapeName: "multilinestring1", + Desc: "multipolygon does not intersect multilinestring", + Expected: nil, + }, + } + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "multilinestring", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeMultiPolygonQueryWithRelation("intersects", + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for multipolygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestGeometryCollectionIntersects(t *testing.T) { + tests := []struct { + QueryShape [][][][][]float64 + DocShapeVertices [][][][][]float64 + DocShapeName string + Desc string + Expected []string + Types []string + }{ + { + QueryShape: [][][][][]float64{{{{}}}}, + DocShapeVertices: [][][][][]float64{{{{}}}}, + DocShapeName: "geometrycollection1", + Desc: "empty geometry collections", + Expected: nil, + Types: []string{""}, + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetupGeometryCollection(t, test.DocShapeName, test.Types, + test.DocShapeVertices, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeGeometryCollectionRelationQuery("intersects", + indexReader, test.QueryShape, test.Types, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for geometry collection: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestGeometryCollectionPointIntersects(t *testing.T) { + tests := []struct { + QueryShape [][][][][]float64 + DocShapeVertices []float64 + DocShapeName string + Desc string + Expected []string + Types []string + }{ + { + QueryShape: [][][][][]float64{{{{{4, 5}}}}}, + DocShapeVertices: []float64{4, 5}, + DocShapeName: "point1", + Desc: "point coincident with point in geometry collection", + Expected: []string{"point1"}, + Types: []string{"point"}, + }, + { + QueryShape: [][][][][]float64{{{{{4, 5}, {6, 7}}}}}, + DocShapeVertices: []float64{4, 5}, + DocShapeName: "point1", + Desc: "point on vertex of linestring in geometry collection", + Expected: []string{"point1"}, + Types: []string{"linestring"}, + }, + { + QueryShape: [][][][][]float64{{{{{1, 1}, {2, 2}, {0, 2}, {1, 1}}}, {{{5, 6}}}}}, + DocShapeVertices: []float64{1.5, 1.9}, + DocShapeName: "point1", + Desc: "point inside polygon in geometry collection", + Expected: []string{"point1"}, + Types: []string{"polygon", "point"}, + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "point", + [][][][]float64{{{test.DocShapeVertices}}}, i) + if err != nil { + t.Fatal(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeGeometryCollectionRelationQuery("intersects", + indexReader, test.QueryShape, test.Types, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for geometry collection: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestGeometryCollectionLinestringIntersects(t *testing.T) { + tests := []struct { + QueryShape [][][][][]float64 + DocShapeVertices [][]float64 + DocShapeName string + Desc string + Expected []string + Types []string + }{ + { + QueryShape: [][][][][]float64{{{{{4, 5}, {6, 7}, {7, 8}}}}}, + DocShapeVertices: [][]float64{{6, 7}, {7, 8}}, + DocShapeName: "linestring1", + Desc: "linestring intersecting with linestring in geometry collection", + Expected: []string{"linestring1"}, + Types: []string{"linestring"}, + }, + { + QueryShape: [][][][][]float64{{{{{1.5, 1.9}}}}}, + DocShapeVertices: [][]float64{{1.5, 1.9}, {2.5, 2.8}}, + DocShapeName: "linestring1", + Desc: "linestring intersects point in geometry collection at vertex", + Expected: []string{"linestring1"}, + Types: []string{"point"}, + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "linestring", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeGeometryCollectionRelationQuery("intersects", + indexReader, test.QueryShape, test.Types, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for geometry collection: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestGeometryCollectionPolygonIntersects(t *testing.T) { + tests := []struct { + QueryShape [][][][][]float64 + DocShapeVertices [][][]float64 + DocShapeName string + Desc string + Expected []string + Types []string + }{ + { + QueryShape: [][][][][]float64{{{{{4, 5}, {6, 7}, {7, 8}, {4, 5}}}, {{{1, 2}, {2, 3}, {3, 4}, {1, 2}}}}}, + DocShapeVertices: [][][]float64{{{4, 5}, {6, 7}, {7, 8}, {4, 5}}}, + DocShapeName: "polygon1", + Desc: "polygon coincides with one of the polygons in multipolygon in geometry collection", + Expected: []string{"polygon1"}, + Types: []string{"multipolygon"}, + }, + { + QueryShape: [][][][][]float64{{{{{14, 15}}}}}, + DocShapeVertices: [][][]float64{{{4, 5}, {6, 7}, {7, 8}, {4, 5}}}, + DocShapeName: "polygon1", + Desc: "polygon does not intersect point in geometry collection", + Expected: nil, + Types: []string{"point"}, + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "polygon", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeGeometryCollectionRelationQuery("intersects", + indexReader, test.QueryShape, test.Types, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for geometry collection: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPointGeometryCollectionIntersects(t *testing.T) { + tests := []struct { + QueryShape []float64 + DocShapeVertices [][][][][]float64 + DocShapeName string + Desc string + Expected []string + Types []string + }{ + { + QueryShape: []float64{1.0, 2.0}, + DocShapeVertices: [][][][][]float64{{{{}}}}, + DocShapeName: "geometrycollection1", + Desc: "geometry collection does not intersect with a point", + Expected: nil, + Types: []string{""}, + }, + { + QueryShape: []float64{1.0, 2.0}, + DocShapeVertices: [][][][][]float64{{{{{1.0, 2.0}}}}}, + DocShapeName: "geometrycollection1", + Desc: "geometry collection intersects with a point", + Expected: []string{"geometrycollection1"}, + Types: []string{"point"}, + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetupGeometryCollection(t, test.DocShapeName, test.Types, + test.DocShapeVertices, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePointRelationQuery("intersects", + false, indexReader, [][]float64{test.QueryShape}, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for point: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} diff --git a/search/searcher/geoshape_within_test.go b/search/searcher/geoshape_within_test.go new file mode 100644 index 0000000..7d40384 --- /dev/null +++ b/search/searcher/geoshape_within_test.go @@ -0,0 +1,1352 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "fmt" + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/document" + index "github.com/blevesearch/bleve_index_api" +) + +var ( + leftRect [][][]float64 = [][][]float64{{{-1, 0}, {0, 0}, {0, 1}, {-1, 1}, {-1, 0}}} + leftRectPoint []float64 = []float64{-0.5, 0.5} + rightRect [][][]float64 = [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}} + rightRectPoint []float64 = []float64{0.5, 0.5} +) + +func testCaseSetupGeometryCollection(t *testing.T, docShapeName string, types []string, docShapeVertices [][][][][]float64, + i index.Index, +) (index.IndexReader, func() error, error) { + doc := document.NewDocument(docShapeName) + gcField := document.NewGeometryCollectionFieldWithIndexingOptions("geometry", + []uint64{}, docShapeVertices, types, document.DefaultGeoShapeIndexingOptions) + if gcField == nil { + return nil, nil, fmt.Errorf("the GC field is nil") + } + doc.AddField(gcField) + addGeoCollectionV2Parallel(doc, "geometry", docShapeVertices, types) + if doc == nil { + return nil, nil, fmt.Errorf("the doc is nil") + } + err := i.Update(doc) + if err != nil { + t.Error(err.Error()) + } + + indexReader, err := i.Reader() + if err != nil { + t.Fatal(err) + } + + closeFn := func() error { + err = i.Delete(doc.ID()) + if err != nil { + return err + } + err = indexReader.Close() + if err != nil { + return err + } + return nil + } + + return indexReader, closeFn, nil +} + +func TestPointWithin(t *testing.T) { + tests := []struct { + QueryShape []float64 + DocShapeVertices []float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: []float64{1.0, 1.0}, + DocShapeVertices: []float64{1.0, 1.0}, + DocShapeName: "point1", + Expected: []string{"point1"}, + Desc: "point contains itself", + QueryType: "within", + }, + { + QueryShape: []float64{1.0, 1.0}, + DocShapeVertices: []float64{1.0, 1.1}, + DocShapeName: "point1", + Expected: nil, + Desc: "point does not contain a different point", + QueryType: "within", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "point", + [][][][]float64{{{test.DocShapeVertices}}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePointRelationQuery(test.QueryType, + false, indexReader, [][]float64{test.QueryShape}, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for point: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestMultiPointWithin(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices [][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: [][]float64{{1.0, 1.0}, {2.0, 2.0}}, + DocShapeVertices: [][]float64{{1.0, 1.0}}, + DocShapeName: "multipoint1", + Expected: []string{"multipoint1"}, + Desc: "single multipoint common", + QueryType: "within", + }, + { + QueryShape: [][]float64{{1.0, 1.0}}, + DocShapeVertices: [][]float64{{1.0, 1.0}, {2.0, 2.0}}, + DocShapeName: "multipoint1", + Expected: nil, + Desc: "multipoint not covered by multiple multipoints", + QueryType: "within", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "multipoint", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePointRelationQuery(test.QueryType, + true, indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for multipoint: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestEnvelopePointWithin(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices []float64 + DocShapeName string + Desc string + Expected []string + QueryType string + }{ + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: []float64{0.5, 0.5}, + DocShapeName: "point1", + Desc: "point completely within bounded rectangle", + Expected: []string{"point1"}, + QueryType: "within", + }, + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: []float64{0, 1}, + DocShapeName: "point1", + Desc: "point on vertex of bounded rectangle", + Expected: []string{"point1"}, + QueryType: "within", + }, + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: []float64{10, 11}, + DocShapeName: "point1", + Desc: "point outside bounded rectangle", + Expected: nil, + QueryType: "within", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "point", + [][][][]float64{{{test.DocShapeVertices}}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeEnvelopeRelationQuery(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for Envelope: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestEnvelopeLinestringWithin(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices [][]float64 + DocShapeName string + Desc string + Expected []string + QueryType string + }{ + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: [][]float64{{0.5, 0.5}, {0.75, 0.75}}, + DocShapeName: "linestring1", + Desc: "linestring completely within bounded rectangle", + Expected: []string{"linestring1"}, + QueryType: "within", + }, + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: [][]float64{{0.5, 0.5}, {1.75, 1.75}}, + DocShapeName: "linestring1", + Desc: "linestring partially within bounded rectangle", + Expected: nil, + QueryType: "within", + }, + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: [][]float64{{1.5, 2.5}, {2.75, 2.75}}, + DocShapeName: "linestring1", + Desc: "linestring completely outside bounded rectangle", + Expected: nil, + QueryType: "within", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "linestring", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeEnvelopeRelationQuery(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for Envelope: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestEnvelopePolygonWithin(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices [][][]float64 + DocShapeName string + Desc string + Expected []string + QueryType string + }{ + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: [][][]float64{{{0.5, 0.5}, {1, 0.5}, {1, 1}, {0.5, 1}, {0.5, 0.5}}}, + DocShapeName: "polygon1", + Desc: "polygon completely within bounded rectangle", + Expected: nil, + QueryType: "within", + }, + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: [][][]float64{{{0.5, 0.5}, {1.5, 0.5}, {1.5, 1.5}, {0.5, 1.5}, {0.5, 0.5}}}, + DocShapeName: "polygon1", + Desc: "polygon partially within bounded rectangle", + Expected: nil, + QueryType: "within", + }, + { + QueryShape: [][]float64{{0, 1}, {1, 0}}, + DocShapeVertices: [][][]float64{{{10.5, 10.5}, {11.5, 10.5}, {11.5, 11.5}, {10.5, 11.5}, {10.5, 10.5}}}, + DocShapeName: "polygon1", + Desc: "polygon completely outside bounded rectangle", + Expected: nil, + QueryType: "within", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "polygon", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeEnvelopeRelationQuery(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for Envelope: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPointLinestringWithin(t *testing.T) { + tests := []struct { + QueryShape []float64 + DocShapeVertices [][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: []float64{1.0, 1.0}, + DocShapeVertices: [][]float64{{1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}}, + DocShapeName: "linestring1", + Expected: nil, + Desc: "point does not cover different linestring", + QueryType: "within", + }, + { + QueryShape: []float64{179.1, 0.0}, + DocShapeVertices: [][]float64{{-179.0, 0.0}, {179.0, 0.0}}, + DocShapeName: "linestring1", + Expected: nil, + Desc: "point across latitudinal boundary of linestring", + QueryType: "within", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "linestring", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePointRelationQuery(test.QueryType, + false, indexReader, [][]float64{test.QueryShape}, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for point: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPointPolygonWithin(t *testing.T) { + tests := []struct { + QueryShape []float64 + DocShapeVertices [][][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: []float64{1.0, 1.0}, + DocShapeVertices: [][][]float64{{{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}}}, + DocShapeName: "polygon1", + Expected: nil, + Desc: "point not within polygon", + QueryType: "within", + }, + { // from binary predicates file + QueryShape: rightRectPoint, + DocShapeVertices: rightRect, + DocShapeName: "polygon1", + Expected: nil, // will return nil since a point only returns non-nil for a coincident point + // even if the point is on the polygon + Desc: "point on rectangle vertex", + QueryType: "within", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "polygon", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePointRelationQuery(test.QueryType, + false, indexReader, [][]float64{test.QueryShape}, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for point: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestLinestringPointWithin(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices []float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: [][]float64{{1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}}, + DocShapeVertices: []float64{1.0, 1.0}, + DocShapeName: "point1", + Expected: []string{"point1"}, + Desc: "point at start of linestring", + QueryType: "within", + }, + { + QueryShape: [][]float64{{1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}}, + DocShapeVertices: []float64{2.0, 2.0}, + DocShapeName: "point1", + Expected: []string{"point1"}, + Desc: "point in the middle of linestring", + QueryType: "within", + }, + { + QueryShape: [][]float64{{1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}}, + DocShapeVertices: []float64{3.0, 3.0}, + DocShapeName: "point1", + Expected: []string{"point1"}, + Desc: "point at end of linestring", + QueryType: "within", + }, + { + QueryShape: [][]float64{{1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}}, + DocShapeVertices: []float64{1.5, 1.50017}, + DocShapeName: "point1", + Expected: nil, + Desc: "point in between linestring", + QueryType: "within", + }, + { + QueryShape: [][]float64{{1.0, 1.0}, {2.0, 2.0}, {3.0, 3.0}}, + DocShapeVertices: []float64{4, 5}, + DocShapeName: "point1", + Expected: nil, + Desc: "point not contained by linestring", + QueryType: "within", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "point", + [][][][]float64{{{test.DocShapeVertices}}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeLinestringQueryWithRelation(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for linestring: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestMultiPointMultiLinestringWithin(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices [][][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: [][]float64{{2, 2}, {2.1, 2.1}}, + DocShapeVertices: [][][]float64{{{1, 1}, {1.1, 1.1}}, {{2, 2}, {2.1, 2.1}}}, + DocShapeName: "multilinestring1", + Expected: nil, // nil since multipoint within multiline is always nil + Desc: "multilinestring covering multipoint", + QueryType: "within", + }, + { + QueryShape: [][]float64{{2, 2}, {1, 1}, {3, 3}}, + DocShapeVertices: [][][]float64{{{1, 1}, {1.1, 1.1}}, {{2, 2}, {2.1, 2.1}}}, + DocShapeName: "multipoint1", + Expected: nil, + Desc: "multilinestring not covering multipoint", + QueryType: "within", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "multilinestring", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePointRelationQuery(test.QueryType, + true, indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for multilinestring: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestLinestringWithin(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices [][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: [][]float64{{1, 1}, {2, 2}, {3, 3}}, + DocShapeVertices: [][]float64{{1, 1}, {2, 2}, {3, 3}, {4, 4}}, + DocShapeName: "linestring1", + Expected: nil, + Desc: "longer linestring", + QueryType: "within", + }, + { + QueryShape: [][]float64{{1, 1}, {2, 2}, {3, 3}}, + DocShapeVertices: [][]float64{{1, 1}, {2, 2}, {3, 3}}, + DocShapeName: "linestring1", + Expected: nil, + Desc: "coincident linestrings", + QueryType: "within", + }, + } + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "linestring", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeLinestringQueryWithRelation(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for linestring: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestLinestringGeometryCollectionWithin(t *testing.T) { + tests := []struct { + QueryShape [][]float64 + DocShapeVertices [][][][][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + Types []string + }{ + { + QueryShape: [][]float64{{1, 1}, {2, 2}}, + DocShapeVertices: [][][][][]float64{{{{{1, 1}}}}}, + DocShapeName: "geometrycollection1", + Expected: nil, // LS is not a closed shape + Desc: "geometry collection with a point on vertex of linestring", + Types: []string{"point"}, + QueryType: "within", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetupGeometryCollection(t, test.DocShapeName, test.Types, + test.DocShapeVertices, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeLinestringQueryWithRelation(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for linestring: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPolygonPointWithin(t *testing.T) { + tests := []struct { + QueryShape [][][]float64 + DocShapeVertices []float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + DocShapeVertices: []float64{0.5, 0.5}, + DocShapeName: "point1", + Expected: []string{"point1"}, + Desc: "point within polygon", + QueryType: "within", + }, + { + QueryShape: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + DocShapeVertices: []float64{5.5, 5.5}, + DocShapeName: "point1", + Expected: nil, + Desc: "point not within polygon", + QueryType: "within", + }, + { + QueryShape: [][][]float64{{ + {0, 0}, + {1, 0}, + {1, 1}, + {0, 1}, + {0, 0}, + {0.2, 0.2}, + {0.2, 0.4}, + {0.4, 0.4}, + {0.4, 0.2}, + {0.2, 0.2}, + }}, + DocShapeVertices: []float64{0.3, 0.3}, + DocShapeName: "point1", + Expected: nil, + Desc: "point within polygon hole", + QueryType: "within", + }, + { + QueryShape: [][][]float64{{{0.0, 0.0}, {1.0, 0.0}, {1.0, 1.0}, {0.0, 1.0}, {0.0, 0.0}}}, + DocShapeVertices: []float64{1.0, 0.0}, + DocShapeName: "point1", + Expected: []string{"point1"}, + Desc: "point on polygon vertex", + QueryType: "within", + }, + { + QueryShape: [][][]float64{{{1, 1}, {2, 2}, {0, 2}, {1, 1}}}, + DocShapeVertices: []float64{1.5, 1.5001714}, + DocShapeName: "point1", + Expected: []string{"point1"}, + Desc: "point inside polygon", + QueryType: "within", + }, + { + QueryShape: [][][]float64{{{150, 85}, {-20, -85}, {-30, 85}, {160, -85}, {150, 85}}}, + DocShapeVertices: []float64{170, 85}, + DocShapeName: "point1", + Expected: nil, + Desc: "point outside the polygon's latitudinal boundary", + QueryType: "within", + }, + { + // from binary predicates tests + QueryShape: leftRect, + DocShapeVertices: leftRectPoint, + DocShapeName: "point1", + Expected: []string{"point1"}, + Desc: "point in left rectangle", + QueryType: "within", + }, + { + // from binary predicates tests + QueryShape: rightRect, + DocShapeVertices: rightRectPoint, + DocShapeName: "point1", + Expected: []string{"point1"}, + Desc: "point in right rectangle", + QueryType: "within", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "point", + [][][][]float64{{{test.DocShapeVertices}}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePolygonQueryWithRelation(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for polygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPolygonLinestringWithin(t *testing.T) { + tests := []struct { + QueryShape [][][]float64 + DocShapeVertices [][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + DocShapeVertices: [][]float64{{0.1, 0.1}, {0.4, 0.4}}, + DocShapeName: "linestring1", + Expected: []string{"linestring1"}, + Desc: "linestring within polygon", + QueryType: "within", + }, + { + QueryShape: [][][]float64{ + {{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}, + {{0.2, 0.2}, {0.2, 0.4}, {0.4, 0.4}, {0.4, 0.2}, {0.2, 0.2}}, + }, + DocShapeVertices: [][]float64{{0.3, 0.3}, {0.55, 0.55}}, + DocShapeName: "linestring1", + Expected: nil, + Desc: "linestring intersecting with polygon hole", + QueryType: "within", + }, + { + QueryShape: [][][]float64{ + {{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}, + {{0.2, 0.2}, {0.2, 0.4}, {0.4, 0.4}, {0.4, 0.2}, {0.2, 0.2}}, + }, + DocShapeVertices: [][]float64{{0.3, 0.3}, {4.0, 4.0}}, + DocShapeName: "linestring1", + Expected: nil, + Desc: "linestring intersecting with polygon hole and outside", + QueryType: "within", + }, + { + QueryShape: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + DocShapeVertices: [][]float64{{-1, -1}, {-2, -2}}, + DocShapeName: "linestring1", + Expected: nil, + Desc: "linestring outside polygon", + QueryType: "within", + }, + { + QueryShape: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + DocShapeVertices: [][]float64{{-0.5, -0.5}, {0.5, 0.5}}, + DocShapeName: "linestring1", + Expected: nil, + Desc: "linestring intersecting polygon", + QueryType: "within", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "linestring", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePolygonQueryWithRelation(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for polygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestPolygonWithin(t *testing.T) { + tests := []struct { + QueryShape [][][]float64 + DocShapeVertices [][][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + DocShapeVertices: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + DocShapeName: "polygon1", + Expected: []string{"polygon1"}, + Desc: "coincident polygon", + QueryType: "within", + }, + { + QueryShape: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + DocShapeVertices: [][][]float64{{{0.2, 0.2}, {1, 0}, {1, 1}, {0, 1}, {0.2, 0.2}}}, + DocShapeName: "polygon1", + Expected: []string{"polygon1"}, + Desc: "polygon covers an intersecting window of itself", + QueryType: "within", + }, + { + QueryShape: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + DocShapeVertices: [][][]float64{{{0.1, 0.1}, {0.2, 0.1}, {0.2, 0.2}, {0.1, 0.2}, {0.1, 0.1}}}, + DocShapeName: "polygon1", + Expected: []string{"polygon1"}, + Desc: "polygon covers a nested version of itself", + QueryType: "within", + }, + { + QueryShape: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + DocShapeVertices: [][][]float64{{{-1, 0}, {1, 0}, {1, 1}, {-1, 1}, {-1, 0}}}, + DocShapeName: "polygon1", + Expected: nil, + Desc: "intersecting polygons", + QueryType: "within", + }, + { + QueryShape: [][][]float64{{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + DocShapeVertices: [][][]float64{{{3, 3}, {4, 3}, {4, 4}, {3, 4}, {3, 3}}}, + DocShapeName: "polygon1", + Expected: nil, + Desc: "polygon totally out of range", + QueryType: "within", + }, + { + QueryShape: leftRect, + DocShapeVertices: rightRect, + DocShapeName: "polygon1", + Expected: nil, + Desc: "left and right polygons,sharing an edge", + QueryType: "within", + }, + } + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "polygon", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePolygonQueryWithRelation(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for polygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestMultiPolygonMultiPointWithin(t *testing.T) { + tests := []struct { + QueryShape [][][][]float64 + DocShapeVertices [][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: [][][][]float64{ + {{{30, 25}, {45, 40}, {10, 40}, {30, 20}, {30, 25}}}, + {{{15, 5}, {40, 10}, {10, 20}, {5, 10}, {15, 5}}}, + }, + DocShapeVertices: [][]float64{{30, 20}, {15, 5}}, + DocShapeName: "multipoint1", + Expected: []string{"multipoint1"}, + Desc: "multipolygon covers multipoint", + QueryType: "within", + }, + { + QueryShape: [][][][]float64{ + {{{15, 5}, {40, 10}, {10, 20}, {5, 10}, {15, 5}}}, + {{{30, 20}, {45, 40}, {10, 40}, {30, 20}}}, + }, + DocShapeVertices: [][]float64{{30, 20}, {30, 30}, {45, 66}}, + DocShapeName: "multipoint1", + Expected: nil, + Desc: "multipolygon does not cover multipoint", + QueryType: "within", + }, + { + QueryShape: [][][][]float64{ + {{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}, + {{{1, 0}, {2, 0}, {2, 1}, {1, 1}, {1, 0}}}, + }, + DocShapeVertices: [][]float64{{0.5, 0.5}, {1.5, 0.5}}, + DocShapeName: "multipoint1", + Expected: []string{"multipoint1"}, + Desc: "multiple multipolygons required to cover multipoint", + QueryType: "within", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "multipoint", + [][][][]float64{{test.DocShapeVertices}}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeMultiPolygonQueryWithRelation(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for polygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestMultiLinestringWithin(t *testing.T) { + tests := []struct { + QueryShape [][][]float64 + DocShapeVertices [][][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: [][][]float64{{{1, 2}, {2, 3}, {3, 4}}, {{5, 6}, {6.5, 7.8}}}, + DocShapeVertices: [][][]float64{{{1, 2}, {2, 3}, {3, 4}}}, + DocShapeName: "multilinestring1", + Expected: nil, + Desc: "multilinestrings with common linestrings", + QueryType: "within", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "multilinestring", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeMultiLinestringQueryWithRelation("within", + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for multilinestring: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestMultiPolygonMultiLinestringWithin(t *testing.T) { + tests := []struct { + QueryShape [][][][]float64 + DocShapeVertices [][][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: [][][][]float64{ + {{{15, 5}, {40, 10}, {10, 20}, {5, 10}, {15, 5}}}, + {{{30, 20}, {45, 40}, {10, 40}, {30, 20}}}, + }, + DocShapeVertices: [][][]float64{{{45, 40}, {10, 40}}, {{45, 40}, {10, 40}, {30, 20}}}, + DocShapeName: "multilinestring1", + Expected: []string{"multilinestring1"}, + Desc: "multilinestring intersecting at the edge of multipolygon", + QueryType: "within", + }, + { + QueryShape: [][][][]float64{ + {{{15, 5}, {40, 10}, {10, 20}, {5, 10}, {15, 5}}}, + {{{30, 20}, {45, 40}, {10, 40}, {30, 20}}}, + }, + DocShapeVertices: [][][]float64{{{48, 40}, {8, 40}}, {{48, 40}, {8, 40}, {30, 12}}}, + DocShapeName: "multilinestring1", + Expected: nil, + Desc: "multipolygon does not cover multilinestring", + QueryType: "within", + }, + } + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "multilinestring", + [][][][]float64{test.DocShapeVertices}, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeMultiPolygonQueryWithRelation(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for multipolygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestMultiPolygonWithin(t *testing.T) { + tests := []struct { + QueryShape [][][][]float64 + DocShapeVertices [][][][]float64 + DocShapeName string + Expected []string + Desc string + QueryType string + }{ + { + QueryShape: [][][][]float64{ + {{{16, 6}, {41, 11}, {11, 21}, {6, 11}, {16, 6}}}, + {{{31, 21}, {46, 41}, {11, 41}, {31, 21}}}, + }, + DocShapeVertices: [][][][]float64{{{{31, 21}, {46, 41}, {11, 41}, {31, 21}}}}, + DocShapeName: "multipolygon1", + Expected: []string{"multipolygon1"}, + Desc: "multipolygon covers another multipolygon", + QueryType: "within", + }, + { + QueryShape: [][][][]float64{ + {{{16, 6}, {41, 11}, {11, 21}, {6, 11}, {16, 6}}}, + {{{31, 21}, {46, 41}, {11, 41}, {31, 21}}}, + }, + DocShapeVertices: [][][][]float64{{{{31, 21}, {46, 41}, {16, 46}, {31, 21}}}}, + DocShapeName: "multipolygon1", + Expected: nil, + Desc: "multipolygon does not cover multipolygon", + QueryType: "within", + }, + } + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetup(t, test.DocShapeName, "multipolygon", + test.DocShapeVertices, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeMultiPolygonQueryWithRelation(test.QueryType, + indexReader, test.QueryShape, "geometry") + if err != nil { + t.Error(err.Error()) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for multipolygon: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestGeometryCollectionWithin(t *testing.T) { + tests := []struct { + QueryShape [][][][][]float64 + DocShapeVertices [][][][][]float64 + DocShapeName string + Desc string + Expected []string + QueryType string + QueryShapeTypes []string + DocShapeTypes []string + }{ + { + QueryShape: [][][][][]float64{{{{}}}}, + DocShapeVertices: [][][][][]float64{{{{}}}}, + DocShapeName: "geometrycollection1", + Desc: "empty geometry collections", + Expected: nil, + QueryType: "within", + QueryShapeTypes: []string{""}, + DocShapeTypes: []string{""}, + }, + { + QueryShape: [][][][][]float64{{{{{1, 2}, {2, 3}}}}}, + DocShapeVertices: [][][][][]float64{{{{{1, 2}}}}}, + DocShapeName: "geometrycollection1", + Desc: "geometry collection with a linestring", + Expected: []string{"geometrycollection1"}, + QueryShapeTypes: []string{"linestring"}, + DocShapeTypes: []string{"point"}, + QueryType: "within", + }, + { + QueryShape: [][][][][]float64{{{{{1, 2}, {2, 3}, {5, 6}}}}}, + DocShapeVertices: [][][][][]float64{{{{{1, 2}}}}}, + DocShapeName: "geometrycollection1", + Desc: "geometry collections with common points and multipoints", + Expected: []string{"geometrycollection1"}, + QueryShapeTypes: []string{"multipoint"}, + DocShapeTypes: []string{"point"}, + QueryType: "within", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetupGeometryCollection(t, test.DocShapeName, test.DocShapeTypes, + test.DocShapeVertices, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapeGeometryCollectionRelationQuery(test.QueryType, + indexReader, test.QueryShape, test.QueryShapeTypes, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for geometry collection: %+v", + test.Expected, got, test.QueryShape) + } + }) + + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} + +func TestGeometryCollectionPointWithin(t *testing.T) { + tests := []struct { + QueryShape []float64 + DocShapeVertices [][][][][]float64 + DocShapeName string + Desc string + Expected []string + Types []string + QueryType string + }{ + { + QueryShape: []float64{1.0, 2.0}, + DocShapeVertices: [][][][][]float64{{{{}}}}, + DocShapeName: "geometrycollection1", + Desc: "empty geometry collection not within a point", + Expected: nil, + Types: []string{""}, + QueryType: "within", + }, + } + + i := setupIndex(t) + + for _, test := range tests { + indexReader, closeFn, err := testCaseSetupGeometryCollection(t, test.DocShapeName, test.Types, + test.DocShapeVertices, i) + if err != nil { + t.Error(err.Error()) + } + + t.Run(test.Desc, func(t *testing.T) { + got, err := runGeoShapePointRelationQuery("intersects", + false, indexReader, [][]float64{test.QueryShape}, "geometry") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.Expected) { + t.Errorf("expected %v, got %v for point: %+v", + test.Expected, got, test.QueryShape) + } + }) + err = closeFn() + if err != nil { + t.Error(err.Error()) + } + } +} diff --git a/search/searcher/optimize_knn.go b/search/searcher/optimize_knn.go new file mode 100644 index 0000000..efe262b --- /dev/null +++ b/search/searcher/optimize_knn.go @@ -0,0 +1,53 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package searcher + +import ( + "context" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func optimizeKNN(ctx context.Context, indexReader index.IndexReader, + qsearchers []search.Searcher) error { + var octx index.VectorOptimizableContext + var err error + + for _, searcher := range qsearchers { + // Only applicable to KNN Searchers. + o, ok := searcher.(index.VectorOptimizable) + if !ok { + continue + } + + octx, err = o.VectorOptimize(ctx, octx) + if err != nil { + return err + } + } + + // No KNN searchers. + if octx == nil { + return nil + } + + // Postings lists and iterators replaced in the pointer to the + // vector reader + return octx.Finish() +} diff --git a/search/searcher/optimize_no_knn.go b/search/searcher/optimize_no_knn.go new file mode 100644 index 0000000..bd5d91f --- /dev/null +++ b/search/searcher/optimize_no_knn.go @@ -0,0 +1,31 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !vectors +// +build !vectors + +package searcher + +import ( + "context" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func optimizeKNN(ctx context.Context, indexReader index.IndexReader, + qsearchers []search.Searcher) error { + // No-op + return nil +} diff --git a/search/searcher/ordered_searchers_list.go b/search/searcher/ordered_searchers_list.go new file mode 100644 index 0000000..ac9da56 --- /dev/null +++ b/search/searcher/ordered_searchers_list.go @@ -0,0 +1,55 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "github.com/blevesearch/bleve/v2/search" +) + +type OrderedSearcherList []search.Searcher + +// sort.Interface + +func (otrl OrderedSearcherList) Len() int { + return len(otrl) +} + +func (otrl OrderedSearcherList) Less(i, j int) bool { + return otrl[i].Count() < otrl[j].Count() +} + +func (otrl OrderedSearcherList) Swap(i, j int) { + otrl[i], otrl[j] = otrl[j], otrl[i] +} + +type OrderedPositionalSearcherList struct { + searchers []search.Searcher + index []int +} + +// sort.Interface + +func (otrl OrderedPositionalSearcherList) Len() int { + return len(otrl.searchers) +} + +func (otrl OrderedPositionalSearcherList) Less(i, j int) bool { + return otrl.searchers[i].Count() < otrl.searchers[j].Count() +} + +func (otrl OrderedPositionalSearcherList) Swap(i, j int) { + otrl.searchers[i], otrl.searchers[j] = otrl.searchers[j], otrl.searchers[i] + otrl.index[i], otrl.index[j] = otrl.index[j], otrl.index[i] +} diff --git a/search/searcher/search_boolean.go b/search/searcher/search_boolean.go new file mode 100644 index 0000000..bf207f8 --- /dev/null +++ b/search/searcher/search_boolean.go @@ -0,0 +1,451 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "math" + "reflect" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeBooleanSearcher int + +func init() { + var bs BooleanSearcher + reflectStaticSizeBooleanSearcher = int(reflect.TypeOf(bs).Size()) +} + +type BooleanSearcher struct { + indexReader index.IndexReader + mustSearcher search.Searcher + shouldSearcher search.Searcher + mustNotSearcher search.Searcher + queryNorm float64 + currMust *search.DocumentMatch + currShould *search.DocumentMatch + currMustNot *search.DocumentMatch + currentID index.IndexInternalID + min uint64 + scorer *scorer.ConjunctionQueryScorer + matches []*search.DocumentMatch + initialized bool + done bool +} + +func NewBooleanSearcher(ctx context.Context, indexReader index.IndexReader, mustSearcher search.Searcher, shouldSearcher search.Searcher, mustNotSearcher search.Searcher, options search.SearcherOptions) (*BooleanSearcher, error) { + // build our searcher + rv := BooleanSearcher{ + indexReader: indexReader, + mustSearcher: mustSearcher, + shouldSearcher: shouldSearcher, + mustNotSearcher: mustNotSearcher, + scorer: scorer.NewConjunctionQueryScorer(options), + matches: make([]*search.DocumentMatch, 2), + } + rv.computeQueryNorm() + return &rv, nil +} + +func (s *BooleanSearcher) Size() int { + sizeInBytes := reflectStaticSizeBooleanSearcher + size.SizeOfPtr + + if s.mustSearcher != nil { + sizeInBytes += s.mustSearcher.Size() + } + + if s.shouldSearcher != nil { + sizeInBytes += s.shouldSearcher.Size() + } + + if s.mustNotSearcher != nil { + sizeInBytes += s.mustNotSearcher.Size() + } + + sizeInBytes += s.scorer.Size() + + for _, entry := range s.matches { + if entry != nil { + sizeInBytes += entry.Size() + } + } + + return sizeInBytes +} + +func (s *BooleanSearcher) computeQueryNorm() { + // first calculate sum of squared weights + sumOfSquaredWeights := 0.0 + if s.mustSearcher != nil { + sumOfSquaredWeights += s.mustSearcher.Weight() + } + if s.shouldSearcher != nil { + sumOfSquaredWeights += s.shouldSearcher.Weight() + } + + // now compute query norm from this + s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) + // finally tell all the downstream searchers the norm + if s.mustSearcher != nil { + s.mustSearcher.SetQueryNorm(s.queryNorm) + } + if s.shouldSearcher != nil { + s.shouldSearcher.SetQueryNorm(s.queryNorm) + } +} + +func (s *BooleanSearcher) initSearchers(ctx *search.SearchContext) error { + var err error + // get all searchers pointing at their first match + if s.mustSearcher != nil { + if s.currMust != nil { + ctx.DocumentMatchPool.Put(s.currMust) + } + s.currMust, err = s.mustSearcher.Next(ctx) + if err != nil { + return err + } + } + + if s.shouldSearcher != nil { + if s.currShould != nil { + ctx.DocumentMatchPool.Put(s.currShould) + } + s.currShould, err = s.shouldSearcher.Next(ctx) + if err != nil { + return err + } + } + + if s.mustNotSearcher != nil { + if s.currMustNot != nil { + ctx.DocumentMatchPool.Put(s.currMustNot) + } + s.currMustNot, err = s.mustNotSearcher.Next(ctx) + if err != nil { + return err + } + } + + if s.mustSearcher != nil && s.currMust != nil { + s.currentID = s.currMust.IndexInternalID + } else if s.mustSearcher == nil && s.currShould != nil { + s.currentID = s.currShould.IndexInternalID + } else { + s.currentID = nil + } + + s.initialized = true + return nil +} + +func (s *BooleanSearcher) advanceNextMust(ctx *search.SearchContext, skipReturn *search.DocumentMatch) error { + var err error + + if s.mustSearcher != nil { + if s.currMust != skipReturn { + ctx.DocumentMatchPool.Put(s.currMust) + } + s.currMust, err = s.mustSearcher.Next(ctx) + if err != nil { + return err + } + } else { + if s.currShould != skipReturn { + ctx.DocumentMatchPool.Put(s.currShould) + } + s.currShould, err = s.shouldSearcher.Next(ctx) + if err != nil { + return err + } + } + + if s.mustSearcher != nil && s.currMust != nil { + s.currentID = s.currMust.IndexInternalID + } else if s.mustSearcher == nil && s.currShould != nil { + s.currentID = s.currShould.IndexInternalID + } else { + s.currentID = nil + } + return nil +} + +func (s *BooleanSearcher) Weight() float64 { + var rv float64 + if s.mustSearcher != nil { + rv += s.mustSearcher.Weight() + } + if s.shouldSearcher != nil { + rv += s.shouldSearcher.Weight() + } + + return rv +} + +func (s *BooleanSearcher) SetQueryNorm(qnorm float64) { + if s.mustSearcher != nil { + s.mustSearcher.SetQueryNorm(qnorm) + } + if s.shouldSearcher != nil { + s.shouldSearcher.SetQueryNorm(qnorm) + } +} + +func (s *BooleanSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + + if s.done { + return nil, nil + } + + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + + var err error + var rv *search.DocumentMatch + + for s.currentID != nil { + if s.currMustNot != nil { + cmp := s.currMustNot.IndexInternalID.Compare(s.currentID) + if cmp < 0 { + ctx.DocumentMatchPool.Put(s.currMustNot) + // advance must not searcher to our candidate entry + s.currMustNot, err = s.mustNotSearcher.Advance(ctx, s.currentID) + if err != nil { + return nil, err + } + if s.currMustNot != nil && s.currMustNot.IndexInternalID.Equals(s.currentID) { + // the candidate is excluded + err = s.advanceNextMust(ctx, nil) + if err != nil { + return nil, err + } + continue + } + } else if cmp == 0 { + // the candidate is excluded + err = s.advanceNextMust(ctx, nil) + if err != nil { + return nil, err + } + continue + } + } + + shouldCmpOrNil := 1 // NOTE: shouldCmp will also be 1 when currShould == nil. + if s.currShould != nil { + shouldCmpOrNil = s.currShould.IndexInternalID.Compare(s.currentID) + } + + if shouldCmpOrNil < 0 { + ctx.DocumentMatchPool.Put(s.currShould) + // advance should searcher to our candidate entry + s.currShould, err = s.shouldSearcher.Advance(ctx, s.currentID) + if err != nil { + return nil, err + } + if s.currShould != nil && s.currShould.IndexInternalID.Equals(s.currentID) { + // score bonus matches should + var cons []*search.DocumentMatch + if s.currMust != nil { + cons = s.matches + cons[0] = s.currMust + cons[1] = s.currShould + } else { + cons = s.matches[0:1] + cons[0] = s.currShould + } + rv = s.scorer.Score(ctx, cons) + err = s.advanceNextMust(ctx, rv) + if err != nil { + return nil, err + } + break + } else if s.shouldSearcher.Min() == 0 { + // match is OK anyway + cons := s.matches[0:1] + cons[0] = s.currMust + rv = s.scorer.Score(ctx, cons) + err = s.advanceNextMust(ctx, rv) + if err != nil { + return nil, err + } + break + } + } else if shouldCmpOrNil == 0 { + // score bonus matches should + var cons []*search.DocumentMatch + if s.currMust != nil { + cons = s.matches + cons[0] = s.currMust + cons[1] = s.currShould + } else { + cons = s.matches[0:1] + cons[0] = s.currShould + } + rv = s.scorer.Score(ctx, cons) + err = s.advanceNextMust(ctx, rv) + if err != nil { + return nil, err + } + break + } else if s.shouldSearcher == nil || s.shouldSearcher.Min() == 0 { + // match is OK anyway + cons := s.matches[0:1] + cons[0] = s.currMust + rv = s.scorer.Score(ctx, cons) + err = s.advanceNextMust(ctx, rv) + if err != nil { + return nil, err + } + break + } + + err = s.advanceNextMust(ctx, nil) + if err != nil { + return nil, err + } + } + + if rv == nil { + s.done = true + } + + return rv, nil +} + +func (s *BooleanSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + + if s.done { + return nil, nil + } + + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + + // Advance the searcher only if the cursor is trailing the lookup ID + if s.currentID == nil || s.currentID.Compare(ID) < 0 { + var err error + if s.mustSearcher != nil { + if s.currMust != nil { + ctx.DocumentMatchPool.Put(s.currMust) + } + s.currMust, err = s.mustSearcher.Advance(ctx, ID) + if err != nil { + return nil, err + } + } + + if s.shouldSearcher != nil { + if s.currShould != nil { + ctx.DocumentMatchPool.Put(s.currShould) + } + s.currShould, err = s.shouldSearcher.Advance(ctx, ID) + if err != nil { + return nil, err + } + } + + if s.mustNotSearcher != nil { + // Additional check for mustNotSearcher, whose cursor isn't tracked by + // currentID to prevent it from moving when the searcher's tracked + // position is already ahead of or at the requested ID. + if s.currMustNot == nil || s.currMustNot.IndexInternalID.Compare(ID) < 0 { + if s.currMustNot != nil { + ctx.DocumentMatchPool.Put(s.currMustNot) + } + s.currMustNot, err = s.mustNotSearcher.Advance(ctx, ID) + if err != nil { + return nil, err + } + } + } + + if s.mustSearcher != nil && s.currMust != nil { + s.currentID = s.currMust.IndexInternalID + } else if s.mustSearcher == nil && s.currShould != nil { + s.currentID = s.currShould.IndexInternalID + } else { + s.currentID = nil + } + } + + return s.Next(ctx) +} + +func (s *BooleanSearcher) Count() uint64 { + + // for now return a worst case + var sum uint64 + if s.mustSearcher != nil { + sum += s.mustSearcher.Count() + } + if s.shouldSearcher != nil { + sum += s.shouldSearcher.Count() + } + return sum +} + +func (s *BooleanSearcher) Close() error { + var err0, err1, err2 error + if s.mustSearcher != nil { + err0 = s.mustSearcher.Close() + } + if s.shouldSearcher != nil { + err1 = s.shouldSearcher.Close() + } + if s.mustNotSearcher != nil { + err2 = s.mustNotSearcher.Close() + } + if err0 != nil { + return err0 + } + if err1 != nil { + return err1 + } + if err2 != nil { + return err2 + } + return nil +} + +func (s *BooleanSearcher) Min() int { + return 0 +} + +func (s *BooleanSearcher) DocumentMatchPoolSize() int { + rv := 3 + if s.mustSearcher != nil { + rv += s.mustSearcher.DocumentMatchPoolSize() + } + if s.shouldSearcher != nil { + rv += s.shouldSearcher.DocumentMatchPoolSize() + } + if s.mustNotSearcher != nil { + rv += s.mustNotSearcher.DocumentMatchPoolSize() + } + return rv +} diff --git a/search/searcher/search_boolean_test.go b/search/searcher/search_boolean_test.go new file mode 100644 index 0000000..1c803e9 --- /dev/null +++ b/search/searcher/search_boolean_test.go @@ -0,0 +1,382 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "testing" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func TestBooleanSearch(t *testing.T) { + if twoDocIndex == nil { + t.Fatal("its null") + } + twoDocIndexReader, err := twoDocIndex.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := twoDocIndexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + explainTrue := search.SearcherOptions{Explain: true} + + // test 0 + beerTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "beer", "desc", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + mustSearcher, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher}, explainTrue) + if err != nil { + t.Fatal(err) + } + martyTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "marty", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + dustinTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "dustin", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + shouldSearcher, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{martyTermSearcher, dustinTermSearcher}, 0, explainTrue) + if err != nil { + t.Fatal(err) + } + steveTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "steve", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + mustNotSearcher, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{steveTermSearcher}, 0, explainTrue) + if err != nil { + t.Fatal(err) + } + booleanSearcher, err := NewBooleanSearcher(context.TODO(), twoDocIndexReader, mustSearcher, shouldSearcher, mustNotSearcher, explainTrue) + if err != nil { + t.Fatal(err) + } + + // test 1 + martyTermSearcher2, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "marty", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + dustinTermSearcher2, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "dustin", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + shouldSearcher2, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{martyTermSearcher2, dustinTermSearcher2}, 0, explainTrue) + if err != nil { + t.Fatal(err) + } + steveTermSearcher2, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "steve", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + mustNotSearcher2, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{steveTermSearcher2}, 0, explainTrue) + if err != nil { + t.Fatal(err) + } + booleanSearcher2, err := NewBooleanSearcher(context.TODO(), twoDocIndexReader, nil, shouldSearcher2, mustNotSearcher2, explainTrue) + if err != nil { + t.Fatal(err) + } + + // test 2 + steveTermSearcher3, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "steve", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + mustNotSearcher3, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{steveTermSearcher3}, 0, explainTrue) + if err != nil { + t.Fatal(err) + } + booleanSearcher3, err := NewBooleanSearcher(context.TODO(), twoDocIndexReader, nil, nil, mustNotSearcher3, explainTrue) + if err != nil { + t.Fatal(err) + } + + // test 3 + beerTermSearcher4, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "beer", "desc", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + mustSearcher4, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher4}, explainTrue) + if err != nil { + t.Fatal(err) + } + steveTermSearcher4, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "steve", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + mustNotSearcher4, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{steveTermSearcher4}, 0, explainTrue) + if err != nil { + t.Fatal(err) + } + booleanSearcher4, err := NewBooleanSearcher(context.TODO(), twoDocIndexReader, mustSearcher4, nil, mustNotSearcher4, explainTrue) + if err != nil { + t.Fatal(err) + } + + // test 4 + beerTermSearcher5, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "beer", "desc", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + mustSearcher5, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher5}, explainTrue) + if err != nil { + t.Fatal(err) + } + steveTermSearcher5, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "steve", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + martyTermSearcher5, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "marty", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + mustNotSearcher5, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{steveTermSearcher5, martyTermSearcher5}, 0, explainTrue) + if err != nil { + t.Fatal(err) + } + booleanSearcher5, err := NewBooleanSearcher(context.TODO(), twoDocIndexReader, mustSearcher5, nil, mustNotSearcher5, explainTrue) + if err != nil { + t.Fatal(err) + } + + // test 5 + beerTermSearcher6, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "beer", "desc", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + mustSearcher6, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher6}, explainTrue) + if err != nil { + t.Fatal(err) + } + martyTermSearcher6, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "marty", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + dustinTermSearcher6, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "dustin", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + shouldSearcher6, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{martyTermSearcher6, dustinTermSearcher6}, 2, explainTrue) + if err != nil { + t.Fatal(err) + } + booleanSearcher6, err := NewBooleanSearcher(context.TODO(), twoDocIndexReader, mustSearcher6, shouldSearcher6, nil, explainTrue) + if err != nil { + t.Fatal(err) + } + + // test 6 + beerTermSearcher7, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "beer", "desc", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + mustSearcher7, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher7}, explainTrue) + if err != nil { + t.Fatal(err) + } + booleanSearcher7, err := NewBooleanSearcher(context.TODO(), twoDocIndexReader, mustSearcher7, nil, nil, explainTrue) + if err != nil { + t.Fatal(err) + } + martyTermSearcher7, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "marty", "name", 5.0, explainTrue) + if err != nil { + t.Fatal(err) + } + conjunctionSearcher7, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{martyTermSearcher7, booleanSearcher7}, explainTrue) + if err != nil { + t.Fatal(err) + } + + // test 7 + beerTermSearcher8, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "beer", "desc", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + mustSearcher8, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher8}, explainTrue) + if err != nil { + t.Fatal(err) + } + martyTermSearcher8, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "marty", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + dustinTermSearcher8, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "dustin", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + shouldSearcher8, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{martyTermSearcher8, dustinTermSearcher8}, 0, explainTrue) + if err != nil { + t.Fatal(err) + } + steveTermSearcher8, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "steve", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + mustNotSearcher8, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{steveTermSearcher8}, 0, explainTrue) + if err != nil { + t.Fatal(err) + } + booleanSearcher8, err := NewBooleanSearcher(context.TODO(), twoDocIndexReader, mustSearcher8, shouldSearcher8, mustNotSearcher8, explainTrue) + if err != nil { + t.Fatal(err) + } + dustinTermSearcher8a, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "dustin", "name", 5.0, explainTrue) + if err != nil { + t.Fatal(err) + } + conjunctionSearcher8, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{booleanSearcher8, dustinTermSearcher8a}, explainTrue) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + searcher search.Searcher + results []*search.DocumentMatch + }{ + { + searcher: booleanSearcher, + results: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("1"), + Score: 0.9818005051949021, + }, + { + IndexInternalID: index.IndexInternalID("3"), + Score: 0.808709699395535, + }, + { + IndexInternalID: index.IndexInternalID("4"), + Score: 0.34618161159873423, + }, + }, + }, + { + searcher: booleanSearcher2, + results: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("1"), + Score: 0.6775110856165737, + }, + { + IndexInternalID: index.IndexInternalID("3"), + Score: 0.6775110856165737, + }, + }, + }, + // no MUST or SHOULD clauses yields no results + { + searcher: booleanSearcher3, + results: []*search.DocumentMatch{}, + }, + { + searcher: booleanSearcher4, + results: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("1"), + Score: 1.0, + }, + { + IndexInternalID: index.IndexInternalID("3"), + Score: 0.5, + }, + { + IndexInternalID: index.IndexInternalID("4"), + Score: 1.0, + }, + }, + }, + { + searcher: booleanSearcher5, + results: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("3"), + Score: 0.5, + }, + { + IndexInternalID: index.IndexInternalID("4"), + Score: 1.0, + }, + }, + }, + { + searcher: booleanSearcher6, + results: []*search.DocumentMatch{}, + }, + // test a conjunction query with a nested boolean + { + searcher: conjunctionSearcher7, + results: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("1"), + Score: 2.0097428702814377, + }, + }, + }, + { + searcher: conjunctionSearcher8, + results: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("3"), + Score: 2.0681575785068107, + }, + }, + }, + } + + for testIndex, test := range tests { + defer func() { + err := test.searcher.Close() + if err != nil { + t.Fatal(err) + } + }() + + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0), + } + next, err := test.searcher.Next(ctx) + i := 0 + for err == nil && next != nil { + if i < len(test.results) { + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) + } + if !scoresCloseEnough(next.Score, test.results[i].Score) { + t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) + t.Logf("scoring explanation: %s", next.Expl) + } + } + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) + i++ + } + if err != nil { + t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) + } + if len(test.results) != i { + t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) + } + } +} diff --git a/search/searcher/search_conjunction.go b/search/searcher/search_conjunction.go new file mode 100644 index 0000000..57d8855 --- /dev/null +++ b/search/searcher/search_conjunction.go @@ -0,0 +1,285 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "math" + "reflect" + "sort" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeConjunctionSearcher int + +func init() { + var cs ConjunctionSearcher + reflectStaticSizeConjunctionSearcher = int(reflect.TypeOf(cs).Size()) +} + +type ConjunctionSearcher struct { + indexReader index.IndexReader + searchers []search.Searcher + queryNorm float64 + currs []*search.DocumentMatch + maxIDIdx int + scorer *scorer.ConjunctionQueryScorer + initialized bool + options search.SearcherOptions + bytesRead uint64 +} + +func NewConjunctionSearcher(ctx context.Context, indexReader index.IndexReader, + qsearchers []search.Searcher, options search.SearcherOptions) ( + search.Searcher, error, +) { + // build the sorted downstream searchers + searchers := make(OrderedSearcherList, len(qsearchers)) + copy(searchers, qsearchers) + sort.Sort(searchers) + + // attempt the "unadorned" conjunction optimization only when we + // do not need extra information like freq-norm's or term vectors + if len(searchers) > 1 && + options.Score == "none" && !options.IncludeTermVectors { + rv, err := optimizeCompositeSearcher(ctx, "conjunction:unadorned", + indexReader, searchers, options) + if err != nil || rv != nil { + return rv, err + } + } + + // build our searcher + rv := ConjunctionSearcher{ + indexReader: indexReader, + options: options, + searchers: searchers, + currs: make([]*search.DocumentMatch, len(searchers)), + scorer: scorer.NewConjunctionQueryScorer(options), + } + rv.computeQueryNorm() + + // attempt push-down conjunction optimization when there's >1 searchers + if len(searchers) > 1 { + rv, err := optimizeCompositeSearcher(ctx, "conjunction", + indexReader, searchers, options) + if err != nil || rv != nil { + return rv, err + } + } + + return &rv, nil +} + +func (s *ConjunctionSearcher) computeQueryNorm() { + // first calculate sum of squared weights + sumOfSquaredWeights := 0.0 + for _, searcher := range s.searchers { + sumOfSquaredWeights += searcher.Weight() + } + // now compute query norm from this + s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) + // finally tell all the downstream searchers the norm + for _, searcher := range s.searchers { + searcher.SetQueryNorm(s.queryNorm) + } +} + +func (s *ConjunctionSearcher) Size() int { + sizeInBytes := reflectStaticSizeConjunctionSearcher + size.SizeOfPtr + + s.scorer.Size() + + for _, entry := range s.searchers { + sizeInBytes += entry.Size() + } + + for _, entry := range s.currs { + if entry != nil { + sizeInBytes += entry.Size() + } + } + + return sizeInBytes +} + +func (s *ConjunctionSearcher) initSearchers(ctx *search.SearchContext) error { + var err error + // get all searchers pointing at their first match + for i, searcher := range s.searchers { + if s.currs[i] != nil { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = searcher.Next(ctx) + if err != nil { + return err + } + } + s.initialized = true + return nil +} + +func (s *ConjunctionSearcher) Weight() float64 { + var rv float64 + for _, searcher := range s.searchers { + rv += searcher.Weight() + } + return rv +} + +func (s *ConjunctionSearcher) SetQueryNorm(qnorm float64) { + for _, searcher := range s.searchers { + searcher.SetQueryNorm(qnorm) + } +} + +func (s *ConjunctionSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + var rv *search.DocumentMatch + var err error +OUTER: + for s.maxIDIdx < len(s.currs) && s.currs[s.maxIDIdx] != nil { + maxID := s.currs[s.maxIDIdx].IndexInternalID + + i := 0 + for i < len(s.currs) { + if s.currs[i] == nil { + return nil, nil + } + + if i == s.maxIDIdx { + i++ + continue + } + + cmp := maxID.Compare(s.currs[i].IndexInternalID) + if cmp == 0 { + i++ + continue + } + + if cmp < 0 { + // maxID < currs[i], so we found a new maxIDIdx + s.maxIDIdx = i + + // advance the positions where [0 <= x < i], since we + // know they were equal to the former max entry + maxID = s.currs[s.maxIDIdx].IndexInternalID + for x := 0; x < i; x++ { + err = s.advanceChild(ctx, x, maxID) + if err != nil { + return nil, err + } + } + + continue OUTER + } + + // maxID > currs[i], so need to advance searchers[i] + err = s.advanceChild(ctx, i, maxID) + if err != nil { + return nil, err + } + + // don't bump i, so that we'll examine the just-advanced + // currs[i] again + } + + // if we get here, a doc matched all readers, so score and add it + rv = s.scorer.Score(ctx, s.currs) + + // we know all the searchers are pointing at the same thing + // so they all need to be bumped + for i, searcher := range s.searchers { + if s.currs[i] != rv { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = searcher.Next(ctx) + if err != nil { + return nil, err + } + } + + // don't continue now, wait for the next call to Next() + break + } + return rv, nil +} + +func (s *ConjunctionSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + for i := range s.searchers { + if s.currs[i] != nil && s.currs[i].IndexInternalID.Compare(ID) >= 0 { + continue + } + err := s.advanceChild(ctx, i, ID) + if err != nil { + return nil, err + } + } + return s.Next(ctx) +} + +func (s *ConjunctionSearcher) advanceChild(ctx *search.SearchContext, i int, ID index.IndexInternalID) (err error) { + if s.currs[i] != nil { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = s.searchers[i].Advance(ctx, ID) + return err +} + +func (s *ConjunctionSearcher) Count() uint64 { + // for now return a worst case + var sum uint64 + for _, searcher := range s.searchers { + sum += searcher.Count() + } + return sum +} + +func (s *ConjunctionSearcher) Close() (rv error) { + for _, searcher := range s.searchers { + err := searcher.Close() + if err != nil && rv == nil { + rv = err + } + } + return rv +} + +func (s *ConjunctionSearcher) Min() int { + return 0 +} + +func (s *ConjunctionSearcher) DocumentMatchPoolSize() int { + rv := len(s.currs) + for _, s := range s.searchers { + rv += s.DocumentMatchPoolSize() + } + return rv +} diff --git a/search/searcher/search_conjunction_nested.go b/search/searcher/search_conjunction_nested.go new file mode 100644 index 0000000..9ab18da --- /dev/null +++ b/search/searcher/search_conjunction_nested.go @@ -0,0 +1,480 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "fmt" + "math" + "reflect" + "slices" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeNestedConjunctionSearcher int + +func init() { + var ncs NestedConjunctionSearcher + reflectStaticSizeNestedConjunctionSearcher = int(reflect.TypeOf(ncs).Size()) +} + +type NestedConjunctionSearcher struct { + nestedReader index.NestedReader + searchers []search.Searcher + queryNorm float64 + currs []*search.DocumentMatch + currAncestors [][]index.AncestorID + currKeys []index.AncestorID + initialized bool + joinIdx int + options search.SearcherOptions + docQueue *CoalesceQueue + // reusable ID buffer for Advance() calls + advanceID index.IndexInternalID + // reusable buffer for Advance() calls + ancestors []index.AncestorID +} + +func NewNestedConjunctionSearcher(ctx context.Context, indexReader index.IndexReader, + searchers []search.Searcher, joinIdx int, options search.SearcherOptions) (search.Searcher, error) { + + var nr index.NestedReader + var ok bool + if nr, ok = indexReader.(index.NestedReader); !ok { + return nil, fmt.Errorf("indexReader does not support nested documents") + } + + // build our searcher + rv := NestedConjunctionSearcher{ + nestedReader: nr, + options: options, + searchers: searchers, + currs: make([]*search.DocumentMatch, len(searchers)), + currAncestors: make([][]index.AncestorID, len(searchers)), + currKeys: make([]index.AncestorID, len(searchers)), + joinIdx: joinIdx, + docQueue: NewCoalesceQueue(), + } + rv.computeQueryNorm() + + return &rv, nil +} + +func (s *NestedConjunctionSearcher) computeQueryNorm() { + // first calculate sum of squared weights + sumOfSquaredWeights := 0.0 + for _, searcher := range s.searchers { + sumOfSquaredWeights += searcher.Weight() + } + // now compute query norm from this + s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) + // finally tell all the downstream searchers the norm + for _, searcher := range s.searchers { + searcher.SetQueryNorm(s.queryNorm) + } +} + +func (s *NestedConjunctionSearcher) Size() int { + sizeInBytes := reflectStaticSizeNestedConjunctionSearcher + size.SizeOfPtr + + for _, entry := range s.searchers { + sizeInBytes += entry.Size() + } + + for _, entry := range s.currs { + if entry != nil { + sizeInBytes += entry.Size() + } + } + + return sizeInBytes +} + +func (s *NestedConjunctionSearcher) Weight() float64 { + var rv float64 + for _, searcher := range s.searchers { + rv += searcher.Weight() + } + return rv +} + +func (s *NestedConjunctionSearcher) SetQueryNorm(qnorm float64) { + for _, searcher := range s.searchers { + searcher.SetQueryNorm(qnorm) + } +} + +func (s *NestedConjunctionSearcher) Count() uint64 { + // for now return a worst case + var sum uint64 + for _, searcher := range s.searchers { + sum += searcher.Count() + } + return sum +} + +func (s *NestedConjunctionSearcher) Close() (rv error) { + for _, searcher := range s.searchers { + err := searcher.Close() + if err != nil && rv == nil { + rv = err + } + } + return rv +} + +func (s *NestedConjunctionSearcher) Min() int { + return 0 +} + +func (s *NestedConjunctionSearcher) DocumentMatchPoolSize() int { + rv := len(s.currs) + for _, s := range s.searchers { + rv += s.DocumentMatchPoolSize() + } + return rv +} + +func (s *NestedConjunctionSearcher) initialize(ctx *search.SearchContext) (bool, error) { + var err error + for i, searcher := range s.searchers { + if s.currs[i] != nil { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = searcher.Next(ctx) + if err != nil { + return false, err + } + if s.currs[i] == nil { + // one of the searchers is exhausted, so we are done + return true, nil + } + // get the ancestry chain for this match + s.currAncestors[i], err = s.nestedReader.Ancestors(s.currs[i].IndexInternalID, s.currAncestors[i][:0]) + if err != nil { + return false, err + } + // check if the ancestry chain is > joinIdx, if not we reset the joinIdx + // to the minimum possible value across all searchers, ideally this will be + // done in query construction time itself, by using the covering depth across + // all sub-queries, but we do this here as a fallback + if s.joinIdx >= len(s.currAncestors[i]) { + s.joinIdx = len(s.currAncestors[i]) - 1 + } + } + // build currKeys for each searcher, do it here as we may have adjusted joinIdx + for i := range s.searchers { + s.currKeys[i] = ancestorFromRoot(s.currAncestors[i], s.joinIdx) + } + s.initialized = true + return false, nil +} + +func (s *NestedConjunctionSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + // initialize on first call to Next, by getting first match + // from each searcher and their ancestry chains + if !s.initialized { + done, err := s.initialize(ctx) + if err != nil { + return nil, err + } + if done { + return nil, nil + } + } + // check if the docQueue has any buffered matches + if s.docQueue.Len() > 0 { + return s.docQueue.Dequeue(ctx), nil + } + // now enter the main alignment loop + n := len(s.searchers) +OUTER: + for { + // pick the pivot searcher with the highest key (ancestor at joinIdx level) + if s.currs[0] == nil { + return nil, nil + } + maxKey := s.currKeys[0] + for i := 1; i < n; i++ { + // currs[i] is nil means one of the searchers is exhausted + if s.currs[i] == nil { + return nil, nil + } + currKey := s.currKeys[i] + if maxKey.Compare(currKey) < 0 { + maxKey = currKey + } + } + // store maxkey as advanceID only once only if needed + var advanceID index.IndexInternalID + // flag to track if all searchers are aligned + var aligned bool = true + // now try to align all other searchers to the + // we check if the a searchers key matches maxKey + // if not, we advance the pivot searcher to maxKey + // else do nothing and move to the next searcher + for i := 0; i < n; i++ { + cmp := s.currKeys[i].Compare(maxKey) + if cmp < 0 { + // not aligned, so advance this searcher to maxKey + // convert maxKey to advanceID only once + if advanceID == nil { + advanceID = s.toAdvanceID(maxKey) + } + var err error + ctx.DocumentMatchPool.Put(s.currs[i]) + s.currs[i], err = s.searchers[i].Advance(ctx, advanceID) + if err != nil { + return nil, err + } + if s.currs[i] == nil { + // one of the searchers is exhausted, so we are done + return nil, nil + } + // recalc ancestors + s.currAncestors[i], err = s.nestedReader.Ancestors(s.currs[i].IndexInternalID, s.currAncestors[i][:0]) + if err != nil { + return nil, err + } + // recalc key + s.currKeys[i] = ancestorFromRoot(s.currAncestors[i], s.joinIdx) + // recalc cmp + cmp = s.currKeys[i].Compare(maxKey) + } + if cmp != 0 { + // not aligned + aligned = false + } + } + // now check if all the searchers are aligned at the same maxKey + // if they are not aligned, we need to restart the loop of picking + // the pivot searcher with the highest key + if !aligned { + continue OUTER + } + // if we are here, all the searchers are aligned at maxKey + // now we need to buffer all the intermediate matches for every + // searcher at this key, until either the searcher's key changes + // or the searcher is exhausted + var err error + for i := 0; i < n; i++ { + for { + // buffer the current match + s.docQueue.Enqueue(s.currs[i]) + // advance to next match + s.currs[i], err = s.searchers[i].Next(ctx) + if err != nil { + return nil, err + } + if s.currs[i] == nil { + // searcher exhausted, break out + break + } + // recalc ancestors + s.currAncestors[i], err = s.nestedReader.Ancestors(s.currs[i].IndexInternalID, s.currAncestors[i][:0]) + if err != nil { + return nil, err + } + // recalc key + s.currKeys[i] = ancestorFromRoot(s.currAncestors[i], s.joinIdx) + // check if key has changed + if !s.currKeys[i].Equals(maxKey) { + // key changed, break out + break + } + } + } + // finalize the docQueue for dequeueing + s.docQueue.Finalize() + // finally return the first buffered match + return s.docQueue.Dequeue(ctx), nil + } +} + +// ancestorFromRoot gets the AncestorID at the given position from the root +// if pos is 0, it returns the root AncestorID, and so on +func ancestorFromRoot(ancestors []index.AncestorID, pos int) index.AncestorID { + return ancestors[len(ancestors)-pos-1] +} + +// toAdvanceID converts an AncestorID to IndexInternalID, reusing the advanceID buffer. +// The returned ID is safe to pass to Advance() since Advance() never retains references. +func (s *NestedConjunctionSearcher) toAdvanceID(key index.AncestorID) index.IndexInternalID { + // Reset length to 0 while preserving capacity for buffer reuse + s.advanceID = s.advanceID[:0] + // Convert key to IndexInternalID, reusing the underlying buffer + s.advanceID = key.ToIndexInternalID(s.advanceID) + return s.advanceID +} + +func (s *NestedConjunctionSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + if !s.initialized { + done, err := s.initialize(ctx) + if err != nil { + return nil, err + } + if done { + return nil, nil + } + } + // first check if the docQueue has any buffered matches + // if so we first check if any of them can satisfy the Advance(ID) + for s.docQueue.Len() > 0 { + dm := s.docQueue.Dequeue(ctx) + if dm.IndexInternalID.Compare(ID) >= 0 { + return dm, nil + } + // otherwise recycle this match + ctx.DocumentMatchPool.Put(dm) + } + var err error + // now we first get the ancestry chain for the given ID + s.ancestors, err = s.nestedReader.Ancestors(ID, s.ancestors[:0]) + if err != nil { + return nil, err + } + // we now follow the the following logic for each searcher: + // let S be the length of the ancestry chain for the searcher + // let I be the length of the ancestry chain for the given ID + // 1. if S > I: + // then we just Advance() the searcher to the given ID if required + // 2. else if S <= I: + // then we get the AncestorID at position (S - 1) from the root of + // the given ID's ancestry chain, and Advance() the searcher to + // it if required + for i, searcher := range s.searchers { + if s.currs[i] == nil { + return nil, nil // already exhausted, nothing to do + } + var targetID index.IndexInternalID + S := len(s.currAncestors[i]) + I := len(s.ancestors) + if S > I { + // case 1: S > I + targetID = ID + } else { + // case 2: S <= I + targetID = s.toAdvanceID(ancestorFromRoot(s.ancestors, S-1)) + } + if s.currs[i].IndexInternalID.Compare(targetID) < 0 { + // need to advance this searcher + ctx.DocumentMatchPool.Put(s.currs[i]) + s.currs[i], err = searcher.Advance(ctx, targetID) + if err != nil { + return nil, err + } + if s.currs[i] == nil { + // one of the searchers is exhausted, so we are done + return nil, nil + } + // recalc ancestors + s.currAncestors[i], err = s.nestedReader.Ancestors(s.currs[i].IndexInternalID, s.currAncestors[i][:0]) + if err != nil { + return nil, err + } + // recalc key + s.currKeys[i] = ancestorFromRoot(s.currAncestors[i], s.joinIdx) + } + } + // we need to call Next() in a loop until we reach or exceed the given ID + // the Next() call basically gives us a match that is aligned correctly, but + // if joinIdx < I, we can have multiple matches for the same joinIdx ancestor + // and they may be < ID, so we need to loop + for { + next, err := s.Next(ctx) + if err != nil { + return nil, err + } + if next == nil { + return nil, nil + } + if next.IndexInternalID.Compare(ID) >= 0 { + return next, nil + } + ctx.DocumentMatchPool.Put(next) + } +} + +// ------------------------------------------------------------------------------------------ +type CoalesceQueue struct { + order []*search.DocumentMatch // queue of DocumentMatch +} + +func NewCoalesceQueue() *CoalesceQueue { + cq := &CoalesceQueue{ + order: make([]*search.DocumentMatch, 0), + } + return cq +} + +// Enqueue appends the given DocumentMatch to the queue. Coalescing of duplicates +// is deferred until Dequeue, after Finalize has sorted items by IndexInternalID. +func (cq *CoalesceQueue) Enqueue(it *search.DocumentMatch) { + // append to order slice (this is a stack) + cq.order = append(cq.order, it) +} + +// Finalize prepares the queue for dequeue operations by sorting the items based on +// their IndexInternalID values. This MUST be called before any Dequeue operations, +// and after all Enqueue operations are complete. The sort is done in descending order +// so that dequeueing will basically be popping from the end of the slice, allowing for +// slice reuse. +func (cq *CoalesceQueue) Finalize() { + slices.SortFunc(cq.order, func(a, b *search.DocumentMatch) int { + return b.IndexInternalID.Compare(a.IndexInternalID) + }) +} + +// Dequeue removes and returns the next DocumentMatch in sorted order, merging any +// consecutive duplicates. Merged items are recycled via ctx.DocumentMatchPool. +// Returns nil when the queue is empty. +func (cq *CoalesceQueue) Dequeue(ctx *search.SearchContext) *search.DocumentMatch { + if cq.Len() == 0 { + return nil + } + + // pop from end of slice + rv := cq.order[len(cq.order)-1] + cq.order = cq.order[:len(cq.order)-1] + + // merge duplicates + for cq.Len() > 0 { + // peek at next item + next := cq.order[len(cq.order)-1] + if !rv.IndexInternalID.Equals(next.IndexInternalID) { + // different ID, stop merging + break + } + // pop the next item + cq.order = cq.order[:len(cq.order)-1] + // same ID, merge + rv.Score += next.Score + rv.Expl = rv.Expl.MergeWith(next.Expl) + rv.FieldTermLocations = search.MergeFieldTermLocationsFromMatch( + rv.FieldTermLocations, next) + // recycle the merged item + ctx.DocumentMatchPool.Put(next) + } + + return rv +} + +// Len returns the number of DocumentMatch items currently in the queue. +func (cq *CoalesceQueue) Len() int { + return len(cq.order) +} diff --git a/search/searcher/search_conjunction_test.go b/search/searcher/search_conjunction_test.go new file mode 100644 index 0000000..27d2405 --- /dev/null +++ b/search/searcher/search_conjunction_test.go @@ -0,0 +1,438 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "os" + "strings" + "testing" + + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func TestConjunctionSearch(t *testing.T) { + twoDocIndexReader, err := twoDocIndex.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := twoDocIndexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + explainTrue := search.SearcherOptions{Explain: true} + + // test 0 + beerTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "beer", "desc", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + martyTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "marty", "name", 5.0, explainTrue) + if err != nil { + t.Fatal(err) + } + beerAndMartySearcher, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher, martyTermSearcher}, explainTrue) + if err != nil { + t.Fatal(err) + } + + // test 1 + angstTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "angst", "desc", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + beerTermSearcher2, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "beer", "desc", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + angstAndBeerSearcher, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{angstTermSearcher, beerTermSearcher2}, explainTrue) + if err != nil { + t.Fatal(err) + } + + // test 2 + beerTermSearcher3, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "beer", "desc", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + jackTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "jack", "name", 5.0, explainTrue) + if err != nil { + t.Fatal(err) + } + beerAndJackSearcher, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher3, jackTermSearcher}, explainTrue) + if err != nil { + t.Fatal(err) + } + + // test 3 + beerTermSearcher4, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "beer", "desc", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + misterTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "mister", "title", 5.0, explainTrue) + if err != nil { + t.Fatal(err) + } + beerAndMisterSearcher, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher4, misterTermSearcher}, explainTrue) + if err != nil { + t.Fatal(err) + } + + // test 4 + couchbaseTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "couchbase", "street", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + misterTermSearcher2, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "mister", "title", 5.0, explainTrue) + if err != nil { + t.Fatal(err) + } + couchbaseAndMisterSearcher, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{couchbaseTermSearcher, misterTermSearcher2}, explainTrue) + if err != nil { + t.Fatal(err) + } + + // test 5 + beerTermSearcher5, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "beer", "desc", 5.0, explainTrue) + if err != nil { + t.Fatal(err) + } + couchbaseTermSearcher2, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "couchbase", "street", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + misterTermSearcher3, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "mister", "title", 5.0, explainTrue) + if err != nil { + t.Fatal(err) + } + couchbaseAndMisterSearcher2, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{couchbaseTermSearcher2, misterTermSearcher3}, explainTrue) + if err != nil { + t.Fatal(err) + } + beerAndCouchbaseAndMisterSearcher, err := NewConjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{beerTermSearcher5, couchbaseAndMisterSearcher2}, explainTrue) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + searcher search.Searcher + results []*search.DocumentMatch + }{ + { + searcher: beerAndMartySearcher, + results: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("1"), + Score: 2.0097428702814377, + }, + }, + }, + { + searcher: angstAndBeerSearcher, + results: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("2"), + Score: 1.0807601687084403, + }, + }, + }, + { + searcher: beerAndJackSearcher, + results: []*search.DocumentMatch{}, + }, + { + searcher: beerAndMisterSearcher, + results: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("2"), + Score: 1.2877980334016337, + }, + { + IndexInternalID: index.IndexInternalID("3"), + Score: 1.2877980334016337, + }, + }, + }, + { + searcher: couchbaseAndMisterSearcher, + results: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("2"), + Score: 1.4436599157093672, + }, + }, + }, + { + searcher: beerAndCouchbaseAndMisterSearcher, + results: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("2"), + Score: 1.441614953806971, + }, + }, + }, + } + + for testIndex, test := range tests { + defer func() { + err := test.searcher.Close() + if err != nil { + t.Fatal(err) + } + }() + + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(10, 0), + } + next, err := test.searcher.Next(ctx) + i := 0 + for err == nil && next != nil { + if i < len(test.results) { + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) + } + if !scoresCloseEnough(next.Score, test.results[i].Score) { + t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) + t.Logf("scoring explanation: %s", next.Expl) + } + } + next, err = test.searcher.Next(ctx) + i++ + } + if err != nil { + t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) + } + if len(test.results) != i { + t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) + } + } +} + +type compositeSearchOptimizationTest struct { + fieldTerms []string + expectEmpty string +} + +func TestScorchCompositeSearchOptimizations(t *testing.T) { + dir, _ := os.MkdirTemp("", "scorchTwoDoc") + defer func() { + _ = os.RemoveAll(dir) + }() + + twoDocIndex := initTwoDocScorch(dir) + + twoDocIndexReader, err := twoDocIndex.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := twoDocIndexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + tests := []compositeSearchOptimizationTest{ + { + fieldTerms: []string{}, + expectEmpty: "conjunction,disjunction", + }, + { + fieldTerms: []string{"name:marty"}, + expectEmpty: "", + }, + { + fieldTerms: []string{"name:marty", "desc:beer"}, + expectEmpty: "", + }, + { + fieldTerms: []string{"name:marty", "name:marty"}, + expectEmpty: "", + }, + { + fieldTerms: []string{"name:marty", "desc:beer", "title:mister", "street:couchbase"}, + expectEmpty: "conjunction", + }, + { + fieldTerms: []string{"name:steve", "desc:beer", "title:mister", "street:couchbase"}, + expectEmpty: "", + }, + + { + fieldTerms: []string{"name:NotARealName"}, + expectEmpty: "conjunction,disjunction", + }, + { + fieldTerms: []string{"name:NotARealName", "name:marty"}, + expectEmpty: "conjunction", + }, + { + fieldTerms: []string{"name:NotARealName", "name:marty", "desc:beer"}, + expectEmpty: "conjunction", + }, + { + fieldTerms: []string{"name:NotARealName", "name:marty", "name:marty"}, + expectEmpty: "conjunction", + }, + { + fieldTerms: []string{"name:NotARealName", "name:marty", "desc:beer", "title:mister", "street:couchbase"}, + expectEmpty: "conjunction", + }, + } + + // The theme of this unit test is that given one of the above + // search test cases -- no matter what searcher options we + // provide, across either conjunctions or disjunctions, whether we + // have optimizations that are enabled or disabled, the set of doc + // ID's from the search results from any of those combinations + // should be the same. + searcherOptionsToCompare := []search.SearcherOptions{ + {}, + {Explain: true}, + {IncludeTermVectors: true}, + {IncludeTermVectors: true, Explain: true}, + {Score: "none"}, + {Score: "none", IncludeTermVectors: true}, + {Score: "none", IncludeTermVectors: true, Explain: true}, + {Score: "none", Explain: true}, + } + + testScorchCompositeSearchOptimizations(t, twoDocIndexReader, tests, + searcherOptionsToCompare, "conjunction") + + testScorchCompositeSearchOptimizations(t, twoDocIndexReader, tests, + searcherOptionsToCompare, "disjunction") +} + +func testScorchCompositeSearchOptimizations(t *testing.T, indexReader index.IndexReader, + tests []compositeSearchOptimizationTest, + searcherOptionsToCompare []search.SearcherOptions, + compositeKind string, +) { + for testi := range tests { + resultsToCompare := map[string]bool{} + + testScorchCompositeSearchOptimizationsHelper(t, indexReader, tests, testi, + searcherOptionsToCompare, compositeKind, false, resultsToCompare) + + testScorchCompositeSearchOptimizationsHelper(t, indexReader, tests, testi, + searcherOptionsToCompare, compositeKind, true, resultsToCompare) + } +} + +func testScorchCompositeSearchOptimizationsHelper( + t *testing.T, indexReader index.IndexReader, + tests []compositeSearchOptimizationTest, testi int, + searcherOptionsToCompare []search.SearcherOptions, + compositeKind string, allowOptimizations bool, resultsToCompare map[string]bool, +) { + // Save the global allowed optimization settings to restore later. + optimizeConjunction := scorch.OptimizeConjunction + optimizeConjunctionUnadorned := scorch.OptimizeConjunctionUnadorned + optimizeDisjunctionUnadorned := scorch.OptimizeDisjunctionUnadorned + optimizeDisjunctionUnadornedMinChildCardinality := scorch.OptimizeDisjunctionUnadornedMinChildCardinality + + scorch.OptimizeConjunction = allowOptimizations + scorch.OptimizeConjunctionUnadorned = allowOptimizations + scorch.OptimizeDisjunctionUnadorned = allowOptimizations + + if allowOptimizations { + scorch.OptimizeDisjunctionUnadornedMinChildCardinality = uint64(0) + } + + defer func() { + scorch.OptimizeConjunction = optimizeConjunction + scorch.OptimizeConjunctionUnadorned = optimizeConjunctionUnadorned + scorch.OptimizeDisjunctionUnadorned = optimizeDisjunctionUnadorned + scorch.OptimizeDisjunctionUnadornedMinChildCardinality = optimizeDisjunctionUnadornedMinChildCardinality + }() + + test := tests[testi] + + for searcherOptionsI, searcherOptions := range searcherOptionsToCompare { + // Construct the leaf term searchers. + var searchers []search.Searcher + + for _, fieldTerm := range test.fieldTerms { + ft := strings.Split(fieldTerm, ":") + field := ft[0] + term := ft[1] + + searcher, err := NewTermSearcher(context.TODO(), indexReader, term, field, 1.0, searcherOptions) + if err != nil { + t.Fatal(err) + } + + searchers = append(searchers, searcher) + } + + // Construct the composite searcher. + var cs search.Searcher + var err error + if compositeKind == "conjunction" { + cs, err = NewConjunctionSearcher(context.TODO(), indexReader, searchers, searcherOptions) + } else { + cs, err = NewDisjunctionSearcher(context.TODO(), indexReader, searchers, 0, searcherOptions) + } + if err != nil { + t.Fatal(err) + } + + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(10, 0), + } + + next, err := cs.Next(ctx) + i := 0 + for err == nil && next != nil { + docID, err := indexReader.ExternalID(next.IndexInternalID) + if err != nil { + t.Fatal(err) + } + + if searcherOptionsI == 0 && allowOptimizations == false { + resultsToCompare[string(docID)] = true + } else { + if !resultsToCompare[string(docID)] { + t.Errorf("missing %s", string(docID)) + } + } + + next, err = cs.Next(ctx) + if err != nil { + t.Fatalf("error iterating searcher: %v", err) + } + + i++ + } + + if i != len(resultsToCompare) { + t.Errorf("mismatched count, %d vs %d", i, len(resultsToCompare)) + } + + if i == 0 && !strings.Contains(test.expectEmpty, compositeKind) { + t.Errorf("testi: %d, compositeKind: %s, allowOptimizations: %t,"+ + " searcherOptionsI: %d, searcherOptions: %#v,"+ + " expected some results but got no results on test: %#v", + testi, compositeKind, allowOptimizations, + searcherOptionsI, searcherOptions, test) + } + } +} diff --git a/search/searcher/search_custom_fields.go b/search/searcher/search_custom_fields.go new file mode 100644 index 0000000..4fb439c --- /dev/null +++ b/search/searcher/search_custom_fields.go @@ -0,0 +1,127 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "time" + + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +// loadDocValuesOnHitWithTypes resolves hit.ID and returns the hit's doc-value +// fields in a new map. It does not touch hit.Fields, so these UDF-input fields +// can't leak into SearchRequest.Fields; the caller owns the map. Returns nil +// when dvReader is nil. +// +// fieldTypes maps a field name to its type: datetime fields decode to an +// RFC3339Nano string, numeric fields to float64. nil treats all prefix-coded +// values as numeric. +func loadDocValuesOnHitWithTypes(hit *search.DocumentMatch, dvReader index.DocValueReader, + r index.IndexReader, fieldTypes map[string]string) (map[string]interface{}, error) { + // Always resolve external ID so the callback can read hit.ID. + if hit.ID == "" && r != nil { + extID, err := r.ExternalID(hit.IndexInternalID) + if err != nil { + return nil, err + } + hit.ID = extID + } + + if dvReader == nil { + return nil, nil + } + + fields := make(map[string]interface{}) + err := dvReader.VisitDocValues(hit.IndexInternalID, func(field string, term []byte) { + value := decodeDocValueTerm(term, fieldTypes[field]) + if value != nil { + addFieldValueToMap(fields, field, value) + } + }) + if err != nil { + return nil, err + } + + return fields, nil +} + +// addFieldValueToMap adds value under name, mirroring DocumentMatch.AddFieldValue: +// a repeated field name accumulates into a []interface{} slice. +func addFieldValueToMap(fields map[string]interface{}, name string, value interface{}) { + existingVal, ok := fields[name] + if !ok { + fields[name] = value + return + } + + valSlice, ok := existingVal.([]interface{}) + if ok { + valSlice = append(valSlice, value) + } else { + valSlice = []interface{}{existingVal, value} + } + fields[name] = valSlice +} + +// decodeDocValueTerm converts raw doc value bytes into a typed Go value. +// Numeric fields are prefix-coded int64s (only shift-0 terms carry values). +// Boolean fields are stored as "T" or "F". +// Everything else (text/keyword) is returned as a string. +// +// fieldType is the mapping type string for the field (e.g. "datetime", +// "number"). When fieldType is "datetime", the prefix-coded int64 is +// treated as raw nanoseconds (time.UnixNano()) and converted to a UTC +// RFC3339Nano-formatted string. For numeric fields the int64 is decoded via +// Int64ToFloat64 (IEEE 754 bit reinterpretation). +func decodeDocValueTerm(term []byte, fieldType string) interface{} { + if len(term) == 0 { + return nil + } + + // Check if it's a prefix-coded numeric term. + if valid, shift := numeric.ValidPrefixCodedTermBytes(term); valid { + // Only shift-0 terms carry the actual value. + if shift != 0 { + return nil + } + i64, err := numeric.PrefixCoded(term).Int64() + if err != nil { + return nil + } + if fieldType == "datetime" { + // Datetime doc values store time.UnixNano() directly as int64. + // Convert back to a formatted string so callbacks (including + // JS UDFs) receive a human-readable date like "2022-03-10T00:00:00Z". + return time.Unix(0, i64).UTC().Format(time.RFC3339Nano) + } + // Numeric float64 fields use Float64ToInt64 bit manipulation encoding. + return numeric.Int64ToFloat64(i64) + } + + // Boolean fields are stored as "T" or "F". + if len(term) == 1 { + if term[0] == 'T' { + return true + } + if term[0] == 'F' { + return false + } + } + + // Default: text/keyword — return as string. + return string(term) +} diff --git a/search/searcher/search_custom_filter.go b/search/searcher/search_custom_filter.go new file mode 100644 index 0000000..1a0b2d8 --- /dev/null +++ b/search/searcher/search_custom_filter.go @@ -0,0 +1,146 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "reflect" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeCustomFilterSearcher int + +func init() { + var cfs CustomFilterSearcher + reflectStaticSizeCustomFilterSearcher = int(reflect.TypeOf(cfs).Size()) +} + +// CustomFilterFunc decides whether a hit (with doc-value fields populated) +// should be kept. It receives the search's context so a long-running callback +// (e.g. a JS UDF) can honor the query deadline/cancellation. A non-nil error +// aborts the search so the failure can be surfaced to the caller rather than +// silently dropping the hit. +type CustomFilterFunc func(ctx context.Context, d *search.DocumentMatch) (bool, error) + +// CustomFilterSearcher wraps a child searcher, optionally loads doc values +// into each DocumentMatch, then applies a CustomFilterFunc to decide whether +// to keep the hit. Unlike FilteringSearcher this variant is purpose-built for +// custom queries that need field values at callback time. +type CustomFilterSearcher struct { + ctx context.Context + child search.Searcher + accept CustomFilterFunc + dvReader index.DocValueReader + indexReader index.IndexReader + fieldTypes map[string]string +} + +func NewCustomFilterSearcher(ctx context.Context, child search.Searcher, + filter CustomFilterFunc, dvReader index.DocValueReader, + indexReader index.IndexReader, + fieldTypes map[string]string) *CustomFilterSearcher { + return &CustomFilterSearcher{ + ctx: ctx, + child: child, + accept: filter, + dvReader: dvReader, + indexReader: indexReader, + fieldTypes: fieldTypes, + } +} + +func (f *CustomFilterSearcher) Size() int { + return reflectStaticSizeCustomFilterSearcher + size.SizeOfPtr + + f.child.Size() +} + +func (f *CustomFilterSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + next, err := f.child.Next(ctx) + for next != nil && err == nil { + // Put the loaded fields on the hit only for the callback, so UDF-input + // fields don't override SearchRequest.Fields in the response. + udfFields, lerr := loadDocValuesOnHitWithTypes(next, f.dvReader, f.indexReader, f.fieldTypes) + if lerr != nil { + return nil, lerr + } + priorFields := next.Fields + next.Fields = udfFields + keep, ferr := f.accept(f.ctx, next) + next.Fields = priorFields + if ferr != nil { + return nil, ferr + } + if keep { + return next, nil + } + ctx.DocumentMatchPool.Put(next) + next, err = f.child.Next(ctx) + } + return nil, err +} + +func (f *CustomFilterSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + adv, err := f.child.Advance(ctx, ID) + if err != nil { + return nil, err + } + if adv == nil { + return nil, nil + } + // See Next: put the loaded fields on the hit only for the callback. + udfFields, lerr := loadDocValuesOnHitWithTypes(adv, f.dvReader, f.indexReader, f.fieldTypes) + if lerr != nil { + return nil, lerr + } + priorFields := adv.Fields + adv.Fields = udfFields + keep, ferr := f.accept(f.ctx, adv) + adv.Fields = priorFields + if ferr != nil { + return nil, ferr + } + if keep { + return adv, nil + } + ctx.DocumentMatchPool.Put(adv) + return f.Next(ctx) +} + +func (f *CustomFilterSearcher) Close() error { + return f.child.Close() +} + +func (f *CustomFilterSearcher) Weight() float64 { + return f.child.Weight() +} + +func (f *CustomFilterSearcher) SetQueryNorm(n float64) { + f.child.SetQueryNorm(n) +} + +func (f *CustomFilterSearcher) Count() uint64 { + return f.child.Count() +} + +func (f *CustomFilterSearcher) Min() int { + return f.child.Min() +} + +func (f *CustomFilterSearcher) DocumentMatchPoolSize() int { + return f.child.DocumentMatchPoolSize() +} diff --git a/search/searcher/search_custom_score.go b/search/searcher/search_custom_score.go new file mode 100644 index 0000000..7e01ea6 --- /dev/null +++ b/search/searcher/search_custom_score.go @@ -0,0 +1,158 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "reflect" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeCustomScoreSearcher int + +func init() { + var sfs CustomScoreSearcher + reflectStaticSizeCustomScoreSearcher = int(reflect.TypeOf(sfs).Size()) +} + +// CustomScoreFunc defines a function which can mutate document scores. It +// receives the search's context so a long-running callback (e.g. a JS UDF) can +// honor the query deadline/cancellation. A non-nil error aborts the search so +// the failure can be surfaced to the caller rather than silently falling back +// to the original score. +type CustomScoreFunc func(ctx context.Context, d *search.DocumentMatch) (float64, error) + +// CustomScoreSearcher wraps any other searcher, optionally loads doc values +// into each DocumentMatch, then mutates the score using the supplied +// CustomScoreFunc. +type CustomScoreSearcher struct { + ctx context.Context + child search.Searcher + mutate CustomScoreFunc + dvReader index.DocValueReader + indexReader index.IndexReader + fieldTypes map[string]string + explain bool +} + +func NewCustomScoreSearcher(ctx context.Context, s search.Searcher, mutate CustomScoreFunc, + dvReader index.DocValueReader, indexReader index.IndexReader, + fieldTypes map[string]string, explain bool) *CustomScoreSearcher { + return &CustomScoreSearcher{ + ctx: ctx, + child: s, + mutate: mutate, + dvReader: dvReader, + indexReader: indexReader, + fieldTypes: fieldTypes, + explain: explain, + } +} + +// applyScore mutates the score on the hit and, when explain is enabled, +// replaces the explanation with a single node describing the custom score +// result. A non-nil error from the score function is returned so the caller +// can abort the search. +func (f *CustomScoreSearcher) applyScore(d *search.DocumentMatch) error { + score, err := f.mutate(f.ctx, d) + if err != nil { + return err + } + d.Score = score + if f.explain { + d.Expl = &search.Explanation{ + Value: d.Score, + Message: "custom_score function result", + } + } + return nil +} + +func (f *CustomScoreSearcher) Size() int { + return reflectStaticSizeCustomScoreSearcher + size.SizeOfPtr + + f.child.Size() +} + +func (f *CustomScoreSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + next, err := f.child.Next(ctx) + if err != nil { + return nil, err + } + if next != nil { + // Put the loaded fields on the hit only for scoring, so UDF-input fields + // don't override SearchRequest.Fields in the response. + udfFields, lerr := loadDocValuesOnHitWithTypes(next, f.dvReader, f.indexReader, f.fieldTypes) + if lerr != nil { + return nil, lerr + } + priorFields := next.Fields + next.Fields = udfFields + serr := f.applyScore(next) + next.Fields = priorFields + if serr != nil { + return nil, serr + } + } + return next, nil +} + +func (f *CustomScoreSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + adv, err := f.child.Advance(ctx, ID) + if err != nil { + return nil, err + } + if adv != nil { + // See Next: put the loaded fields on the hit only for scoring. + udfFields, lerr := loadDocValuesOnHitWithTypes(adv, f.dvReader, f.indexReader, f.fieldTypes) + if lerr != nil { + return nil, lerr + } + priorFields := adv.Fields + adv.Fields = udfFields + serr := f.applyScore(adv) + adv.Fields = priorFields + if serr != nil { + return nil, serr + } + } + return adv, nil +} + +func (f *CustomScoreSearcher) Close() error { + return f.child.Close() +} + +func (f *CustomScoreSearcher) Weight() float64 { + return f.child.Weight() +} + +func (f *CustomScoreSearcher) SetQueryNorm(n float64) { + f.child.SetQueryNorm(n) +} + +func (f *CustomScoreSearcher) Count() uint64 { + return f.child.Count() +} + +func (f *CustomScoreSearcher) Min() int { + return f.child.Min() +} + +func (f *CustomScoreSearcher) DocumentMatchPoolSize() int { + return f.child.DocumentMatchPoolSize() +} diff --git a/search/searcher/search_disjunction.go b/search/searcher/search_disjunction.go new file mode 100644 index 0000000..6c15ae6 --- /dev/null +++ b/search/searcher/search_disjunction.go @@ -0,0 +1,142 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "fmt" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +// DisjunctionMaxClauseCount is a compile time setting that applications can +// adjust to non-zero value to cause the DisjunctionSearcher to return an +// error instead of executing searches when the size exceeds this value. +var DisjunctionMaxClauseCount = 0 + +// DisjunctionHeapTakeover is a compile time setting that applications can +// adjust to control when the DisjunctionSearcher will switch from a simple +// slice implementation to a heap implementation. +var DisjunctionHeapTakeover = 10 + +func NewDisjunctionSearcher(ctx context.Context, indexReader index.IndexReader, + qsearchers []search.Searcher, min float64, options search.SearcherOptions) ( + search.Searcher, error) { + return newDisjunctionSearcher(ctx, indexReader, qsearchers, min, options, true) +} + +func optionsDisjunctionOptimizable(options search.SearcherOptions) bool { + rv := options.Score == "none" && !options.IncludeTermVectors + return rv +} + +func newDisjunctionSearcher(ctx context.Context, indexReader index.IndexReader, + qsearchers []search.Searcher, min float64, options search.SearcherOptions, + limit bool) (search.Searcher, error) { + + var disjOverKNN bool + if ctx != nil { + disjOverKNN, _ = ctx.Value(search.IncludeScoreBreakdownKey).(bool) + } + if disjOverKNN { + // The KNN Searcher optimization is a necessary pre-req for the KNN Searchers, + // not an optional optimization like for, say term searchers. + // It's an optimization to repeat search an open vector index when applicable, + // rather than individually opening and searching a vector index. + err := optimizeKNN(ctx, indexReader, qsearchers) + if err != nil { + return nil, err + } + } else { + // attempt the "unadorned" disjunction optimization only when we + // do not need extra information like freq-norm's or term vectors + // and the requested min is simple + if len(qsearchers) > 1 && min <= 1 && + optionsDisjunctionOptimizable(options) { + rv, err := optimizeCompositeSearcher(ctx, "disjunction:unadorned", + indexReader, qsearchers, options) + if err != nil { + return nil, err + } + if rv != nil { + for _, s := range qsearchers { + _ = s.Close() + } + return rv, nil + } + } + } + + if len(qsearchers) > DisjunctionHeapTakeover { + return newDisjunctionHeapSearcher(ctx, indexReader, qsearchers, min, options, + limit) + } + return newDisjunctionSliceSearcher(ctx, indexReader, qsearchers, min, options, + limit) +} + +func optimizeCompositeSearcher(ctx context.Context, optimizationKind string, + indexReader index.IndexReader, qsearchers []search.Searcher, + options search.SearcherOptions) (search.Searcher, error) { + var octx index.OptimizableContext + + for _, searcher := range qsearchers { + o, ok := searcher.(index.Optimizable) + if !ok { + return nil, nil + } + + var err error + octx, err = o.Optimize(optimizationKind, octx) + if err != nil { + return nil, err + } + + if octx == nil { + return nil, nil + } + } + + optimized, err := octx.Finish() + if err != nil || optimized == nil { + return nil, err + } + + tfr, ok := optimized.(index.TermFieldReader) + if !ok { + return nil, nil + } + + termSearcher, err := newTermSearcherFromReader(ctx, indexReader, tfr, + []byte(optimizationKind), "*", 1.0, options) + if err != nil || termSearcher == nil { + return nil, err + } + + return termSearcher, err +} + +func tooManyClauses(count int) bool { + if DisjunctionMaxClauseCount != 0 && count > DisjunctionMaxClauseCount { + return true + } + return false +} + +func tooManyClausesErr(field string, count int) error { + return fmt.Errorf("TooManyClauses over field: `%s` [%d > maxClauseCount,"+ + " which is set to %d]", field, count, DisjunctionMaxClauseCount) +} diff --git a/search/searcher/search_disjunction_heap.go b/search/searcher/search_disjunction_heap.go new file mode 100644 index 0000000..4c68e56 --- /dev/null +++ b/search/searcher/search_disjunction_heap.go @@ -0,0 +1,366 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "container/heap" + "context" + "math" + "reflect" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeDisjunctionHeapSearcher int +var reflectStaticSizeSearcherCurr int + +func init() { + var dhs DisjunctionHeapSearcher + reflectStaticSizeDisjunctionHeapSearcher = int(reflect.TypeOf(dhs).Size()) + + var sc SearcherCurr + reflectStaticSizeSearcherCurr = int(reflect.TypeOf(sc).Size()) +} + +type SearcherCurr struct { + searcher search.Searcher + curr *search.DocumentMatch + matchingIdx int +} + +type DisjunctionHeapSearcher struct { + indexReader index.IndexReader + + numSearchers int + scorer *scorer.DisjunctionQueryScorer + min int + queryNorm float64 + retrieveScoreBreakdown bool + initialized bool + searchers []search.Searcher + heap []*SearcherCurr + + matching []*search.DocumentMatch + matchingIdxs []int + matchingCurrs []*SearcherCurr + + bytesRead uint64 +} + +func newDisjunctionHeapSearcher(ctx context.Context, indexReader index.IndexReader, + searchers []search.Searcher, min float64, options search.SearcherOptions, + limit bool) ( + *DisjunctionHeapSearcher, error) { + if limit && tooManyClauses(len(searchers)) { + return nil, tooManyClausesErr("", len(searchers)) + } + var retrieveScoreBreakdown bool + if ctx != nil { + retrieveScoreBreakdown, _ = ctx.Value(search.IncludeScoreBreakdownKey).(bool) + } + + // build our searcher + rv := DisjunctionHeapSearcher{ + indexReader: indexReader, + searchers: searchers, + numSearchers: len(searchers), + scorer: scorer.NewDisjunctionQueryScorer(options), + min: int(min), + matching: make([]*search.DocumentMatch, len(searchers)), + matchingCurrs: make([]*SearcherCurr, len(searchers)), + matchingIdxs: make([]int, len(searchers)), + retrieveScoreBreakdown: retrieveScoreBreakdown, + heap: make([]*SearcherCurr, 0, len(searchers)), + } + rv.computeQueryNorm() + return &rv, nil +} + +func (s *DisjunctionHeapSearcher) computeQueryNorm() { + // first calculate sum of squared weights + sumOfSquaredWeights := 0.0 + for _, searcher := range s.searchers { + sumOfSquaredWeights += searcher.Weight() + } + // now compute query norm from this + s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) + // finally tell all the downstream searchers the norm + for _, searcher := range s.searchers { + searcher.SetQueryNorm(s.queryNorm) + } +} + +func (s *DisjunctionHeapSearcher) Size() int { + sizeInBytes := reflectStaticSizeDisjunctionHeapSearcher + size.SizeOfPtr + + s.scorer.Size() + + for _, entry := range s.searchers { + sizeInBytes += entry.Size() + } + + for _, entry := range s.matching { + if entry != nil { + sizeInBytes += entry.Size() + } + } + + // for matchingCurrs and heap, just use static size * len + // since searchers and document matches already counted above + sizeInBytes += len(s.matchingCurrs) * reflectStaticSizeSearcherCurr + sizeInBytes += len(s.heap) * reflectStaticSizeSearcherCurr + sizeInBytes += len(s.matchingIdxs) * size.SizeOfInt + + return sizeInBytes +} + +func (s *DisjunctionHeapSearcher) initSearchers(ctx *search.SearchContext) error { + // alloc a single block of SearcherCurrs + block := make([]SearcherCurr, len(s.searchers)) + + // get all searchers pointing at their first match + for i, searcher := range s.searchers { + curr, err := searcher.Next(ctx) + if err != nil { + return err + } + if curr != nil { + block[i].searcher = searcher + block[i].curr = curr + block[i].matchingIdx = i + heap.Push(s, &block[i]) + } + } + + err := s.updateMatches() + if err != nil { + return err + } + s.initialized = true + return nil +} + +func (s *DisjunctionHeapSearcher) updateMatches() error { + matching := s.matching[:0] + matchingCurrs := s.matchingCurrs[:0] + matchingIdxs := s.matchingIdxs[:0] + + if len(s.heap) > 0 { + + // top of the heap is our next hit + next := heap.Pop(s).(*SearcherCurr) + matching = append(matching, next.curr) + matchingCurrs = append(matchingCurrs, next) + matchingIdxs = append(matchingIdxs, next.matchingIdx) + + // now as long as top of heap matches, keep popping + for len(s.heap) > 0 && next.curr.IndexInternalID.Equals(s.heap[0].curr.IndexInternalID) { + next = heap.Pop(s).(*SearcherCurr) + matching = append(matching, next.curr) + matchingCurrs = append(matchingCurrs, next) + matchingIdxs = append(matchingIdxs, next.matchingIdx) + } + } + + s.matching = matching + s.matchingCurrs = matchingCurrs + s.matchingIdxs = matchingIdxs + + return nil +} + +func (s *DisjunctionHeapSearcher) Weight() float64 { + var rv float64 + for _, searcher := range s.searchers { + rv += searcher.Weight() + } + return rv +} + +func (s *DisjunctionHeapSearcher) SetQueryNorm(qnorm float64) { + for _, searcher := range s.searchers { + searcher.SetQueryNorm(qnorm) + } +} + +func (s *DisjunctionHeapSearcher) Next(ctx *search.SearchContext) ( + *search.DocumentMatch, error) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + + var rv *search.DocumentMatch + found := false + for !found && len(s.matching) > 0 { + if len(s.matching) >= s.min { + found = true + if s.retrieveScoreBreakdown { + // just return score and expl breakdown here, since it is a disjunction over knn searchers, + // and the final score and expl is calculated in the knn collector + rv = s.scorer.ScoreAndExplBreakdown(ctx, s.matching, s.matchingIdxs, nil, s.numSearchers) + } else { + // score this match + rv = s.scorer.Score(ctx, s.matching, len(s.matching), s.numSearchers) + } + } + + // invoke next on all the matching searchers + for _, matchingCurr := range s.matchingCurrs { + if matchingCurr.curr != rv { + ctx.DocumentMatchPool.Put(matchingCurr.curr) + } + curr, err := matchingCurr.searcher.Next(ctx) + if err != nil { + return nil, err + } + if curr != nil { + matchingCurr.curr = curr + heap.Push(s, matchingCurr) + } + } + + err := s.updateMatches() + if err != nil { + return nil, err + } + } + + return rv, nil +} + +func (s *DisjunctionHeapSearcher) Advance(ctx *search.SearchContext, + ID index.IndexInternalID) (*search.DocumentMatch, error) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + + // if there is anything in matching, toss it back onto the heap + for _, matchingCurr := range s.matchingCurrs { + heap.Push(s, matchingCurr) + } + s.matching = s.matching[:0] + s.matchingCurrs = s.matchingCurrs[:0] + + // find all searchers that actually need to be advanced + // advance them, using s.matchingCurrs as temp storage + for len(s.heap) > 0 && s.heap[0].curr.IndexInternalID.Compare(ID) < 0 { + searcherCurr := heap.Pop(s).(*SearcherCurr) + ctx.DocumentMatchPool.Put(searcherCurr.curr) + curr, err := searcherCurr.searcher.Advance(ctx, ID) + if err != nil { + return nil, err + } + if curr != nil { + searcherCurr.curr = curr + s.matchingCurrs = append(s.matchingCurrs, searcherCurr) + } + } + // now all of the searchers that we advanced have to be pushed back + for _, matchingCurr := range s.matchingCurrs { + heap.Push(s, matchingCurr) + } + // reset our temp space + s.matchingCurrs = s.matchingCurrs[:0] + + err := s.updateMatches() + if err != nil { + return nil, err + } + + return s.Next(ctx) +} + +func (s *DisjunctionHeapSearcher) Count() uint64 { + // for now return a worst case + var sum uint64 + for _, searcher := range s.searchers { + sum += searcher.Count() + } + return sum +} + +func (s *DisjunctionHeapSearcher) Close() (rv error) { + for _, searcher := range s.searchers { + err := searcher.Close() + if err != nil && rv == nil { + rv = err + } + } + return rv +} + +func (s *DisjunctionHeapSearcher) Min() int { + return s.min +} + +func (s *DisjunctionHeapSearcher) DocumentMatchPoolSize() int { + rv := len(s.searchers) + for _, s := range s.searchers { + rv += s.DocumentMatchPoolSize() + } + return rv +} + +// a disjunction searcher implements the index.Optimizable interface +// but only activates on an edge case where the disjunction is a +// wrapper around a single Optimizable child searcher +func (s *DisjunctionHeapSearcher) Optimize(kind string, octx index.OptimizableContext) ( + index.OptimizableContext, error) { + if len(s.searchers) == 1 { + o, ok := s.searchers[0].(index.Optimizable) + if ok { + return o.Optimize(kind, octx) + } + } + + return nil, nil +} + +// heap impl + +func (s *DisjunctionHeapSearcher) Len() int { return len(s.heap) } + +func (s *DisjunctionHeapSearcher) Less(i, j int) bool { + if s.heap[i].curr == nil { + return true + } else if s.heap[j].curr == nil { + return false + } + return s.heap[i].curr.IndexInternalID.Compare(s.heap[j].curr.IndexInternalID) < 0 +} + +func (s *DisjunctionHeapSearcher) Swap(i, j int) { + s.heap[i], s.heap[j] = s.heap[j], s.heap[i] +} + +func (s *DisjunctionHeapSearcher) Push(x interface{}) { + s.heap = append(s.heap, x.(*SearcherCurr)) +} + +func (s *DisjunctionHeapSearcher) Pop() interface{} { + old := s.heap + n := len(old) + x := old[n-1] + s.heap = old[0 : n-1] + return x +} diff --git a/search/searcher/search_disjunction_slice.go b/search/searcher/search_disjunction_slice.go new file mode 100644 index 0000000..6a92ffa --- /dev/null +++ b/search/searcher/search_disjunction_slice.go @@ -0,0 +1,334 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "math" + "reflect" + "sort" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeDisjunctionSliceSearcher int + +func init() { + var ds DisjunctionSliceSearcher + reflectStaticSizeDisjunctionSliceSearcher = int(reflect.TypeOf(ds).Size()) +} + +type DisjunctionSliceSearcher struct { + indexReader index.IndexReader + searchers []search.Searcher + originalPos []int + numSearchers int + queryNorm float64 + retrieveScoreBreakdown bool + currs []*search.DocumentMatch + scorer *scorer.DisjunctionQueryScorer + min int + matching []*search.DocumentMatch + matchingIdxs []int + initialized bool + bytesRead uint64 +} + +func newDisjunctionSliceSearcher(ctx context.Context, indexReader index.IndexReader, + qsearchers []search.Searcher, min float64, options search.SearcherOptions, + limit bool) ( + *DisjunctionSliceSearcher, error, +) { + if limit && tooManyClauses(len(qsearchers)) { + return nil, tooManyClausesErr("", len(qsearchers)) + } + + var searchers OrderedSearcherList + var originalPos []int + var retrieveScoreBreakdown bool + if ctx != nil { + retrieveScoreBreakdown, _ = ctx.Value(search.IncludeScoreBreakdownKey).(bool) + } + + if retrieveScoreBreakdown { + // needed only when kNN is in picture + sortedSearchers := &OrderedPositionalSearcherList{ + searchers: make([]search.Searcher, len(qsearchers)), + index: make([]int, len(qsearchers)), + } + for i, searcher := range qsearchers { + sortedSearchers.searchers[i] = searcher + sortedSearchers.index[i] = i + } + sort.Sort(sortedSearchers) + searchers = sortedSearchers.searchers + originalPos = sortedSearchers.index + } else { + searchers = make(OrderedSearcherList, len(qsearchers)) + copy(searchers, qsearchers) + sort.Sort(searchers) + } + + rv := DisjunctionSliceSearcher{ + indexReader: indexReader, + searchers: searchers, + originalPos: originalPos, + numSearchers: len(searchers), + currs: make([]*search.DocumentMatch, len(searchers)), + scorer: scorer.NewDisjunctionQueryScorer(options), + min: int(min), + retrieveScoreBreakdown: retrieveScoreBreakdown, + + matching: make([]*search.DocumentMatch, len(searchers)), + matchingIdxs: make([]int, len(searchers)), + } + rv.computeQueryNorm() + return &rv, nil +} + +func (s *DisjunctionSliceSearcher) computeQueryNorm() { + // first calculate sum of squared weights + sumOfSquaredWeights := 0.0 + for _, searcher := range s.searchers { + sumOfSquaredWeights += searcher.Weight() + } + // now compute query norm from this + s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) + // finally tell all the downstream searchers the norm + for _, searcher := range s.searchers { + searcher.SetQueryNorm(s.queryNorm) + } +} + +func (s *DisjunctionSliceSearcher) Size() int { + sizeInBytes := reflectStaticSizeDisjunctionSliceSearcher + size.SizeOfPtr + + s.scorer.Size() + + for _, entry := range s.searchers { + sizeInBytes += entry.Size() + } + + for _, entry := range s.currs { + if entry != nil { + sizeInBytes += entry.Size() + } + } + + for _, entry := range s.matching { + if entry != nil { + sizeInBytes += entry.Size() + } + } + + sizeInBytes += len(s.matchingIdxs) * size.SizeOfInt + sizeInBytes += len(s.originalPos) * size.SizeOfInt + + return sizeInBytes +} + +func (s *DisjunctionSliceSearcher) initSearchers(ctx *search.SearchContext) error { + var err error + // get all searchers pointing at their first match + for i, searcher := range s.searchers { + if s.currs[i] != nil { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = searcher.Next(ctx) + if err != nil { + return err + } + } + + err = s.updateMatches() + if err != nil { + return err + } + + s.initialized = true + return nil +} + +func (s *DisjunctionSliceSearcher) updateMatches() error { + matching := s.matching[:0] + matchingIdxs := s.matchingIdxs[:0] + + for i := 0; i < len(s.currs); i++ { + curr := s.currs[i] + if curr == nil { + continue + } + + if len(matching) > 0 { + cmp := curr.IndexInternalID.Compare(matching[0].IndexInternalID) + if cmp > 0 { + continue + } + + if cmp < 0 { + matching = matching[:0] + matchingIdxs = matchingIdxs[:0] + } + } + matching = append(matching, curr) + matchingIdxs = append(matchingIdxs, i) + } + + s.matching = matching + s.matchingIdxs = matchingIdxs + + return nil +} + +func (s *DisjunctionSliceSearcher) Weight() float64 { + var rv float64 + for _, searcher := range s.searchers { + rv += searcher.Weight() + } + return rv +} + +func (s *DisjunctionSliceSearcher) SetQueryNorm(qnorm float64) { + for _, searcher := range s.searchers { + searcher.SetQueryNorm(qnorm) + } +} + +func (s *DisjunctionSliceSearcher) Next(ctx *search.SearchContext) ( + *search.DocumentMatch, error, +) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + var err error + var rv *search.DocumentMatch + + found := false + for !found && len(s.matching) > 0 { + if len(s.matching) >= s.min { + found = true + if s.retrieveScoreBreakdown { + // just return score and expl breakdown here, since it is a disjunction over knn searchers, + // and the final score and expl is calculated in the knn collector + rv = s.scorer.ScoreAndExplBreakdown(ctx, s.matching, s.matchingIdxs, s.originalPos, s.numSearchers) + } else { + // score this match + rv = s.scorer.Score(ctx, s.matching, len(s.matching), s.numSearchers) + } + } + + // invoke next on all the matching searchers + for _, i := range s.matchingIdxs { + searcher := s.searchers[i] + if s.currs[i] != rv { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = searcher.Next(ctx) + if err != nil { + return nil, err + } + } + + err = s.updateMatches() + if err != nil { + return nil, err + } + } + return rv, nil +} + +func (s *DisjunctionSliceSearcher) Advance(ctx *search.SearchContext, + ID index.IndexInternalID, +) (*search.DocumentMatch, error) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + // get all searchers pointing at their first match + var err error + for i, searcher := range s.searchers { + if s.currs[i] != nil { + if s.currs[i].IndexInternalID.Compare(ID) >= 0 { + continue + } + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = searcher.Advance(ctx, ID) + if err != nil { + return nil, err + } + } + + err = s.updateMatches() + if err != nil { + return nil, err + } + + return s.Next(ctx) +} + +func (s *DisjunctionSliceSearcher) Count() uint64 { + // for now return a worst case + var sum uint64 + for _, searcher := range s.searchers { + sum += searcher.Count() + } + return sum +} + +func (s *DisjunctionSliceSearcher) Close() (rv error) { + for _, searcher := range s.searchers { + err := searcher.Close() + if err != nil && rv == nil { + rv = err + } + } + return rv +} + +func (s *DisjunctionSliceSearcher) Min() int { + return s.min +} + +func (s *DisjunctionSliceSearcher) DocumentMatchPoolSize() int { + rv := len(s.currs) + for _, s := range s.searchers { + rv += s.DocumentMatchPoolSize() + } + return rv +} + +// a disjunction searcher implements the index.Optimizable interface +// but only activates on an edge case where the disjunction is a +// wrapper around a single Optimizable child searcher +func (s *DisjunctionSliceSearcher) Optimize(kind string, octx index.OptimizableContext) ( + index.OptimizableContext, error, +) { + if len(s.searchers) == 1 { + o, ok := s.searchers[0].(index.Optimizable) + if ok { + return o.Optimize(kind, octx) + } + } + + return nil, nil +} diff --git a/search/searcher/search_disjunction_test.go b/search/searcher/search_disjunction_test.go new file mode 100644 index 0000000..e937c2a --- /dev/null +++ b/search/searcher/search_disjunction_test.go @@ -0,0 +1,356 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "os" + "testing" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func TestDisjunctionSearch(t *testing.T) { + twoDocIndexReader, err := twoDocIndex.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := twoDocIndexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + explainTrue := search.SearcherOptions{Explain: true} + + martyTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "marty", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + dustinTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "dustin", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + martyOrDustinSearcher, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{martyTermSearcher, dustinTermSearcher}, 0, explainTrue) + if err != nil { + t.Fatal(err) + } + + martyTermSearcher2, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "marty", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + dustinTermSearcher2, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "dustin", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + martyOrDustinSearcher2, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{martyTermSearcher2, dustinTermSearcher2}, 0, explainTrue) + if err != nil { + t.Fatal(err) + } + + raviTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "ravi", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + nestedRaviOrMartyOrDustinSearcher, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{raviTermSearcher, martyOrDustinSearcher2}, 0, explainTrue) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + searcher search.Searcher + results []*search.DocumentMatch + }{ + { + searcher: martyOrDustinSearcher, + results: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("1"), + Score: 0.6775110856165737, + }, + { + IndexInternalID: index.IndexInternalID("3"), + Score: 0.6775110856165737, + }, + }, + }, + // test a nested disjunction + { + searcher: nestedRaviOrMartyOrDustinSearcher, + results: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("1"), + Score: 0.2765927424732821, + }, + { + IndexInternalID: index.IndexInternalID("3"), + Score: 0.2765927424732821, + }, + { + IndexInternalID: index.IndexInternalID("4"), + Score: 0.5531854849465642, + }, + }, + }, + } + + for testIndex, test := range tests { + defer func() { + err := test.searcher.Close() + if err != nil { + t.Fatal(err) + } + }() + + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0), + } + next, err := test.searcher.Next(ctx) + i := 0 + for err == nil && next != nil { + if i < len(test.results) { + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) + } + if !scoresCloseEnough(next.Score, test.results[i].Score) { + t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) + t.Logf("scoring explanation: %s", next.Expl) + } + } + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) + i++ + } + if err != nil { + t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) + } + if len(test.results) != i { + t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) + } + } +} + +func TestDisjunctionAdvance(t *testing.T) { + twoDocIndexReader, err := twoDocIndex.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := twoDocIndexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + explainTrue := search.SearcherOptions{Explain: true} + + martyTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "marty", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + dustinTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "dustin", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + martyOrDustinSearcher, err := NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{martyTermSearcher, dustinTermSearcher}, 0, explainTrue) + if err != nil { + t.Fatal(err) + } + + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(martyOrDustinSearcher.DocumentMatchPoolSize(), 0), + } + match, err := martyOrDustinSearcher.Advance(ctx, index.IndexInternalID("3")) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if match == nil { + t.Errorf("expected 3, got nil") + } +} + +func TestDisjunctionSearchTooMany(t *testing.T) { + // set to max to a low non-zero value + DisjunctionMaxClauseCount = 2 + defer func() { + // reset it after the test + DisjunctionMaxClauseCount = 0 + }() + + twoDocIndexReader, err := twoDocIndex.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := twoDocIndexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + explainTrue := search.SearcherOptions{Explain: true} + + martyTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "marty", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + dustinTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "dustin", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + steveTermSearcher, err := NewTermSearcher(context.TODO(), twoDocIndexReader, "steve", "name", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + _, err = NewDisjunctionSearcher(context.TODO(), twoDocIndexReader, []search.Searcher{martyTermSearcher, dustinTermSearcher, steveTermSearcher}, 0, explainTrue) + if err == nil { + t.Fatal(err) + } +} + +func TestUnadornedDisjunctionAdvance(t *testing.T) { + dir, _ := os.MkdirTemp("", "scorchTwoDoc") + defer func() { + _ = os.RemoveAll(dir) + }() + twoDocIndex := initTwoDocScorch(dir) + twoDocIndexReader, err := twoDocIndex.Reader() + if err != nil { + t.Fatal(err) + } + defer func() { + err := twoDocIndexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + getNewOptimizedCompositeSearcher := func(t *testing.T) search.Searcher { + optimizedCompositeSearcherOptions := search.SearcherOptions{Explain: false, IncludeTermVectors: false, Score: "none"} + martyTermSearcher, err := NewTermSearcher(context.Background(), twoDocIndexReader, "marty", "name", 1.0, optimizedCompositeSearcherOptions) + if err != nil { + t.Fatal(err) + } + dustinTermSearcher, err := NewTermSearcher(context.Background(), twoDocIndexReader, "dustin", "name", 1.0, optimizedCompositeSearcherOptions) + if err != nil { + t.Fatal(err) + } + steveTermSearcher, err := NewTermSearcher(context.Background(), twoDocIndexReader, "steve", "name", 1.0, optimizedCompositeSearcherOptions) + if err != nil { + t.Fatal(err) + } + martyOrDustinOrSteveSearcher, err := NewDisjunctionSearcher(context.Background(), twoDocIndexReader, []search.Searcher{martyTermSearcher, dustinTermSearcher, steveTermSearcher}, 0, optimizedCompositeSearcherOptions) + if err != nil { + t.Fatal(err) + } + return martyOrDustinOrSteveSearcher + } + martyOrDustinOrSteveSearcher := getNewOptimizedCompositeSearcher(t) + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(martyOrDustinOrSteveSearcher.DocumentMatchPoolSize(), 0), + } + // get the correct order using only next calls + dm, err := martyOrDustinOrSteveSearcher.Next(ctx) + if err != nil { + t.Fatal(err) + } + expectedDocIDs := []index.IndexInternalID{} + for dm != nil && err == nil { + expectedDocIDs = append(expectedDocIDs, dm.IndexInternalID) + dm, err = martyOrDustinOrSteveSearcher.Next(ctx) + } + if err != nil { + t.Fatal(err) + } + if len(expectedDocIDs) != 3 { + t.Fatalf("expected 3 results, got %d", len(expectedDocIDs)) + } + // Test 1 - Advance in reverse direction after getting the correct order using only next calls + // Next(->) - Next(->) - Next(->) - Advance(<-) - Advance(<-) + for i := len(expectedDocIDs) - 1; i >= 0; i-- { + xID := expectedDocIDs[i] + dm, err = martyOrDustinOrSteveSearcher.Advance(ctx, xID) + if err != nil { + t.Fatal(err) + } + if dm == nil { + t.Fatalf("expected to find %v", xID) + } + if !dm.IndexInternalID.Equals(xID) { + t.Fatalf("expected %v, got %v", xID, dm.IndexInternalID) + } + } + // Test 2 - Advance in forward direction after getting the correct order using only next calls + // Next(->) - Next(->) - Next(->) - Advance(ResetTo0) - Advance(->) - Advance(->) + martyOrDustinOrSteveSearcher = getNewOptimizedCompositeSearcher(t) + for i := 0; i < len(expectedDocIDs); i++ { + xID := expectedDocIDs[i] + dm, err = martyOrDustinOrSteveSearcher.Advance(ctx, xID) + if err != nil { + t.Fatal(err) + } + if dm == nil { + t.Fatalf("expected to find %v", xID) + } + if !dm.IndexInternalID.Equals(xID) { + t.Fatalf("expected %v, got %v", xID, dm.IndexInternalID) + } + } + // Test 3 - Alternate Next and Advance calls + // Next(->) -> Next(->) -> Advance(<-) -> Next(->) -> Next(->) -> Advance(<-) -> Advance(<-) -> Next(->) + martyOrDustinOrSteveSearcher = getNewOptimizedCompositeSearcher(t) + goNext := func(expectedDocID index.IndexInternalID) { + dm, err = martyOrDustinOrSteveSearcher.Next(ctx) + if err != nil { + t.Fatal(err) + } + if dm == nil { + t.Fatal("expected a document, got nil") + } + if !dm.IndexInternalID.Equals(expectedDocID) { + t.Fatalf("expected %v, got %v", expectedDocID, dm.IndexInternalID) + } + } + goBack := func(goTo index.IndexInternalID) { + dm, err = martyOrDustinOrSteveSearcher.Advance(ctx, goTo) + if err != nil { + t.Fatal(err) + } + if dm == nil { + t.Fatalf("expected to find %v", goTo) + } + if !dm.IndexInternalID.Equals(goTo) { + t.Fatalf("expected %v, got %v", goTo, dm.IndexInternalID) + } + } + // Next (->) + goNext(expectedDocIDs[0]) + // Next (->) + goNext(expectedDocIDs[1]) + // Advance (<-) + goBack(expectedDocIDs[0]) + // Next (->) + goNext(expectedDocIDs[1]) + // Next (->) + goNext(expectedDocIDs[2]) + // Advance (<-) + goBack(expectedDocIDs[1]) + // Advance (<-) + goBack(expectedDocIDs[0]) + // Next (->) + goNext(expectedDocIDs[1]) +} diff --git a/search/searcher/search_docid.go b/search/searcher/search_docid.go new file mode 100644 index 0000000..720fd32 --- /dev/null +++ b/search/searcher/search_docid.go @@ -0,0 +1,110 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "reflect" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeDocIDSearcher int + +func init() { + var ds DocIDSearcher + reflectStaticSizeDocIDSearcher = int(reflect.TypeOf(ds).Size()) +} + +// DocIDSearcher returns documents matching a predefined set of identifiers. +type DocIDSearcher struct { + reader index.DocIDReader + scorer *scorer.ConstantScorer + count int +} + +func NewDocIDSearcher(ctx context.Context, indexReader index.IndexReader, ids []string, boost float64, + options search.SearcherOptions) (searcher *DocIDSearcher, err error) { + + reader, err := indexReader.DocIDReaderOnly(ids) + if err != nil { + return nil, err + } + scorer := scorer.NewConstantScorer(1.0, boost, options) + return &DocIDSearcher{ + scorer: scorer, + reader: reader, + count: len(ids), + }, nil +} + +func (s *DocIDSearcher) Size() int { + return reflectStaticSizeDocIDSearcher + size.SizeOfPtr + + s.reader.Size() + + s.scorer.Size() +} + +func (s *DocIDSearcher) Count() uint64 { + return uint64(s.count) +} + +func (s *DocIDSearcher) Weight() float64 { + return s.scorer.Weight() +} + +func (s *DocIDSearcher) SetQueryNorm(qnorm float64) { + s.scorer.SetQueryNorm(qnorm) +} + +func (s *DocIDSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + docidMatch, err := s.reader.Next() + if err != nil { + return nil, err + } + if docidMatch == nil { + return nil, nil + } + + docMatch := s.scorer.Score(ctx, docidMatch) + return docMatch, nil +} + +func (s *DocIDSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + docidMatch, err := s.reader.Advance(ID) + if err != nil { + return nil, err + } + if docidMatch == nil { + return nil, nil + } + + docMatch := s.scorer.Score(ctx, docidMatch) + return docMatch, nil +} + +func (s *DocIDSearcher) Close() error { + return s.reader.Close() +} + +func (s *DocIDSearcher) Min() int { + return 0 +} + +func (s *DocIDSearcher) DocumentMatchPoolSize() int { + return 1 +} diff --git a/search/searcher/search_docid_test.go b/search/searcher/search_docid_test.go new file mode 100644 index 0000000..65fedc6 --- /dev/null +++ b/search/searcher/search_docid_test.go @@ -0,0 +1,146 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "testing" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/index/upsidedown" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) { + analysisQueue := index.NewAnalysisQueue(1) + i, err := upsidedown.NewUpsideDownCouch( + gtreap.Name, + map[string]interface{}{ + "path": "", + }, + analysisQueue) + if err != nil { + t.Fatal(err) + } + err = i.Open() + if err != nil { + t.Fatal(err) + } + for _, id := range indexed { + doc := document.NewDocument(id) + doc.AddField(document.NewTextField("desc", []uint64{}, []byte("beer"))) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + } + + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + explainOff := search.SearcherOptions{Explain: false} + + searcher, err := NewDocIDSearcher(context.TODO(), indexReader, searched, 1.0, explainOff) + if err != nil { + t.Fatal(err) + } + defer func() { + err := searcher.Close() + if err != nil { + t.Fatal(err) + } + }() + + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(searcher.DocumentMatchPoolSize(), 0), + } + + // Check the sequence + for i, id := range wanted { + m, err := searcher.Next(ctx) + if err != nil { + t.Fatal(err) + } + if !index.IndexInternalID(id).Equals(m.IndexInternalID) { + t.Fatalf("expected %v at position %v, got %v", id, i, m.IndexInternalID) + } + ctx.DocumentMatchPool.Put(m) + } + m, err := searcher.Next(ctx) + if err != nil { + t.Fatal(err) + } + if m != nil { + t.Fatalf("expected nil past the end of the sequence, got %v", m.IndexInternalID) + } + ctx.DocumentMatchPool.Put(m) + + // Check seeking + for _, id := range wanted { + if len(id) != 2 { + t.Fatalf("expected identifier must be 2 characters long, got %v", id) + } + before := id[:1] + for _, target := range []string{before, id} { + m, err := searcher.Advance(ctx, index.IndexInternalID(target)) + if err != nil { + t.Fatal(err) + } + if m == nil || !m.IndexInternalID.Equals(index.IndexInternalID(id)) { + t.Fatalf("advancing to %v returned %v instead of %v", before, m, id) + } + ctx.DocumentMatchPool.Put(m) + } + } + // Seek after the end of the sequence + after := "zzz" + m, err = searcher.Advance(ctx, index.IndexInternalID(after)) + if err != nil { + t.Fatal(err) + } + if m != nil { + t.Fatalf("advancing past the end of the sequence should return nil, got %v", m) + } + ctx.DocumentMatchPool.Put(m) +} + +func TestDocIDSearcherEmptySearchEmptyIndex(t *testing.T) { + testDocIDSearcher(t, nil, nil, nil) +} + +func TestDocIDSearcherEmptyIndex(t *testing.T) { + testDocIDSearcher(t, nil, []string{"aa", "bb"}, nil) +} + +func TestDocIDSearcherEmptySearch(t *testing.T) { + testDocIDSearcher(t, []string{"aa", "bb"}, nil, nil) +} + +func TestDocIDSearcherValid(t *testing.T) { + // Test missing, out of order and duplicate inputs + testDocIDSearcher(t, []string{"aa", "bb", "cc"}, + []string{"ee", "bb", "aa", "bb"}, + []string{"aa", "bb"}) +} diff --git a/search/searcher/search_filter.go b/search/searcher/search_filter.go new file mode 100644 index 0000000..ef070c7 --- /dev/null +++ b/search/searcher/search_filter.go @@ -0,0 +1,110 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "reflect" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeFilteringSearcher int + +func init() { + var fs FilteringSearcher + reflectStaticSizeFilteringSearcher = int(reflect.TypeOf(fs).Size()) +} + +// FilterFunc defines a function which can filter documents +// returning true means keep the document +// returning false means do not keep the document +type FilterFunc func(sctx *search.SearchContext, d *search.DocumentMatch) bool + +// FilteringSearcher wraps any other searcher, but checks any Next/Advance +// call against the supplied FilterFunc +type FilteringSearcher struct { + child search.Searcher + accept FilterFunc +} + +func NewFilteringSearcher(ctx context.Context, s search.Searcher, filter FilterFunc) *FilteringSearcher { + return &FilteringSearcher{ + child: s, + accept: filter, + } +} + +func (f *FilteringSearcher) Size() int { + return reflectStaticSizeFilteringSearcher + size.SizeOfPtr + + f.child.Size() +} + +func (f *FilteringSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + next, err := f.child.Next(ctx) + for next != nil && err == nil { + if f.accept(ctx, next) { + return next, nil + } + // recycle this document match now, since + // we do not need it anymore + ctx.DocumentMatchPool.Put(next) + next, err = f.child.Next(ctx) + } + return nil, err +} + +func (f *FilteringSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + adv, err := f.child.Advance(ctx, ID) + if err != nil { + return nil, err + } + if adv == nil { + return nil, nil + } + if f.accept(ctx, adv) { + return adv, nil + } + // recycle this document match now, since + // we do not need it anymore + ctx.DocumentMatchPool.Put(adv) + return f.Next(ctx) +} + +func (f *FilteringSearcher) Close() error { + return f.child.Close() +} + +func (f *FilteringSearcher) Weight() float64 { + return f.child.Weight() +} + +func (f *FilteringSearcher) SetQueryNorm(n float64) { + f.child.SetQueryNorm(n) +} + +func (f *FilteringSearcher) Count() uint64 { + return f.child.Count() +} + +func (f *FilteringSearcher) Min() int { + return f.child.Min() +} + +func (f *FilteringSearcher) DocumentMatchPoolSize() int { + return f.child.DocumentMatchPoolSize() +} diff --git a/search/searcher/search_fuzzy.go b/search/searcher/search_fuzzy.go new file mode 100644 index 0000000..f53502f --- /dev/null +++ b/search/searcher/search_fuzzy.go @@ -0,0 +1,258 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "fmt" + "strings" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +var MaxFuzziness = 2 + +// AutoFuzzinessHighThreshold is the threshold for the term length +// above which the fuzziness is set to MaxFuzziness when the fuzziness +// mode is set to AutoFuzziness. +var AutoFuzzinessHighThreshold = 5 + +// AutoFuzzinessLowThreshold is the threshold for the term length +// below which the fuzziness is set to zero when the fuzziness mode +// is set to AutoFuzziness. +// For terms with length between AutoFuzzinessLowThreshold and +// AutoFuzzinessHighThreshold, the fuzziness is set to +// MaxFuzziness - 1. +var AutoFuzzinessLowThreshold = 2 + +func NewFuzzySearcher(ctx context.Context, indexReader index.IndexReader, term string, + prefix, fuzziness int, field string, boost float64, + options search.SearcherOptions) (search.Searcher, error) { + + if fuzziness > MaxFuzziness { + return nil, fmt.Errorf("fuzziness exceeds max (%d)", MaxFuzziness) + } + + if fuzziness < 0 { + return nil, fmt.Errorf("invalid fuzziness, negative") + } + if fuzziness == 0 { + // no fuzziness, just do a term search + // check if the call is made from a phrase searcher + // and if so, add the term to the fuzzy term matches + // since the fuzzy candidate terms are not collected + // for a term search, and the only candidate term is + // the term itself + if ctx != nil { + fuzzyTermMatches := ctx.Value(search.FuzzyMatchPhraseKey) + if fuzzyTermMatches != nil { + fuzzyTermMatches.(map[string][]string)[term] = []string{term} + } + } + return NewTermSearcher(ctx, indexReader, term, field, boost, options) + } + + // Note: we don't byte slice the term for a prefix because of runes. + // we instead slice the term at the first rune boundary at/after prefix index + prefixTerm := term + for i := range term { + if i >= prefix { + prefixTerm = term[:i] + break + } + } + fuzzyCandidates, err := findFuzzyCandidateTerms(ctx, indexReader, term, fuzziness, + field, prefixTerm) + if err != nil { + return nil, err + } + + var candidates []string + var editDistances []uint8 + var dictBytesRead uint64 + if fuzzyCandidates != nil { + candidates = fuzzyCandidates.candidates + editDistances = fuzzyCandidates.editDistances + dictBytesRead = fuzzyCandidates.bytesRead + } + + var fuzzyTermMatches map[string][]string + if ctx != nil { + reportIOStats(ctx, dictBytesRead) + search.RecordSearchCost(ctx, search.AddM, dictBytesRead) + if ftm := ctx.Value(search.FuzzyMatchPhraseKey); ftm != nil { + fuzzyTermMatches = ftm.(map[string][]string) + } + } + if fuzzyTermMatches != nil { + fuzzyTermMatches[term] = candidates + } + // check if the candidates are empty or have one term which is the term itself + if len(candidates) == 0 || (len(candidates) == 1 && candidates[0] == term) { + if fuzzyTermMatches != nil { + fuzzyTermMatches[term] = []string{term} + } + return NewTermSearcher(ctx, indexReader, term, field, boost, options) + } + + return NewMultiTermSearcherBoosted(ctx, indexReader, candidates, field, + boost, editDistances, options, true) +} + +func GetAutoFuzziness(term string) int { + termLength := len(term) + if termLength > AutoFuzzinessHighThreshold { + return MaxFuzziness + } else if termLength > AutoFuzzinessLowThreshold { + return MaxFuzziness - 1 + } + return 0 +} + +func NewAutoFuzzySearcher(ctx context.Context, indexReader index.IndexReader, term string, + prefix int, field string, boost float64, options search.SearcherOptions) (search.Searcher, error) { + return NewFuzzySearcher(ctx, indexReader, term, prefix, GetAutoFuzziness(term), field, boost, options) +} + +type fuzzyCandidates struct { + candidates []string + editDistances []uint8 + bytesRead uint64 +} + +func reportIOStats(ctx context.Context, bytesRead uint64) { + // The fuzzy, regexp like queries essentially load a dictionary, + // which potentially incurs a cost that must be accounted by + // using the callback to report the value. + if ctx != nil { + statsCallbackFn := ctx.Value(search.SearchIOStatsCallbackKey) + if statsCallbackFn != nil { + statsCallbackFn.(search.SearchIOStatsCallbackFunc)(bytesRead) + } + } +} + +func findFuzzyCandidateTerms(ctx context.Context, indexReader index.IndexReader, term string, + fuzziness int, field, prefixTerm string) (rv *fuzzyCandidates, err error) { + rv = &fuzzyCandidates{ + candidates: make([]string, 0), + editDistances: make([]uint8, 0), + } + + // in case of advanced reader implementations directly call + // the levenshtein automaton based iterator to collect the + // candidate terms + if ir, ok := indexReader.(index.IndexReaderFuzzy); ok { + var synonymTerms map[string][]string + if ctx != nil { + if fts, ok := ctx.Value(search.FieldTermSynonymMapKey).(search.FieldTermSynonymMap); ok { + synonymTerms = fts[field] + } + } + // termSet is used to de-duplicate synonym terms against the candidates + // gathered across all the segments' dictionaries. + var termSet map[string]struct{} + if len(synonymTerms) > 0 { + termSet = make(map[string]struct{}, len(synonymTerms)) + } + addCandidateTerm := func(term string, editDistance uint8) error { + if termSet != nil { + if _, exists := termSet[term]; exists { + return nil + } + termSet[term] = struct{}{} + } + rv.candidates = append(rv.candidates, term) + rv.editDistances = append(rv.editDistances, editDistance) + if tooManyClauses(len(rv.candidates)) { + return tooManyClausesErr(field, len(rv.candidates)) + } + return nil + } + fieldDict, a, err := ir.FieldDictFuzzyAutomaton(field, term, fuzziness, prefixTerm) + if err != nil { + return nil, err + } + defer func() { + if cerr := fieldDict.Close(); cerr != nil && err == nil { + err = cerr + } + }() + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + err = addCandidateTerm(tfd.Term, tfd.EditDistance) + if err != nil { + return nil, err + } + tfd, err = fieldDict.Next() + } + if err != nil { + return nil, err + } + for term := range synonymTerms { + if _, exists := termSet[term]; exists { + continue + } + if !strings.HasPrefix(term, prefixTerm) { + continue + } + match, editDistance := a.MatchAndDistance(term) + if match { + err = addCandidateTerm(term, editDistance) + if err != nil { + return nil, err + } + } + } + rv.bytesRead = fieldDict.BytesRead() + return rv, nil + } + + var fieldDict index.FieldDict + if len(prefixTerm) > 0 { + fieldDict, err = indexReader.FieldDictPrefix(field, []byte(prefixTerm)) + } else { + fieldDict, err = indexReader.FieldDict(field) + } + if err != nil { + return nil, err + } + defer func() { + if cerr := fieldDict.Close(); cerr != nil && err == nil { + err = cerr + } + }() + + // enumerate terms and check levenshtein distance + var reuse []int + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + var ld int + var exceeded bool + ld, exceeded, reuse = search.LevenshteinDistanceMaxReuseSlice(term, tfd.Term, fuzziness, reuse) + if !exceeded && ld <= fuzziness { + rv.candidates = append(rv.candidates, tfd.Term) + rv.editDistances = append(rv.editDistances, uint8(ld)) + if tooManyClauses(len(rv.candidates)) { + return nil, tooManyClausesErr(field, len(rv.candidates)) + } + } + tfd, err = fieldDict.Next() + } + + rv.bytesRead = fieldDict.BytesRead() + return rv, err +} diff --git a/search/searcher/search_fuzzy_bench_test.go b/search/searcher/search_fuzzy_bench_test.go new file mode 100644 index 0000000..63b90f6 --- /dev/null +++ b/search/searcher/search_fuzzy_bench_test.go @@ -0,0 +1,127 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "fmt" + "testing" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +// buildFuzzyBenchIndex builds a scorch index whose "desc" field has numTerms +// distinct terms spread across numDocs documents (perDoc terms per document), +// so candidate terms have real multi-doc postings lists. +func buildFuzzyBenchIndex(b *testing.B, numTerms, numDocs, perDoc int) index.Index { + analysisQueue := index.NewAnalysisQueue(1) + idx, err := scorch.NewScorch(scorch.Name, + map[string]interface{}{"path": b.TempDir()}, analysisQueue) + if err != nil { + b.Fatal(err) + } + if err = idx.Open(); err != nil { + b.Fatal(err) + } + + terms := make([]string, numTerms) + for i := range terms { + terms[i] = fmt.Sprintf("term%04d", i) + } + + batch := index.NewBatch() + for d := 0; d < numDocs; d++ { + doc := document.NewDocument(fmt.Sprintf("%d", d)) + buf := make([]byte, 0, perDoc*9) + for j := 0; j < perDoc; j++ { + if j > 0 { + buf = append(buf, ' ') + } + buf = append(buf, terms[(d*perDoc+j)%numTerms]...) + } + doc.AddField(document.NewTextFieldCustom("desc", nil, buf, + twoDocIndexDescIndexingOptions, testAnalyzer)) + batch.Update(doc) + } + if err = idx.Batch(batch); err != nil { + b.Fatal(err) + } + return idx +} + +// BenchmarkFuzzyCandidateCollection isolates candidate-term collection +// (findFuzzyCandidateTerms) over a real scorch index for a fuzziness-2 query. +// This is the step the fuzzy optimizations touch; the full NewFuzzySearcher is +// dominated by building one term searcher per candidate downstream, which these +// changes do not affect and which would dilute the measurement. +func BenchmarkFuzzyCandidateCollection(b *testing.B) { + idx := buildFuzzyBenchIndex(b, 1000, 3000, 40) + defer func() { _ = idx.Close() }() + + r, err := idx.Reader() + if err != nil { + b.Fatal(err) + } + defer func() { _ = r.Close() }() + + ctx := context.Background() + + // sanity: report how many candidates the query matches, once. + fc, err := findFuzzyCandidateTerms(ctx, r, "term0500", 2, "desc", "") + if err != nil { + b.Fatal(err) + } + b.Logf("candidates=%d", len(fc.candidates)) + + b.ResetTimer() + b.ReportAllocs() + for n := 0; n < b.N; n++ { + _, err := findFuzzyCandidateTerms(ctx, r, "term0500", 2, "desc", "") + if err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkFuzzySearcherEndToEnd measures the full NewFuzzySearcher path for +// reference (candidate collection + building the boosted disjunction). +func BenchmarkFuzzySearcherEndToEnd(b *testing.B) { + idx := buildFuzzyBenchIndex(b, 1000, 3000, 40) + defer func() { _ = idx.Close() }() + + r, err := idx.Reader() + if err != nil { + b.Fatal(err) + } + defer func() { _ = r.Close() }() + + opts := search.SearcherOptions{Score: "none"} + ctx := context.Background() + + b.ResetTimer() + b.ReportAllocs() + for n := 0; n < b.N; n++ { + s, err := NewFuzzySearcher(ctx, r, "term0500", 0, 2, "desc", 1.0, opts) + if err != nil { + b.Fatal(err) + } + if err = s.Close(); err != nil { + b.Fatal(err) + } + } +} diff --git a/search/searcher/search_fuzzy_test.go b/search/searcher/search_fuzzy_test.go new file mode 100644 index 0000000..f6c2b79 --- /dev/null +++ b/search/searcher/search_fuzzy_test.go @@ -0,0 +1,156 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "testing" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func TestFuzzySearch(t *testing.T) { + twoDocIndexReader, err := twoDocIndex.Reader() + if err != nil { + t.Error(err) + } + + defer func() { + err := twoDocIndexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + explainTrue := search.SearcherOptions{Explain: true} + + fuzzySearcherbeet, err := NewFuzzySearcher(context.TODO(), twoDocIndexReader, "beet", 0, 1, "desc", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + + fuzzySearcherdouches, err := NewFuzzySearcher(context.TODO(), twoDocIndexReader, "douches", 0, 2, "desc", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + + fuzzySearcheraplee, err := NewFuzzySearcher(context.TODO(), twoDocIndexReader, "aplee", 0, 2, "desc", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + + fuzzySearcherprefix, err := NewFuzzySearcher(context.TODO(), twoDocIndexReader, "water", 3, 2, "desc", 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + searcher search.Searcher + results []*search.DocumentMatch + }{ + { + searcher: fuzzySearcherbeet, + results: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("1"), + Score: 1.0, + }, + { + IndexInternalID: index.IndexInternalID("2"), + Score: 0.5, + }, + { + IndexInternalID: index.IndexInternalID("3"), + Score: 0.5, + }, + { + IndexInternalID: index.IndexInternalID("4"), + Score: 0.9999999838027345, + }, + }, + }, + { + searcher: fuzzySearcherdouches, + results: []*search.DocumentMatch{}, + }, + { + searcher: fuzzySearcheraplee, + results: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("3"), + Score: 0.9581453659370776, + }, + }, + }, + { + searcher: fuzzySearcherprefix, + results: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("5"), + Score: 1.916290731874155, + }, + }, + }, + } + + for testIndex, test := range tests { + defer func() { + err := test.searcher.Close() + if err != nil { + t.Fatal(err) + } + }() + + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0), + } + next, err := test.searcher.Next(ctx) + i := 0 + for err == nil && next != nil { + if i < len(test.results) { + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) + } + if next.Score != test.results[i].Score { + t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) + t.Logf("scoring explanation: %s", next.Expl) + } + } + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) + i++ + } + if err != nil { + t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) + } + if len(test.results) != i { + t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) + } + } +} + +func TestFuzzySearchLimitErrors(t *testing.T) { + explainTrue := search.SearcherOptions{Explain: true} + _, err := NewFuzzySearcher(context.TODO(), nil, "water", 3, 3, "desc", 1.0, explainTrue) + if err == nil { + t.Fatal("`fuzziness exceeds max (2)` error expected") + } + + _, err = NewFuzzySearcher(context.TODO(), nil, "water", 3, -1, "desc", 1.0, explainTrue) + if err == nil { + t.Fatal("`invalid fuzziness, negative` error expected") + } +} diff --git a/search/searcher/search_geoboundingbox.go b/search/searcher/search_geoboundingbox.go new file mode 100644 index 0000000..a146fa6 --- /dev/null +++ b/search/searcher/search_geoboundingbox.go @@ -0,0 +1,313 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +type filterFunc func(key []byte) bool + +var ( + GeoBitsShift1 = geo.GeoBits << 1 + GeoBitsShift1Minus1 = GeoBitsShift1 - 1 +) + +func NewGeoBoundingBoxSearcher(ctx context.Context, indexReader index.IndexReader, minLon, minLat, + maxLon, maxLat float64, field string, boost float64, + options search.SearcherOptions, checkBoundaries bool) ( + search.Searcher, error, +) { + if tp, ok := indexReader.(index.SpatialIndexPlugin); ok { + sp, err := tp.GetSpatialAnalyzerPlugin("s2") + if err == nil { + terms := sp.GetQueryTokens(geo.NewBoundedRectangle(minLat, + minLon, maxLat, maxLon)) + boxSearcher, err := NewMultiTermSearcher(ctx, indexReader, + terms, field, boost, options, false) + if err != nil { + return nil, err + } + + dvReader, err := indexReader.DocValueReader([]string{field}) + if err != nil { + return nil, err + } + + return NewFilteringSearcher(ctx, boxSearcher, buildRectFilter(ctx, dvReader, + minLon, minLat, maxLon, maxLat)), nil + } + } + + // indexes without the spatial plugin override would continue here. + + // track list of opened searchers, for cleanup on early exit + var openedSearchers []search.Searcher + cleanupOpenedSearchers := func() { + for _, s := range openedSearchers { + _ = s.Close() + } + } + + // do math to produce list of terms needed for this search + onBoundaryTerms, notOnBoundaryTerms, err := ComputeGeoRange(context.TODO(), 0, GeoBitsShift1Minus1, + minLon, minLat, maxLon, maxLat, checkBoundaries, indexReader, field) + if err != nil { + return nil, err + } + + var onBoundarySearcher search.Searcher + dvReader, err := indexReader.DocValueReader([]string{field}) + if err != nil { + return nil, err + } + + if len(onBoundaryTerms) > 0 { + rawOnBoundarySearcher, err := NewMultiTermSearcherBytes(ctx, indexReader, + onBoundaryTerms, field, boost, options, false) + if err != nil { + return nil, err + } + // add filter to check points near the boundary + onBoundarySearcher = NewFilteringSearcher(ctx, rawOnBoundarySearcher, + buildRectFilter(ctx, dvReader, minLon, minLat, maxLon, maxLat)) + openedSearchers = append(openedSearchers, onBoundarySearcher) + } + + var notOnBoundarySearcher search.Searcher + if len(notOnBoundaryTerms) > 0 { + var err error + notOnBoundarySearcher, err = NewMultiTermSearcherBytes(ctx, indexReader, + notOnBoundaryTerms, field, boost, options, false) + if err != nil { + cleanupOpenedSearchers() + return nil, err + } + openedSearchers = append(openedSearchers, notOnBoundarySearcher) + } + + if onBoundarySearcher != nil && notOnBoundarySearcher != nil { + rv, err := NewDisjunctionSearcher(ctx, indexReader, + []search.Searcher{ + onBoundarySearcher, + notOnBoundarySearcher, + }, + 0, options) + if err != nil { + cleanupOpenedSearchers() + return nil, err + } + return rv, nil + } else if onBoundarySearcher != nil { + return onBoundarySearcher, nil + } else if notOnBoundarySearcher != nil { + return notOnBoundarySearcher, nil + } + + return NewMatchNoneSearcher(indexReader) +} + +var ( + geoMaxShift = document.GeoPrecisionStep * 4 + geoDetailLevel = ((geo.GeoBits << 1) - geoMaxShift) / 2 +) + +type closeFunc func() error + +func ComputeGeoRange(ctx context.Context, term uint64, shift uint, + sminLon, sminLat, smaxLon, smaxLat float64, checkBoundaries bool, + indexReader index.IndexReader, field string) ( + onBoundary [][]byte, notOnBoundary [][]byte, err error, +) { + isIndexed, closeF, err := buildIsIndexedFunc(ctx, indexReader, field) + if closeF != nil { + defer func() { + cerr := closeF() + if cerr != nil { + err = cerr + } + }() + } + + grc := &geoRangeCompute{ + preallocBytesLen: 32, + preallocBytes: make([]byte, 32), + sminLon: sminLon, + sminLat: sminLat, + smaxLon: smaxLon, + smaxLat: smaxLat, + checkBoundaries: checkBoundaries, + isIndexed: isIndexed, + } + + grc.computeGeoRange(term, shift) + + return grc.onBoundary, grc.notOnBoundary, nil +} + +func buildIsIndexedFunc(ctx context.Context, indexReader index.IndexReader, field string) (isIndexed filterFunc, closeF closeFunc, err error) { + if irr, ok := indexReader.(index.IndexReaderContains); ok { + fieldDict, err := irr.FieldDictContains(field) + if err != nil { + return nil, nil, err + } + + isIndexed = func(term []byte) bool { + found, err := fieldDict.Contains(term) + return err == nil && found + } + + closeF = func() error { + if fd, ok := fieldDict.(index.FieldDict); ok { + err := fd.Close() + if err != nil { + return err + } + } + return nil + } + } else if indexReader != nil { + isIndexed = func(term []byte) bool { + reader, err := indexReader.TermFieldReader(ctx, term, field, false, false, false) + if err != nil || reader == nil { + return false + } + if reader.Count() == 0 { + _ = reader.Close() + return false + } + _ = reader.Close() + return true + } + } else { + isIndexed = func([]byte) bool { + return true + } + } + return isIndexed, closeF, err +} + +func buildRectFilter(ctx context.Context, dvReader index.DocValueReader, + minLon, minLat, maxLon, maxLat float64, +) FilterFunc { + // reuse the following for each document match that is checked using the filter + var lons, lats []float64 + var found bool + dvVisitor := func(_ string, term []byte) { + if found { + // avoid redundant work if already found + return + } + // only consider the values which are shifted 0 + prefixCoded := numeric.PrefixCoded(term) + shift, err := prefixCoded.Shift() + if err == nil && shift == 0 { + var i64 int64 + i64, err = prefixCoded.Int64() + if err == nil { + lons = append(lons, geo.MortonUnhashLon(uint64(i64))) + lats = append(lats, geo.MortonUnhashLat(uint64(i64))) + found = true + } + } + } + return func(sctx *search.SearchContext, d *search.DocumentMatch) bool { + // check geo matches against all numeric type terms indexed + lons, lats = lons[:0], lats[:0] + found = false + if err := dvReader.VisitDocValues(d.IndexInternalID, dvVisitor); err == nil && found { + bytes := dvReader.BytesRead() + if bytes > 0 { + reportIOStats(ctx, bytes) + search.RecordSearchCost(ctx, search.AddM, bytes) + } + for i := range lons { + if geo.BoundingBoxContains(lons[i], lats[i], + minLon, minLat, maxLon, maxLat) { + return true + } + } + } + return false + } +} + +type geoRangeCompute struct { + preallocBytesLen int + preallocBytes []byte + sminLon, sminLat, smaxLon, smaxLat float64 + checkBoundaries bool + onBoundary, notOnBoundary [][]byte + isIndexed func(term []byte) bool +} + +func (grc *geoRangeCompute) makePrefixCoded(in int64, shift uint) (rv numeric.PrefixCoded) { + if len(grc.preallocBytes) <= 0 { + grc.preallocBytesLen = grc.preallocBytesLen * 2 + grc.preallocBytes = make([]byte, grc.preallocBytesLen) + } + + rv, grc.preallocBytes, _ = numeric.NewPrefixCodedInt64Prealloc(in, shift, grc.preallocBytes) + + return rv +} + +func (grc *geoRangeCompute) computeGeoRange(term uint64, shift uint) { + split := term | uint64(0x1)<> 1 + + within := res%document.GeoPrecisionStep == 0 && + geo.RectWithin(minLon, minLat, maxLon, maxLat, + grc.sminLon, grc.sminLat, grc.smaxLon, grc.smaxLat) + if within || (level == geoDetailLevel && + geo.RectIntersects(minLon, minLat, maxLon, maxLat, + grc.sminLon, grc.sminLat, grc.smaxLon, grc.smaxLat)) { + codedTerm := grc.makePrefixCoded(int64(start), res) + if grc.isIndexed(codedTerm) { + if !within && grc.checkBoundaries { + grc.onBoundary = append(grc.onBoundary, codedTerm) + } else { + grc.notOnBoundary = append(grc.notOnBoundary, codedTerm) + } + } + } else if level < geoDetailLevel && + geo.RectIntersects(minLon, minLat, maxLon, maxLat, + grc.sminLon, grc.sminLat, grc.smaxLon, grc.smaxLat) { + grc.computeGeoRange(start, res-1) + } +} diff --git a/search/searcher/search_geoboundingbox_test.go b/search/searcher/search_geoboundingbox_test.go new file mode 100644 index 0000000..4715b5b --- /dev/null +++ b/search/searcher/search_geoboundingbox_test.go @@ -0,0 +1,317 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/index/upsidedown" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func TestGeoBoundingBox(t *testing.T) { + tests := []struct { + minLon float64 + minLat float64 + maxLon float64 + maxLat float64 + field string + want []string + }{ + {10.001, 10.001, 20.002, 20.002, "loc", nil}, + {0.001, 0.001, 0.002, 0.002, "loc", []string{"a"}}, + {0.001, 0.001, 1.002, 1.002, "loc", []string{"a", "b"}}, + {0.001, 0.001, 9.002, 9.002, "loc", []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}}, + // same upper-left, bottom-right point + {25, 25, 25, 25, "loc", nil}, + // box that would return points, but points reversed + {0.002, 0.002, 0.001, 0.001, "loc", nil}, + } + + i := setupGeo(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for _, test := range tests { + got, err := testGeoBoundingBoxSearch(indexReader, test.minLon, test.minLat, test.maxLon, test.maxLat, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("expected %v, got %v for %f %f %f %f %s", test.want, got, test.minLon, test.minLat, test.maxLon, test.maxLat, test.field) + } + + } +} + +func testGeoBoundingBoxSearch(i index.IndexReader, minLon, minLat, maxLon, maxLat float64, field string) ([]string, error) { + var rv []string + gbs, err := NewGeoBoundingBoxSearcher(context.TODO(), i, minLon, minLat, maxLon, maxLat, field, 1.0, search.SearcherOptions{}, true) + if err != nil { + return nil, err + } + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(gbs.DocumentMatchPoolSize(), 0), + } + docMatch, err := gbs.Next(ctx) + for docMatch != nil && err == nil { + rv = append(rv, string(docMatch.IndexInternalID)) + docMatch, err = gbs.Next(ctx) + } + if err != nil { + return nil, err + } + return rv, nil +} + +func setupGeo(t *testing.T) index.Index { + analysisQueue := index.NewAnalysisQueue(1) + i, err := upsidedown.NewUpsideDownCouch( + gtreap.Name, + map[string]interface{}{ + "path": "", + }, + analysisQueue) + if err != nil { + t.Fatal(err) + } + err = i.Open() + if err != nil { + t.Fatal(err) + } + doc := document.NewDocument("a") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, 0.0015, 0.0015)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + doc = document.NewDocument("b") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, 1.0015, 1.0015)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + doc = document.NewDocument("c") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, 2.0015, 2.0015)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + doc = document.NewDocument("d") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, 3.0015, 3.0015)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + doc = document.NewDocument("e") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, 4.0015, 4.0015)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + doc = document.NewDocument("f") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, 5.0015, 5.0015)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + doc = document.NewDocument("g") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, 6.0015, 6.0015)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + doc = document.NewDocument("h") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, 7.0015, 7.0015)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + doc = document.NewDocument("i") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, 8.0015, 8.0015)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + doc = document.NewDocument("j") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, 9.0015, 9.0015)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + return i +} + +func TestComputeGeoRange(t *testing.T) { + tests := []struct { + degs float64 + onBoundary int + offBoundary int + err string + }{ + {0.01, 4, 0, ""}, + {0.1, 56, 144, ""}, + {100.0, 32768, 258560, ""}, + } + + for testi, test := range tests { + onBoundaryRes, offBoundaryRes, err := ComputeGeoRange(context.TODO(), 0, GeoBitsShift1Minus1, + -1.0*test.degs, -1.0*test.degs, test.degs, test.degs, true, nil, "") + if (err != nil) != (test.err != "") { + t.Errorf("test: %+v, err: %v", test, err) + } + if len(onBoundaryRes) != test.onBoundary { + t.Errorf("test: %+v, onBoundaryRes: %v", test, len(onBoundaryRes)) + } + if len(offBoundaryRes) != test.offBoundary { + t.Errorf("test: %+v, offBoundaryRes: %v", test, len(offBoundaryRes)) + } + + onBROrig, offBROrig := origComputeGeoRange(0, GeoBitsShift1Minus1, + -1.0*test.degs, -1.0*test.degs, test.degs, test.degs, true) + if !reflect.DeepEqual(onBoundaryRes, onBROrig) { + t.Errorf("testi: %d, test: %+v, onBoundaryRes != onBROrig,\n onBoundaryRes:%v,\n onBROrig: %v", + testi, test, onBoundaryRes, onBROrig) + } + if !reflect.DeepEqual(offBoundaryRes, offBROrig) { + t.Errorf("testi: %d, test: %+v, offBoundaryRes, offBROrig,\n offBoundaryRes: %v,\n offBROrig: %v", + testi, test, offBoundaryRes, offBROrig) + } + } +} + +// -------------------------------------------------------------------- + +func BenchmarkComputeGeoRangePt01(b *testing.B) { + onBoundary := 4 + offBoundary := 0 + benchmarkComputeGeoRange(b, -0.01, -0.01, 0.01, 0.01, onBoundary, offBoundary) +} + +func BenchmarkComputeGeoRangePt1(b *testing.B) { + onBoundary := 56 + offBoundary := 144 + benchmarkComputeGeoRange(b, -0.1, -0.1, 0.1, 0.1, onBoundary, offBoundary) +} + +func BenchmarkComputeGeoRange10(b *testing.B) { + onBoundary := 5464 + offBoundary := 53704 + benchmarkComputeGeoRange(b, -10.0, -10.0, 10.0, 10.0, onBoundary, offBoundary) +} + +func BenchmarkComputeGeoRange100(b *testing.B) { + onBoundary := 32768 + offBoundary := 258560 + benchmarkComputeGeoRange(b, -100.0, -100.0, 100.0, 100.0, onBoundary, offBoundary) +} + +// -------------------------------------------------------------------- + +func benchmarkComputeGeoRange(b *testing.B, + minLon, minLat, maxLon, maxLat float64, onBoundary, offBoundary int, +) { + checkBoundaries := true + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + onBoundaryRes, offBoundaryRes, err := ComputeGeoRange(context.TODO(), 0, GeoBitsShift1Minus1, minLon, minLat, maxLon, maxLat, checkBoundaries, nil, "") + if err != nil { + b.Fatalf("expected no err") + } + if len(onBoundaryRes) != onBoundary || len(offBoundaryRes) != offBoundary { + b.Fatalf("boundaries not matching") + } + } +} + +// -------------------------------------------------------------------- + +// original, non-optimized implementation of ComputeGeoRange +func origComputeGeoRange(term uint64, shift uint, + sminLon, sminLat, smaxLon, smaxLat float64, + checkBoundaries bool) ( + onBoundary [][]byte, notOnBoundary [][]byte, +) { + split := term | uint64(0x1)<> 1 + + within := res%document.GeoPrecisionStep == 0 && + geo.RectWithin(minLon, minLat, maxLon, maxLat, + sminLon, sminLat, smaxLon, smaxLat) + if within || (level == geoDetailLevel && + geo.RectIntersects(minLon, minLat, maxLon, maxLat, + sminLon, sminLat, smaxLon, smaxLat)) { + if !within && checkBoundaries { + return [][]byte{ + numeric.MustNewPrefixCodedInt64(int64(start), res), + }, nil + } + return nil, + [][]byte{ + numeric.MustNewPrefixCodedInt64(int64(start), res), + } + } else if level < geoDetailLevel && + geo.RectIntersects(minLon, minLat, maxLon, maxLat, + sminLon, sminLat, smaxLon, smaxLat) { + return origComputeGeoRange(start, res-1, sminLon, sminLat, smaxLon, smaxLat, + checkBoundaries) + } + return nil, nil +} diff --git a/search/searcher/search_geopointdistance.go b/search/searcher/search_geopointdistance.go new file mode 100644 index 0000000..7591bcc --- /dev/null +++ b/search/searcher/search_geopointdistance.go @@ -0,0 +1,157 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func NewGeoPointDistanceSearcher(ctx context.Context, indexReader index.IndexReader, centerLon, + centerLat, dist float64, field string, boost float64, + options search.SearcherOptions) (search.Searcher, error) { + var rectSearcher search.Searcher + if tp, ok := indexReader.(index.SpatialIndexPlugin); ok { + sp, err := tp.GetSpatialAnalyzerPlugin("s2") + if err == nil { + terms := sp.GetQueryTokens(geo.NewPointDistance(centerLat, + centerLon, dist)) + rectSearcher, err = NewMultiTermSearcher(ctx, indexReader, terms, + field, boost, options, false) + if err != nil { + return nil, err + } + } + } + + // indexes without the spatial plugin override would get + // initialized here. + if rectSearcher == nil { + // compute bounding box containing the circle + topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, err := + geo.RectFromPointDistance(centerLon, centerLat, dist) + if err != nil { + return nil, err + } + + // build a searcher for the box + rectSearcher, err = boxSearcher(ctx, indexReader, + topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, + field, boost, options, false) + if err != nil { + return nil, err + } + } + + dvReader, err := indexReader.DocValueReader([]string{field}) + if err != nil { + return nil, err + } + + // wrap it in a filtering searcher which checks the actual distance + return NewFilteringSearcher(ctx, rectSearcher, + buildDistFilter(ctx, dvReader, centerLon, centerLat, dist)), nil +} + +// boxSearcher builds a searcher for the described bounding box +// if the desired box crosses the dateline, it is automatically split into +// two boxes joined through a disjunction searcher +func boxSearcher(ctx context.Context, indexReader index.IndexReader, + topLeftLon, topLeftLat, bottomRightLon, bottomRightLat float64, + field string, boost float64, options search.SearcherOptions, checkBoundaries bool) ( + search.Searcher, error) { + if bottomRightLon < topLeftLon { + // cross date line, rewrite as two parts + + leftSearcher, err := NewGeoBoundingBoxSearcher(ctx, indexReader, + -180, bottomRightLat, bottomRightLon, topLeftLat, + field, boost, options, checkBoundaries) + if err != nil { + return nil, err + } + rightSearcher, err := NewGeoBoundingBoxSearcher(ctx, indexReader, + topLeftLon, bottomRightLat, 180, topLeftLat, field, boost, options, + checkBoundaries) + if err != nil { + _ = leftSearcher.Close() + return nil, err + } + + boxSearcher, err := NewDisjunctionSearcher(ctx, indexReader, + []search.Searcher{leftSearcher, rightSearcher}, 0, options) + if err != nil { + _ = leftSearcher.Close() + _ = rightSearcher.Close() + return nil, err + } + return boxSearcher, nil + } + + // build geoboundingbox searcher for that bounding box + boxSearcher, err := NewGeoBoundingBoxSearcher(ctx, indexReader, + topLeftLon, bottomRightLat, bottomRightLon, topLeftLat, field, boost, + options, checkBoundaries) + if err != nil { + return nil, err + } + return boxSearcher, nil +} + +func buildDistFilter(ctx context.Context, dvReader index.DocValueReader, + centerLon, centerLat, maxDist float64) FilterFunc { + // reuse the following for each document match that is checked using the filter + var lons, lats []float64 + var found bool + dvVisitor := func(_ string, term []byte) { + if found { + // avoid redundant work if already found + return + } + // only consider the values which are shifted 0 + prefixCoded := numeric.PrefixCoded(term) + shift, err := prefixCoded.Shift() + if err == nil && shift == 0 { + i64, err := prefixCoded.Int64() + if err == nil { + lons = append(lons, geo.MortonUnhashLon(uint64(i64))) + lats = append(lats, geo.MortonUnhashLat(uint64(i64))) + found = true + } + } + } + return func(sctx *search.SearchContext, d *search.DocumentMatch) bool { + // check geo matches against all numeric type terms indexed + lons, lats = lons[:0], lats[:0] + found = false + if err := dvReader.VisitDocValues(d.IndexInternalID, dvVisitor); err == nil && found { + bytes := dvReader.BytesRead() + if bytes > 0 { + reportIOStats(ctx, bytes) + search.RecordSearchCost(ctx, search.AddM, bytes) + } + for i := range lons { + dist := geo.Haversin(lons[i], lats[i], centerLon, centerLat) + if dist <= maxDist/1000 { + return true + } + } + } + return false + } +} diff --git a/search/searcher/search_geopointdistance_test.go b/search/searcher/search_geopointdistance_test.go new file mode 100644 index 0000000..20f65c8 --- /dev/null +++ b/search/searcher/search_geopointdistance_test.go @@ -0,0 +1,157 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func TestGeoPointDistanceSearcher(t *testing.T) { + tests := []struct { + centerLon float64 + centerLat float64 + dist float64 + field string + want []string + }{ + // approx 110567m per degree at equator + {0.0, 0.0, 0, "loc", nil}, + {0.0, 0.0, 110567, "loc", []string{"a"}}, + {0.0, 0.0, 2 * 110567, "loc", []string{"a", "b"}}, + // stretching our approximation here + {0.0, 0.0, 15 * 110567, "loc", []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}}, + } + + i := setupGeo(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for _, test := range tests { + got, err := testGeoPointDistanceSearch(indexReader, test.centerLon, test.centerLat, test.dist, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("expected %v, got %v for %f %f %f %s", test.want, got, test.centerLon, test.centerLat, test.dist, test.field) + } + + } +} + +func testGeoPointDistanceSearch(i index.IndexReader, centerLon, centerLat, dist float64, field string) ([]string, error) { + var rv []string + gds, err := NewGeoPointDistanceSearcher(context.TODO(), i, centerLon, centerLat, dist, field, 1.0, search.SearcherOptions{}) + if err != nil { + return nil, err + } + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(gds.DocumentMatchPoolSize(), 0), + } + docMatch, err := gds.Next(ctx) + for docMatch != nil && err == nil { + rv = append(rv, string(docMatch.IndexInternalID)) + docMatch, err = gds.Next(ctx) + } + if err != nil { + return nil, err + } + return rv, nil +} + +func TestGeoPointDistanceCompare(t *testing.T) { + tests := []struct { + docLat, docLon float64 + centerLat, centerLon float64 + distance string + }{ + // Data points originally from MB-33454. + { + docLat: 33.718, + docLon: -116.8293, + centerLat: 39.59000587, + centerLon: -119.22998428, + distance: "10000mi", + }, + { + docLat: 41.1305, + docLon: -121.6587, + centerLat: 61.28, + centerLon: -149.34, + distance: "10000mi", + }, + } + + for testi, test := range tests { + // compares the results from ComputeGeoRange with original, non-optimized version + compare := func(desc string, + minLon, minLat, maxLon, maxLat float64, checkBoundaries bool, + ) { + // do math to produce list of terms needed for this search + onBoundaryRes, offBoundaryRes, err := ComputeGeoRange(context.TODO(), 0, GeoBitsShift1Minus1, + minLon, minLat, maxLon, maxLat, checkBoundaries, nil, "") + if err != nil { + t.Fatal(err) + } + + onBROrig, offBROrig := origComputeGeoRange(0, GeoBitsShift1Minus1, + minLon, minLat, maxLon, maxLat, checkBoundaries) + if !reflect.DeepEqual(onBoundaryRes, onBROrig) { + t.Fatalf("testi: %d, test: %+v, desc: %s, onBoundaryRes != onBROrig,\n onBoundaryRes:%v,\n onBROrig: %v", + testi, test, desc, onBoundaryRes, onBROrig) + } + if !reflect.DeepEqual(offBoundaryRes, offBROrig) { + t.Fatalf("testi: %d, test: %+v, desc: %s, offBoundaryRes, offBROrig,\n offBoundaryRes: %v,\n offBROrig: %v", + testi, test, desc, offBoundaryRes, offBROrig) + } + } + + // follow the general approach of the GeoPointDistanceSearcher... + dist, err := geo.ParseDistance(test.distance) + if err != nil { + t.Fatal(err) + } + + topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, err := geo.RectFromPointDistance(test.centerLon, test.centerLat, dist) + if err != nil { + t.Fatal(err) + } + + if bottomRightLon < topLeftLon { + // crosses date line, rewrite as two parts + compare("-180/f", -180, bottomRightLat, bottomRightLon, topLeftLat, false) + compare("-180/t", -180, bottomRightLat, bottomRightLon, topLeftLat, true) + + compare("180/f", topLeftLon, bottomRightLat, 180, topLeftLat, false) + compare("180/t", topLeftLon, bottomRightLat, 180, topLeftLat, true) + } else { + compare("reg/f", topLeftLon, bottomRightLat, bottomRightLon, topLeftLat, false) + compare("reg/t", topLeftLon, bottomRightLat, bottomRightLon, topLeftLat, true) + } + } +} diff --git a/search/searcher/search_geopolygon.go b/search/searcher/search_geopolygon.go new file mode 100644 index 0000000..fb6e09b --- /dev/null +++ b/search/searcher/search_geopolygon.go @@ -0,0 +1,154 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "fmt" + "math" + + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func NewGeoBoundedPolygonSearcher(ctx context.Context, indexReader index.IndexReader, + coordinates []geo.Point, field string, boost float64, + options search.SearcherOptions) (search.Searcher, error) { + if len(coordinates) < 3 { + return nil, fmt.Errorf("Too few points specified for the polygon boundary") + } + + var rectSearcher search.Searcher + if sr, ok := indexReader.(index.SpatialIndexPlugin); ok { + tp, err := sr.GetSpatialAnalyzerPlugin("s2") + if err == nil { + terms := tp.GetQueryTokens(geo.NewBoundedPolygon(coordinates)) + rectSearcher, err = NewMultiTermSearcher(ctx, indexReader, terms, + field, boost, options, false) + if err != nil { + return nil, err + } + } + } + + // indexes without the spatial plugin override would get + // initialized here. + if rectSearcher == nil { + // compute the bounding box enclosing the polygon + topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, err := + geo.BoundingRectangleForPolygon(coordinates) + if err != nil { + return nil, err + } + + // build a searcher for the bounding box on the polygon + rectSearcher, err = boxSearcher(ctx, indexReader, + topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, + field, boost, options, true) + if err != nil { + return nil, err + } + } + + dvReader, err := indexReader.DocValueReader([]string{field}) + if err != nil { + return nil, err + } + + // wrap it in a filtering searcher that checks for the polygon inclusivity + return NewFilteringSearcher(ctx, rectSearcher, + buildPolygonFilter(ctx, dvReader, field, coordinates)), nil +} + +const float64EqualityThreshold = 1e-6 + +func almostEqual(a, b float64) bool { + return math.Abs(a-b) <= float64EqualityThreshold +} + +// buildPolygonFilter returns true if the point lies inside the +// polygon. It is based on the ray-casting technique as referred +// here: https://wrf.ecse.rpi.edu/nikola/pubdetails/pnpoly.html +func buildPolygonFilter(ctx context.Context, dvReader index.DocValueReader, field string, + coordinates []geo.Point) FilterFunc { + // reuse the following for each document match that is checked using the filter + var lons, lats []float64 + var found bool + dvVisitor := func(_ string, term []byte) { + if found { + // avoid redundant work if already found + return + } + // only consider the values which are shifted 0 + prefixCoded := numeric.PrefixCoded(term) + shift, err := prefixCoded.Shift() + if err == nil && shift == 0 { + i64, err := prefixCoded.Int64() + if err == nil { + lons = append(lons, geo.MortonUnhashLon(uint64(i64))) + lats = append(lats, geo.MortonUnhashLat(uint64(i64))) + found = true + } + } + } + rayIntersectsSegment := func(point, a, b geo.Point) bool { + return (a.Lat > point.Lat) != (b.Lat > point.Lat) && + point.Lon < (b.Lon-a.Lon)*(point.Lat-a.Lat)/(b.Lat-a.Lat)+a.Lon + } + return func(sctx *search.SearchContext, d *search.DocumentMatch) bool { + // check geo matches against all numeric type terms indexed + lons, lats = lons[:0], lats[:0] + found = false + // Note: this approach works for points which are strictly inside + // the polygon. ie it might fail for certain points on the polygon boundaries. + if err := dvReader.VisitDocValues(d.IndexInternalID, dvVisitor); err == nil && found { + bytes := dvReader.BytesRead() + if bytes > 0 { + reportIOStats(ctx, bytes) + search.RecordSearchCost(ctx, search.AddM, bytes) + } + nVertices := len(coordinates) + if len(coordinates) < 3 { + return false + } + + for i := range lons { + pt := geo.Point{Lon: lons[i], Lat: lats[i]} + inside := rayIntersectsSegment(pt, coordinates[len(coordinates)-1], coordinates[0]) + // check for a direct vertex match + if almostEqual(coordinates[0].Lat, lats[i]) && + almostEqual(coordinates[0].Lon, lons[i]) { + return true + } + + for j := 1; j < nVertices; j++ { + if almostEqual(coordinates[j].Lat, lats[i]) && + almostEqual(coordinates[j].Lon, lons[i]) { + return true + } + if rayIntersectsSegment(pt, coordinates[j-1], coordinates[j]) { + inside = !inside + } + } + if inside { + return true + } + } + } + return false + } +} diff --git a/search/searcher/search_geopolygon_test.go b/search/searcher/search_geopolygon_test.go new file mode 100644 index 0000000..d7dd0e3 --- /dev/null +++ b/search/searcher/search_geopolygon_test.go @@ -0,0 +1,409 @@ +// Copyright (c) 2019 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/index/upsidedown" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func TestSimpleGeoPolygons(t *testing.T) { + tests := []struct { + polygon []geo.Point + field string + want []string + }{ + // test points inside a triangle & on vertices + // r, s - inside and t,u - on vertices. + {[]geo.Point{{Lon: 1.0, Lat: 1.0}, {Lon: 2.0, Lat: 1.9}, {Lon: 2.0, Lat: 1.0}}, "loc", []string{"r", "s", "t", "u"}}, + // non overlapping polygon for the indexed documents + {[]geo.Point{{Lon: 3.0, Lat: 1.0}, {Lon: 4.0, Lat: 2.5}, {Lon: 3.0, Lat: 2}}, "loc", nil}, + } + i := setupGeoPolygonPoints(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for _, test := range tests { + got, err := testGeoPolygonSearch(indexReader, test.polygon, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("expected %v, got %v for polygon: %+v", test.want, got, test.polygon) + } + } +} + +func TestRealGeoPolygons(t *testing.T) { + tests := []struct { + polygon []geo.Point + field string + want []string + }{ + {[]geo.Point{ + {Lon: -80.881, Lat: 35.282}, + {Lon: -80.858, Lat: 35.281}, + {Lon: -80.864, Lat: 35.270}, + }, "loc", []string{"k", "l"}}, + {[]geo.Point{ + {Lon: -82.467, Lat: 36.356}, + {Lon: -78.127, Lat: 36.321}, + {Lon: -80.555, Lat: 32.932}, + {Lon: -84.807, Lat: 33.111}, + }, "loc", []string{"k", "l", "m"}}, + // same polygon vertices + {[]geo.Point{{Lon: -82.467, Lat: 36.356}, {Lon: -82.467, Lat: 36.356}, {Lon: -82.467, Lat: 36.356}, {Lon: -82.467, Lat: 36.356}}, "loc", nil}, + // non-overlaping polygon + {[]geo.Point{{Lon: -89.113, Lat: 36.400}, {Lon: -93.947, Lat: 36.471}, {Lon: -93.947, Lat: 34.031}}, "loc", nil}, + // concave polygon with a document `n` residing inside the hands, but outside the polygon + {[]geo.Point{{Lon: -71.65, Lat: 42.446}, {Lon: -71.649, Lat: 42.428}, {Lon: -71.640, Lat: 42.445}, {Lon: -71.649, Lat: 42.435}}, "loc", nil}, + // V like concave polygon with a document 'p' residing inside the bottom corner + {[]geo.Point{{Lon: -80.304, Lat: 40.740}, {Lon: -80.038, Lat: 40.239}, {Lon: -79.562, Lat: 40.786}, {Lon: -80.018, Lat: 40.328}}, "loc", []string{"p"}}, + {[]geo.Point{ + {Lon: -111.918, Lat: 33.515}, + {Lon: -111.938, Lat: 33.494}, + {Lon: -111.944, Lat: 33.481}, + {Lon: -111.886, Lat: 33.517}, + {Lon: -111.919, Lat: 33.468}, + {Lon: -111.929, Lat: 33.508}, + }, "loc", []string{"q"}}, + // real points near cb bangalore + {[]geo.Point{ + {Lat: 12.974872, Lon: 77.607749}, + {Lat: 12.971725, Lon: 77.610110}, + {Lat: 12.972530, Lon: 77.606912}, + {Lat: 12.975112, Lon: 77.603780}, + }, "loc", []string{"amoeba", "communiti"}}, + } + + i := setupGeoPolygonPoints(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for _, test := range tests { + got, err := testGeoPolygonSearch(indexReader, test.polygon, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("expected %v, got %v for polygon: %+v", test.want, got, test.polygon) + } + } +} + +func TestGeoRectanglePolygon(t *testing.T) { + tests := []struct { + polygon []geo.Point + field string + want []string + }{ + { + []geo.Point{{Lon: 0, Lat: 0}, {Lon: 0, Lat: 50}, {Lon: 50, Lat: 50}, {Lon: 50, Lat: 0}, {Lon: 0, Lat: 0}}, + "loc", + []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}, + }, + } + + i := setupGeo(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for _, test := range tests { + got, err := testGeoPolygonSearch(indexReader, test.polygon, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("expected %v, got %v for polygon: %+v", test.want, got, test.polygon) + } + } +} + +func testGeoPolygonSearch(i index.IndexReader, polygon []geo.Point, field string) ([]string, error) { + var rv []string + gbs, err := NewGeoBoundedPolygonSearcher(context.TODO(), i, polygon, field, 1.0, search.SearcherOptions{}) + if err != nil { + return nil, err + } + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(gbs.DocumentMatchPoolSize(), 0), + } + docMatch, err := gbs.Next(ctx) + for docMatch != nil && err == nil { + rv = append(rv, string(docMatch.IndexInternalID)) + docMatch, err = gbs.Next(ctx) + } + if err != nil { + return nil, err + } + return rv, nil +} + +func setupGeoPolygonPoints(t *testing.T) index.Index { + analysisQueue := index.NewAnalysisQueue(1) + i, err := upsidedown.NewUpsideDownCouch( + gtreap.Name, + map[string]interface{}{ + "path": "", + }, + analysisQueue) + if err != nil { + t.Fatal(err) + } + err = i.Open() + if err != nil { + t.Fatal(err) + } + + doc := document.NewDocument("k") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, -80.86469327, 35.2782)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("l") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, -80.8713, 35.28138)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("m") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, -84.25, 33.153)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("n") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, -89.992, 35.063)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("o") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, -71.648, 42.437)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("p") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, -80.016, 40.314)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("q") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, -111.919, 33.494)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("r") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, 1.5, 1.1)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("s") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, 2, 1.5)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("t") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, 2.0, 1.9)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("u") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, 2.0, 1.0)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("amoeba") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, 77.60490, 12.97467)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("communiti") + doc.AddField(document.NewGeoPointField("loc", []uint64{}, 77.608237, 12.97237)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + return i +} + +type geoPoint struct { + title string + lon float64 + lat float64 +} + +// Test points inside a complex self intersecting polygon +func TestComplexGeoPolygons(t *testing.T) { + tests := []struct { + polygon []geo.Point + points []geoPoint + field string + want []string + }{ + /* + /\ /\ + /__\____/__\ + \ / + \/ + */ + // a, b, c - inside and d - on vertices. + { + []geo.Point{ + {Lon: 6.0, Lat: 2.0}, + {Lon: 3.0, Lat: 4.0}, + {Lon: 9.0, Lat: 6.0}, + {Lon: 3.0, Lat: 8.0}, + {Lon: 6.0, Lat: 10.0}, + {Lon: 6.0, Lat: 2.0}, + }, + []geoPoint{ + {title: "a", lon: 3, lat: 4}, + {title: "b", lon: 7, lat: 6}, + {title: "c", lon: 4, lat: 8.1}, + {title: "d", lon: 6, lat: 10.0}, + {title: "e", lon: 5, lat: 6}, + {title: "f", lon: 7, lat: 5}, + }, + "loc", + []string{"a", "b", "c", "d"}, + }, + /* + ____ + \ / + \/ + /\ + /__\ + */ + { + []geo.Point{ + {Lon: 7.0, Lat: 2.0}, + {Lon: 1.0, Lat: 8.0}, + {Lon: 1.0, Lat: 2.0}, + {Lon: 7.0, Lat: 8.0}, + {Lon: 7.0, Lat: 2.0}, + }, + []geoPoint{ + {title: "a", lon: 6, lat: 5}, + {title: "b", lon: 5, lat: 5}, + {title: "c", lon: 3, lat: 5.0}, + {title: "d", lon: 2, lat: 4.0}, + {title: "e", lon: 5, lat: 3}, + {title: "f", lon: 4, lat: 4}, + }, + "loc", + []string{"a", "b", "c", "d"}, + }, + } + + for _, test := range tests { + i := setupComplexGeoPolygonPoints(t, test.points) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + got, err := testGeoPolygonSearch(indexReader, test.polygon, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("expected %v, got %v for polygon: %+v", test.want, got, test.polygon) + } + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + } +} + +func setupComplexGeoPolygonPoints(t *testing.T, points []geoPoint) index.Index { + analysisQueue := index.NewAnalysisQueue(1) + i, err := upsidedown.NewUpsideDownCouch( + gtreap.Name, + map[string]interface{}{ + "path": "", + }, + analysisQueue) + if err != nil { + t.Fatal(err) + } + err = i.Open() + if err != nil { + t.Fatal(err) + } + for _, point := range points { + doc := document.NewDocument(point.title) + doc.AddField(document.NewGeoPointField("loc", []uint64{}, point.lon, point.lat)) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + } + return i +} diff --git a/search/searcher/search_geoshape.go b/search/searcher/search_geoshape.go new file mode 100644 index 0000000..cd020fa --- /dev/null +++ b/search/searcher/search_geoshape.go @@ -0,0 +1,137 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "bytes" + "context" + + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/geo/geojson" + "github.com/blevesearch/geo/s2" +) + +func NewGeoShapeSearcher(ctx context.Context, indexReader index.IndexReader, shape index.GeoJSON, + relation string, field string, boost float64, + options search.SearcherOptions, +) (search.Searcher, error) { + var err error + var spatialPlugin index.SpatialAnalyzerPlugin + + // check for the spatial plugin from the index. + if sr, ok := indexReader.(index.SpatialIndexPlugin); ok { + spatialPlugin, _ = sr.GetSpatialAnalyzerPlugin("s2") + } + + if spatialPlugin == nil { + // fallback to the default spatial plugin(s2). + spatialPlugin = geo.GetSpatialAnalyzerPlugin("s2") + } + + // obtain the query tokens. + terms := spatialPlugin.GetQueryTokens(shape) + mSearcher, err := NewMultiTermSearcher(ctx, indexReader, terms, + field, boost, options, false) + if err != nil { + return nil, err + } + + dvReader, err := indexReader.DocValueReader([]string{field}) + if err != nil { + return nil, err + } + + return NewFilteringSearcher(ctx, mSearcher, buildRelationFilterOnShapes(ctx, dvReader, field, relation, shape)), nil +} + +func buildRelationFilterOnShapes(ctx context.Context, dvReader index.DocValueReader, field string, + relation string, shape index.GeoJSON, +) FilterFunc { + // this is for accumulating the shape's actual complete value + // spread across multiple docvalue visitor callbacks. + var dvShapeValue []byte + var startReading, finishReading, found bool + var reader *bytes.Reader + + var bufPool *s2.GeoBufferPool + if bufPoolCallback, ok := ctx.Value(search.GeoBufferPoolCallbackKey).(search.GeoBufferPoolCallbackFunc); ok { + bufPool = bufPoolCallback() + } + + dvVisitor := func(_ string, term []byte) { + if found { + // avoid redundant work if already found + return + } + tl := len(term) + // only consider the values which are GlueBytes prefixed or + // if it had already started reading the shape bytes from previous callbacks. + if startReading || tl > geo.GlueBytesOffset { + + if !startReading && bytes.Equal(geo.GlueBytes, term[:geo.GlueBytesOffset]) { + startReading = true + + if bytes.Equal(geo.GlueBytes, term[tl-geo.GlueBytesOffset:]) { + term = term[:tl-geo.GlueBytesOffset] + finishReading = true + } + + dvShapeValue = append(dvShapeValue, term[geo.GlueBytesOffset:]...) + + } else if startReading && !finishReading { + if tl > geo.GlueBytesOffset && + bytes.Equal(geo.GlueBytes, term[tl-geo.GlueBytesOffset:]) { + term = term[:tl-geo.GlueBytesOffset] + finishReading = true + } + + dvShapeValue = append(dvShapeValue, index.DocValueTermSeparator) + dvShapeValue = append(dvShapeValue, term...) + } + + // apply the filter once the entire docvalue is finished reading. + if finishReading { + v, err := geojson.FilterGeoShapesOnRelation(shape, dvShapeValue, relation, &reader, bufPool) + if err == nil && v { + found = true + } + + dvShapeValue = dvShapeValue[:0] + startReading = false + finishReading = false + } + } + } + + return func(sctx *search.SearchContext, d *search.DocumentMatch) bool { + // reset state variables for each document + found = false + startReading = false + finishReading = false + dvShapeValue = dvShapeValue[:0] + if err := dvReader.VisitDocValues(d.IndexInternalID, dvVisitor); err == nil && found { + bytes := dvReader.BytesRead() + if bytes > 0 { + reportIOStats(ctx, bytes) + search.RecordSearchCost(ctx, search.AddM, bytes) + } + return found + } + + return false + } +} diff --git a/search/searcher/search_geoshape_circle_test.go b/search/searcher/search_geoshape_circle_test.go new file mode 100644 index 0000000..7f02256 --- /dev/null +++ b/search/searcher/search_geoshape_circle_test.go @@ -0,0 +1,521 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap" + index "github.com/blevesearch/bleve_index_api" +) + +func TestGeoJsonCircleIntersectsQuery(t *testing.T) { + tests := []struct { + centrePoint []float64 + radiusInMeters string + field string + want []string + }{ + // test intersecting query circle for polygon1. + { + []float64{77.68115043640137, 12.94663769274367}, + "200m", + "geometry", + []string{"polygon1"}, + }, + + // test intersecting query circle for polygon1, circle1 and linestring1. + { + []float64{77.68115043640137, 12.94663769274367}, + "750m", + "geometry", + []string{"polygon1", "circle1", "linestring1"}, + }, + + // test intersecting query circle for linestring2. + { + []float64{77.69591331481932, 12.92756503709986}, + "250m", + "geometry", + []string{"linestring2"}, + }, + + // test intersecting query circle for circle1. + {[]float64{77.6767, 12.9422}, "250m", "geometry", []string{"circle1"}}, + + // test intersecting query circle for point1, envelope1 and linestring3. + { + []float64{81.243896484375, 26.22444694563432}, + "90000m", + "geometry", + []string{"point1", "envelope1", "linestring3"}, + }, + + // test intersecting query circle for envelope. + { + []float64{79.98458862304688, 25.339061458818374}, + "1250m", + "geometry", + []string{"envelope1"}, + }, + + // test intersecting query circle for multipoint. + { + []float64{81.87346458435059, 25.41505910223247}, + "200m", + "geometry", + []string{"multipoint1"}, + }, + + // test intersecting query circle for multilinestring. + { + []float64{81.8669843673706, 25.512661276952272}, + "90m", + "geometry", + []string{"multilinestring1"}, + }, + } + + i := setupGeoJsonShapesIndexForCircleQuery(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapeCircleRelationQuery("intersects", + indexReader, test.centrePoint, test.radiusInMeters, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", + n, test.want, got, test.centrePoint) + } + } +} + +func TestGeoJsonCircleWithInQuery(t *testing.T) { + tests := []struct { + centrePoint []float64 + radiusInMeters string + field string + want []string + }{ + // test query circle containing polygon2 and multilinestring2. + { + []float64{81.85981750488281, 25.546778150624146}, + "3700m", + "geometry", + []string{"polygon2", "multilinestring2"}, + }, + + // test query circle containing multilinestring2. + { + []float64{81.85981750488281, 25.546778150624146}, + "3250m", + "geometry", + []string{"multilinestring2"}, + }, + + // test query circle containing multipoint1. + { + []float64{81.88599586486816, 25.425756968727935}, + "1650m", + "geometry", + []string{"multipoint1"}, + }, + + // test query circle containing circle2. + { + []float64{82.09362030029297, 25.546313513788725}, + "1280m", + "geometry", + []string{"envelope2", "circle2"}, + }, + + // test query circle containing envelope2 and circle2. + { + []float64{82.10289001464844, 25.544919592476727}, + "700m", + "geometry", + []string{"envelope2", "circle2"}, + }, + + // test query circle containing point1 and linestring3. + { + []float64{81.27685546875, 26.1899475672235}, + "5600m", + "geometry", + []string{"point1", "linestring3"}, + }, + } + + i := setupGeoJsonShapesIndexForCircleQuery(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapeCircleRelationQuery("within", indexReader, test.centrePoint, test.radiusInMeters, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", n, test.want, got, test.centrePoint) + } + } +} + +func TestGeoJsonCircleContainsQuery(t *testing.T) { + tests := []struct { + centrePoint []float64 + radiusInMeters string + field string + want []string + }{ + // test query circle within polygon3. + { + []float64{8.549551963806152, 47.3759038562437}, + "180m", + "geometry", + []string{"polygon3"}, + }, + + // test query circle containing envelope3. + { + []float64{8.551011085510254, 47.380117626829275}, + "75m", + "geometry", + []string{"envelope3"}, + }, + + // test query circle exceeding envelope3 with a few meters. + { + []float64{8.551011085510254, 47.380117626829275}, + "78m", + "geometry", nil, + }, + + // test query circle containing circle3. + { + []float64{8.535819053649902, 47.38297989270074}, + "185m", + "geometry", + []string{"circle3"}, + }, + } + + i := setupGeoJsonShapesIndexForCircleQuery(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapeCircleRelationQuery("contains", + indexReader, test.centrePoint, test.radiusInMeters, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", + n, test.want, got, test.centrePoint) + } + } +} + +func runGeoShapeCircleRelationQuery(relation string, i index.IndexReader, + points []float64, radius string, field string, +) ([]string, error) { + s := geo.NewGeoCircle(points, radius) + return executeSearch(relation, i, s, field) +} + +func setupGeoJsonShapesIndexForCircleQuery(t *testing.T) index.Index { + analysisQueue := index.NewAnalysisQueue(1) + i, err := scorch.NewScorch( + gtreap.Name, + map[string]interface{}{ + "path": "", + "spatialPlugin": "s2", + }, + analysisQueue) + if err != nil { + t.Fatal(err) + } + err = i.Open() + if err != nil { + t.Fatal(err) + } + + polygon1 := [][][][]float64{{{ + {77.67248153686523, 12.957679089615821}, + {77.67956256866455, 12.948101542434257}, + {77.68908977508545, 12.948896200093982}, + {77.68934726715086, 12.955211547173878}, + {77.68016338348389, 12.954291440344619}, + {77.67248153686523, 12.957679089615821}, + }}} + doc := document.NewDocument("polygon1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + polygon1, "polygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + polygon2 := [][][][]float64{{{ + {81.84951782226561, 25.522692102524033}, + {81.8557834625244, 25.521762640415535}, + {81.86264991760254, 25.521762640415535}, + {81.86676979064941, 25.521607729364224}, + {81.89560890197754, 25.542673796271302}, + {81.88977241516113, 25.543293330460937}, + {81.84951782226561, 25.522692102524033}, + }}} + doc = document.NewDocument("polygon2") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + polygon2, "polygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + polygon3 := [][][][]float64{{{ + {8.548071384429932, 47.379216780040124}, + {8.547642230987549, 47.3771680227784}, + {8.545818328857422, 47.37677569847655}, + {8.546290397644043, 47.37417465983494}, + {8.551719188690186, 47.37417465983494}, + {8.553242683410645, 47.37679022905829}, + {8.548071384429932, 47.379216780040124}, + }}} + doc = document.NewDocument("polygon3") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + polygon3, "polygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + point1 := [][][][]float64{{{{81.2439, 26.2244}}}} + doc = document.NewDocument("point1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + point1, "point", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + envelope1 := [][][][]float64{{{ + {79.9969482421875, 23.895882703682627}, + {80.7220458984375, 25.750424835909385}, + }}} + doc = document.NewDocument("envelope1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + envelope1, "envelope", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + envelope2 := [][][][]float64{{{ + {82.10409164428711, 25.54360309635522}, + {82.10537910461424, 25.544609829984058}, + }}} + doc = document.NewDocument("envelope2") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + envelope2, "envelope", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + envelope3 := [][][][]float64{{{ + {8.545668125152588, 47.37942019840244}, + {8.552148342132568, 47.383778974713124}, + }}} + doc = document.NewDocument("envelope3") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + envelope3, "envelope", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("circle1") + addGeoCircleFieldV1V2(doc,"geometry", []uint64{}, + []float64{77.67252445220947, 12.936348678099293}, "900m", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("circle2") + addGeoCircleFieldV1V2(doc,"geometry", []uint64{}, + []float64{82.10289001464844, 25.544919592476727}, "100m", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("circle3") + addGeoCircleFieldV1V2(doc,"geometry", []uint64{}, + []float64{ + 8.53363037109375, + 47.38191927423153, + }, "400m", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + linestring := [][][][]float64{{{ + {77.68715858459473, 12.944755587650944}, + {77.69213676452637, 12.945090185150542}, + }}} + doc = document.NewDocument("linestring1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + linestring, "linestring", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + linestring1 := [][][][]float64{{{ + {77.68913269042969, 12.929614580987227}, + {77.70252227783203, 12.929698235482276}, + }}} + doc = document.NewDocument("linestring2") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + linestring1, "linestring", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + linestring2 := [][][][]float64{{{ + {81.26792907714844, 26.170845301716813}, + {81.30157470703125, 26.18440207077121}, + }}} + doc = document.NewDocument("linestring3") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + linestring2, "linestring", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + multilinestring := [][][][]float64{{{ + {81.86170578002928, 25.430407918899984}, + {81.86273574829102, 25.421958559611397}, + }, { + {81.88230514526367, 25.437616536907512}, + {81.90084457397461, 25.431415601111418}, + }, { + {81.86805725097656, 25.514868905100244}, + {81.86702728271484, 25.502474677473746}, + }}} + doc = document.NewDocument("multilinestring1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + multilinestring, "multilinestring", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + multilinestring1 := [][][][]float64{{ + { + {81.84642791748047, 25.561335859046192}, + {81.84230804443358, 25.550495180470026}, + }, + {{81.87423706054688, 25.55142441992021}, {81.88453674316406, 25.555141305670045}}, + {{81.8642807006836, 25.572175556682115}, {81.87458038330078, 25.567839795359724}}, + }} + doc = document.NewDocument("multilinestring2") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + multilinestring1, "multilinestring", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + multipoint1 := [][][][]float64{{{ + {81.87337875366211, 25.432268248708212}, + {81.87355041503906, 25.416299483230368}, + {81.90118789672852, 25.426067037656946}, + }}} + doc = document.NewDocument("multipoint1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + multipoint1, "multipoint", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + polygonWithHole1 := [][][][]float64{{ + { + {77.59991168975829, 12.972232910164502}, + {77.6039457321167, 12.97582941279006}, + {77.60424613952637, 12.98168407323241}, + {77.59974002838135, 12.985489528568463}, + {77.59321689605713, 12.979300406693417}, + {77.59991168975829, 12.972232910164502}, + }, + { + {77.59682178497314, 12.975787593290978}, + {77.60295867919922, 12.975787593290978}, + {77.60295867919922, 12.98143316204164}, + {77.59682178497314, 12.98143316204164}, + {77.59682178497314, 12.975787593290978}, + }, + }} + + doc = document.NewDocument("polygonWithHole1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + polygonWithHole1, "polygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + return i +} diff --git a/search/searcher/search_geoshape_envelope_test.go b/search/searcher/search_geoshape_envelope_test.go new file mode 100644 index 0000000..522e5be --- /dev/null +++ b/search/searcher/search_geoshape_envelope_test.go @@ -0,0 +1,500 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap" + index "github.com/blevesearch/bleve_index_api" +) + +func TestGeoJsonEnvelopeWithInQuery(t *testing.T) { + tests := []struct { + points [][]float64 + field string + want []string + }{ + // test within query envelope for point1. + { + [][]float64{ + {76.256103515625, 16.76772739719064}, + {76.35772705078125, 16.872890378907783}, + }, + "geometry", + []string{"point1"}, + }, + + // test within query envelope for multipoint1. + { + [][]float64{ + {81.046142578125, 17.156537255486093}, + {81.331787109375, 17.96305758238804}, + }, + "geometry", + []string{"multipoint1"}, + }, + + // test within query envelope for partial points in a multipoint1. + { + [][]float64{ + {81.05987548828125, 17.16178591271515}, + {81.36199951171875, 17.861132899477624}, + }, + "geometry", nil, + }, + + // test within query envelope for polygon2 and point1. + { + [][]float64{ + {76.00341796875, 16.573022719182777}, + {76.717529296875, 17.006888277600524}, + }, + "geometry", + []string{"polygon2", "point1"}, + }, + + // test within query envelope for linestring1. + { + [][]float64{ + {76.84112548828125, 16.86500518090961}, + {77.62115478515625, 17.531439701706244}, + }, + "geometry", + []string{"linestring1"}, + }, + + // test within query envelope for multilinestring1. + { + [][]float64{ + {81.683349609375, 17.104042525557904}, + {81.99234008789062, 17.66495983051931}, + }, + "geometry", + []string{"multilinestring1"}, + }, + + // test within query envelope that is intersecting multilinestring1. + { + [][]float64{ + {81.65725708007812, 17.2601707001208}, + {81.95114135742186, 17.66495983051931}, + }, + "geometry", nil, + }, + + // test within query envelope for envelope1 and circle1. + { + [][]float64{ + {74.75372314453125, 17.36636733709516}, + {75.509033203125, 18.038809662036805}, + }, + "geometry", + []string{"envelope1", "circle1"}, + }, + + // test within query envelope for envelope1. + { + [][]float64{ + {74.783935546875, 17.38209494787749}, + {75.96221923828125, 17.727758609852284}, + }, + "geometry", + []string{"envelope1"}, + }, + } + i := setupGeoJsonShapesIndexForEnvelopeQuery(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapeEnvelopeRelationQuery("within", + indexReader, test.points, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", + n, test.want, got, test.points) + } + } +} + +func TestGeoJsonEnvelopeIntersectsQuery(t *testing.T) { + tests := []struct { + points [][]float64 + field string + want []string + }{ + // test intersecting query envelope for partial points in a multipoint1. + { + [][]float64{ + {81.00769042968749, 17.80622614478282}, + {81.199951171875, 17.983957957423037}, + }, + "geometry", + []string{"multipoint1"}, + }, + + // test intersecting query envelope that is intersecting multilinestring1. + { + [][]float64{ + {81.65725708007812, 17.2601707001208}, + {81.95114135742186, 17.66495983051931}, + }, + "geometry", + []string{"multilinestring1"}, + }, + + // test intersecting query envelope for linestring2. + { + [][]float64{ + {81.9854736328125, 18.27369419984127}, + {82.14752197265625, 18.633232565431218}, + }, + "geometry", + []string{"linestring2"}, + }, + + // test intersecting query envelope for circle2. + { + [][]float64{ + {82.6336669921875, 17.82714499951342}, + {82.66387939453125, 17.861132899477624}, + }, + "geometry", + []string{"circle2"}, + }, + + // test intersecting query envelope for polygon3. + { + [][]float64{ + {82.92343139648438, 17.739530934289657}, + {82.98797607421874, 17.79184300887134}, + }, + "geometry", + []string{"polygon3"}, + }, + } + i := setupGeoJsonShapesIndexForEnvelopeQuery(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapeEnvelopeRelationQuery("intersects", indexReader, test.points, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", n, test.want, got, test.points) + } + } +} + +func TestGeoJsonEnvelopeContainsQuery(t *testing.T) { + tests := []struct { + points [][]float64 + field string + want []string + }{ + // test envelope contained within polygon1. + { + [][]float64{ + {8.548285961151123, 47.376092756617446}, + {8.551225662231445, 47.37764752629426}, + }, + "geometry", + []string{"polygon1"}, + }, + + // test envelope partially contained within polygon1. + { + [][]float64{ + {8.549273014068604, 47.376194471922986}, + {8.551654815673828, 47.37827232736301}, + }, + "geometry", nil, + }, + + // test envelope partially contained within polygon1. + { + [][]float64{ + {8.549273014068604, 47.376194471922986}, + {8.551654815673828, 47.37827232736301}, + }, + "geometry", nil, + }, + + // test envelope fully contained within circle3. + { + [][]float64{ + {8.532772064208984, 47.380379160110856}, + {8.534531593322752, 47.38299442157271}, + }, + "geometry", + []string{"circle3"}, + }, + + // test envelope partially contained within circle3. + { + [][]float64{ + {8.532836437225342, 47.38010309716447}, + {8.538415431976318, 47.383081594720466}, + }, + "geometry", nil, + }, + } + i := setupGeoJsonShapesIndexForEnvelopeQuery(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapeEnvelopeRelationQuery("contains", + indexReader, test.points, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", + n, test.want, got, test.points) + } + } +} + +func runGeoShapeEnvelopeRelationQuery(relation string, i index.IndexReader, + points [][]float64, field string, +) ([]string, error) { + s := geo.NewGeoEnvelope(points) + return executeSearch(relation, i, s, field) +} + +func setupGeoJsonShapesIndexForEnvelopeQuery(t *testing.T) index.Index { + analysisQueue := index.NewAnalysisQueue(1) + i, err := scorch.NewScorch( + gtreap.Name, + map[string]interface{}{ + "path": "", + "spatialPlugin": "s2", + }, + analysisQueue) + if err != nil { + t.Fatal(err) + } + err = i.Open() + if err != nil { + t.Fatal(err) + } + + polygon1 := [][][][]float64{{{ + {8.548071384429932, 47.379216780040124}, + {8.547642230987549, 47.3771680227784}, + {8.545818328857422, 47.37677569847655}, + {8.546290397644043, 47.37417465983494}, + {8.551719188690186, 47.37417465983494}, + {8.553242683410645, 47.37679022905829}, + {8.548071384429932, 47.379216780040124}, + }}} + doc := document.NewDocument("polygon1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + polygon1, "polygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + polygon2 := [][][][]float64{{{ + {76.70379638671874, 16.828203242420393}, + {76.36322021484375, 16.58881695544584}, + {76.70928955078125, 16.720385051694}, + {76.70379638671874, 16.828203242420393}, + }}} + doc = document.NewDocument("polygon2") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + polygon2, "polygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + polygon3 := [][][][]float64{{{ + {82.9522705078125, 17.749994573141873}, + {82.94952392578125, 17.692436998627272}, + {82.87673950195312, 17.64009591883757}, + {82.76412963867188, 17.58643052828743}, + {82.8094482421875, 17.522272941245202}, + {82.99621582031249, 17.64009591883757}, + {82.9522705078125, 17.749994573141873}, + }}} + doc = document.NewDocument("polygon3") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + polygon3, "polygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + envelope1 := [][][][]float64{{{ + {74.89654541015625, 17.403062993328923}, + {74.92401123046875, 17.66495983051931}, + }}} + doc = document.NewDocument("envelope1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + envelope1, "envelope", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("circle1") + addGeoCircleFieldV1V2(doc,"geometry", []uint64{}, + []float64{75.0531005859375, 17.675427818339383}, "12900m", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("circle2") + addGeoCircleFieldV1V2(doc,"geometry", []uint64{}, + []float64{82.69683837890625, 17.902955242676995}, "6000m", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("circle3") + addGeoCircleFieldV1V2(doc,"geometry", []uint64{}, + []float64{ + 8.53363037109375, + 47.38191927423153, + }, "400m", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + point1 := [][][][]float64{{{{76.29730224609375, 16.796653031618053}}}} + doc = document.NewDocument("point1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + point1, "point", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + linestring1 := [][][][]float64{{{ + {76.85211181640624, 17.51048642597462}, + {77.24212646484374, 16.93070509876554}, + }}} + doc = document.NewDocument("linestring1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + linestring1, "linestring", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + linestring2 := [][][][]float64{{{ + {81.89208984375, 18.555136195095105}, + {82.21343994140625, 18.059701055000478}, + }}} + doc = document.NewDocument("linestring2") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + linestring2, "linestring", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + multipoint1 := [][][][]float64{{{ + {81.24938964843749, 17.602139123350838}, + {81.30432128906249, 17.56548361143177}, + {81.29058837890625, 17.180155043474496}, + {81.09283447265625, 17.87681743233167}, + }}} + doc = document.NewDocument("multipoint1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + multipoint1, "multipoint", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + multilinestring := [][][][]float64{{ + { + {81.69708251953125, 17.641404631355755}, + {81.90994262695312, 17.642713334367667}, + }, + {{81.6998291015625, 17.620464090732245}, {81.69708251953125, 17.468572623463153}}, + {{81.70120239257811, 17.458092664041494}, {81.81243896484375, 17.311310073048123}}, + {{81.815185546875, 17.3034434020238}, {81.81243896484375, 17.109292665395643}}, + }} + doc = document.NewDocument("multilinestring1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + multilinestring, "multilinestring", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + multilinestring1 := [][][][]float64{{ + { + {77.6015853881836, 12.990089451715061}, + {77.60476112365723, 12.987747683302153}, + }, + {{77.59875297546387, 12.988751301039581}, {77.59446144104004, 12.98197680263484}}, + {{77.60188579559325, 12.982604078764705}, {77.60557651519775, 12.987329508048184}}, + }} + doc = document.NewDocument("multilinestring2") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + multilinestring1, "multilinestring", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + return i +} diff --git a/search/searcher/search_geoshape_geometrycollection_test.go b/search/searcher/search_geoshape_geometrycollection_test.go new file mode 100644 index 0000000..d829929 --- /dev/null +++ b/search/searcher/search_geoshape_geometrycollection_test.go @@ -0,0 +1,672 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap" + index "github.com/blevesearch/bleve_index_api" +) + +func TestGeoJSONIntersectsQueryAgainstGeometryCollection(t *testing.T) { + tests := []struct { + points [][][][][]float64 + types []string + field string + want []string + }{ + // test intersects geometrycollection query for gc_polygon1_linestring1. + { + [][][][][]float64{ + {{{ + {-120.80017089843749, 36.54053616262899}, + {-120.67932128906249, 36.33725319397006}, + {-120.30578613281251, 36.90597988519294}, + {-120.80017089843749, 36.54053616262899}, + }}}, + {{{{-118.24584960937499, 35.32184842037683}, {-117.8668212890625, 35.06597313798418}}}}, + }, + []string{"polygon", "linestring"}, + "geometry", + []string{"gc_polygon1_linestring1"}, + }, + + // test intersects geometrycollection query for gc_polygon1_linestring1. + { + [][][][][]float64{ + {{ + {{-118.3172607421875, 35.250105158539355}, {-117.50976562499999, 35.37561413174875}}, + {{-118.69628906249999, 34.6241677899049}, {-118.3172607421875, 35.03899204678081}}, + {{-117.94921874999999, 35.146862906756304}, {-117.674560546875, 34.41144164327245}}, + }}, + {{{ + {-117.04284667968749, 35.263561862152095}, + {-116.8505859375, 35.263561862152095}, + {-116.8505859375, 35.33529320309328}, + {-117.04284667968749, 35.33529320309328}, + {-117.04284667968749, 35.263561862152095}, + }}}, + }, + []string{"multilinestring", "polygon"}, + "geometry", + []string{"gc_polygon1_linestring1"}, + }, + + // test intersects geometrycollection query for gc_multipolygon1_multilinestring1. + { + [][][][][]float64{ + {{ + {{-115.8563232421875, 38.53957267203905}, {-115.58166503906251, 38.54816542304656}}, + {{-115.8343505859375, 38.45789034424927}, {-115.81237792968749, 38.19502155795575}}, + }}, + {{{{-116.64905548095702, 37.94920616351679}}}}, + }, + []string{"multilinestring", "point"}, + "geometry", + []string{"gc_multipolygon1_multilinestring1"}, + }, + + // test intersects geometrycollection query for gc_polygon1_linestring1 and gc_multipolygon1_multilinestring1. + { + [][][][][]float64{ + {{{{-116.64905548095702, 37.94920616351679}, {-118.29528808593751, 34.52466147177172}}}}, + {{ + {{-115.8563232421875, 38.53957267203905}, {-115.58166503906251, 38.54816542304656}}, + {{-115.8343505859375, 38.45789034424927}, {-115.81237792968749, 38.19502155795575}}, + }}, + }, + []string{"multipoint", "multilinestring"}, + "geometry", + []string{ + "gc_polygon1_linestring1", + "gc_multipolygon1_multilinestring1", + }, + }, + + // test intersects geometrycollection query for gc_polygon1_linestring1 and gc_multipolygon1_multilinestring1. + { + [][][][][]float64{ + {{{ + {-117.46582031249999, 36.146746777814364}, + {-116.70227050781249, 36.146746777814364}, + {-116.70227050781249, 36.69485094156225}, + {-117.46582031249999, 36.69485094156225}, + {-117.46582031249999, 36.146746777814364}, + }}, {{ + {-115.5267333984375, 38.06106741381201}, + {-115.4937744140625, 37.18220222107978}, + {-114.93896484374999, 37.304644804751106}, + {-115.5267333984375, 38.06106741381201}, + }}}, + {{ + {{-115.8563232421875, 38.53957267203905}, {-115.58166503906251, 38.54816542304656}}, + {{-115.8343505859375, 38.45789034424927}, {-115.81237792968749, 38.19502155795575}}, + }}, + }, + []string{"multipolygon", "multilinestring"}, + "geometry", + []string{"gc_point1_multipoint1"}, + }, + } + i := setupGeoJsonShapesIndexForGeometryCollectionQuery(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapeGeometryCollectionRelationQuery("intersects", + indexReader, test.points, test.types, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", + n, test.want, got, test.points) + } + } +} + +func TestGeoJSONWithInQueryAgainstGeometryCollection(t *testing.T) { + tests := []struct { + points [][][][][]float64 + types []string + field string + want []string + }{ + // test within geometrycollection query for gc_multipoint2_multipolygon2_multiline2. + { + [][][][][]float64{ + {{{{-122.40434646606444, 37.73400071182758}, {-122.39730834960938, 37.73691949864062}}}}, + {{{ + {-122.42511749267578, 37.760808496517235}, + {-122.42314338684082, 37.74248523826606}, + {-122.40082740783691, 37.756669194195815}, + {-122.42511749267578, 37.760808496517235}, + }}}, + { + {{ + {-122.46339797973633, 37.76637243960179}, + {-122.46176719665527, 37.7502901437285}, + {-122.43644714355469, 37.75911208915015}, + {-122.46339797973633, 37.76637243960179}, + }}, + {{ + {-122.43653297424315, 37.714720253587004}, + {-122.40563392639159, 37.714720253587004}, + {-122.40563392639159, 37.72904529863455}, + {-122.43653297424315, 37.72904529863455}, + {-122.43653297424315, 37.714720253587004}, + }}, + }, + }, + []string{"linestring", "polygon", "multipolygon"}, + "geometry", + []string{"gc_multipoint2_multipolygon2_multiline2"}, + }, + + // test within geometrycollection query. + { + [][][][][]float64{ + {{{{-122.40434646606444, 37.73400071182758}, {-122.39730834960938, 37.73691949864062}}}}, + { + {{ + {-122.46339797973633, 37.76637243960179}, + {-122.46176719665527, 37.7502901437285}, + {-122.43644714355469, 37.75911208915015}, + {-122.46339797973633, 37.76637243960179}, + }}, + {{ + {-122.43653297424315, 37.714720253587004}, + {-122.40563392639159, 37.714720253587004}, + {-122.40563392639159, 37.72904529863455}, + {-122.43653297424315, 37.72904529863455}, + {-122.43653297424315, 37.714720253587004}, + }}, + }, + }, + []string{"linestring", "multipolygon"}, + "geometry", nil, + }, + + // test within geometrycollection for gc_multipoint2_multipolygon2_multiline2. + { + [][][][][]float64{ + {{{ + {-122.4491500854492, 37.78170504295941}, + {-122.4862289428711, 37.747371884118664}, + {-122.43078231811525, 37.6949593672454}, + {-122.3799705505371, 37.72945260537779}, + {-122.3928451538086, 37.78007695280165}, + {-122.4491500854492, 37.78170504295941}, + }}}, + }, + []string{"polygon"}, + "geometry", + []string{"gc_multipoint2_multipolygon2_multiline2"}, + }, + + // test within geometrycollection for gc_multipolygon3 + // gc_multipolygon3's multipolygons within the geometrycollection is covered by the + // query's geometric collection of a polygon and a multipolygon. + { + [][][][][]float64{ + {{{ + {86.6162109375, 57.26716357153586}, + {85.1220703125, 8119}, + {84.462890625, 56.27996083172844}, + {86.98974609375, 55.70235509327093}, + {87.802734375, 56.77680831656842}, + {86.6162109375, 57.26716357153586}, + }}}, + { + {{ + {75.1025390625, 54.3549556895541}, + {73.1689453125, 54.29088164657006}, + {72.7294921875, 53.08082737207479}, + {74.091796875, 51.998410382390325}, + {76.79443359375, 53.396432127095984}, + {75.1025390625, 54.3549556895541}, + }}, + {{ + {80.1123046875, 55.57834467218206}, + {78.9697265625, 55.65279803318956}, + {78.5302734375, 54.635697306063854}, + {79.87060546875, 54.18815548107151}, + {80.96923828125, 54.80068486732233}, + {80.1123046875, 55.57834467218206}, + }}, + }, + }, + []string{"polygon", "multipolygon"}, + "geometry", + []string{"gc_multipolygon3"}, + }, + } + + i := setupGeoJsonShapesIndexForGeometryCollectionQuery(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapeGeometryCollectionRelationQuery("within", indexReader, test.points, test.types, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", n, test.want, got, test.points) + } + } +} + +func TestGeoJSONContainsQueryAgainstGeometryCollection(t *testing.T) { + tests := []struct { + points [][][][][]float64 + types []string + field string + want []string + }{ + // test contains for a geometrycollection that comprises of a linestring, + // polygon, multipolygon, point and multipoint for polygon2. + { + [][][][][]float64{ + // linestring + {{{{7.457013130187988, 46.966401589723894}, {7.482891082763671, 46.94554547022893}}}}, + // polygon + {{{ + {7.466454505920409, 46.965054389418476}, + {7.46143341064453, 46.9641171865865}, + {7.466325759887694, 46.96101258493027}, + {7.466454505920409, 46.965054389418476}, + }}}, + // multipolygon + { + {{ + {7.4811744689941415, 46.957966385567474}, + {7.478899955749511, 46.95492001277476}, + {7.484478950500488, 46.95509576976545}, + {7.4811744689941415, 46.957966385567474}, + }}, + {{ + {7.466540336608888, 46.94753769790697}, + {7.464609146118165, 46.946219320241674}, + {7.468342781066894, 46.94592634301753}, + {7.466540336608888, 46.94753769790697}, + }}, + {{ + {7.504348754882812, 47.00425575323296}, + {7.501087188720703, 47.001680295206874}, + {7.507266998291015, 47.00191443288521}, + {7.504348754882812, 47.00425575323296}, + }}, + }, + // point + {{{{7.449932098388673, 46.95817142366062}}}}, + // multipoint + {{{{7.479157447814942, 46.96370715518446}, {7.4532365798950195, 46.96657730900153}}}}, + }, + []string{"linestring", "polygon", "multipolygon", "point", "multipoint"}, + "geometry", + []string{"multipolygon4"}, + }, + + // test contains for a geometrycollection query with one point inside the multipoint lying outside + // polygon2. + { + [][][][][]float64{ + // linestring + {{{{7.457013130187988, 46.966401589723894}, {7.482891082763671, 46.94554547022893}}}}, + // polygon + {{{ + {7.466454505920409, 46.965054389418476}, + {7.46143341064453, 46.9641171865865}, + {7.466325759887694, 46.96101258493027}, + {7.466454505920409, 46.965054389418476}, + }}}, + // multipolygon + { + {{ + {7.4811744689941415, 46.957966385567474}, + {7.478899955749511, 46.95492001277476}, + {7.484478950500488, 46.95509576976545}, + {7.4811744689941415, 46.957966385567474}, + }}, + {{ + {7.466540336608888, 46.94753769790697}, + {7.464609146118165, 46.946219320241674}, + {7.468342781066894, 46.94592634301753}, + {7.466540336608888, 46.94753769790697}, + }}, + }, + // point + {{{{7.449932098388673, 46.95817142366062}}}}, + // multipoint + {{{{7.479157447814942, 46.96370715518446}, {7.475638389587402, 46.965200825877794}}}}, + }, + []string{"linestring", "polygon", "multipolygon", "point", "multipoint"}, + "geometry", + nil, + }, + + // test contains for a geometrycollection query with one point inside the multipoint lying outside + // polygon2. + { + [][][][][]float64{ + // linestring + {{{{7.457013130187988, 46.966401589723894}, {7.482891082763671, 46.94554547022893}}}}, + // polygon + {{{ + {7.466454505920409, 46.965054389418476}, + {7.46143341064453, 46.9641171865865}, + {7.466325759887694, 46.96101258493027}, + {7.466454505920409, 46.965054389418476}, + }}}, + // multipolygon + { + {{ + {7.4811744689941415, 46.957966385567474}, + {7.478899955749511, 46.95492001277476}, + {7.484478950500488, 46.95509576976545}, + {7.4811744689941415, 46.957966385567474}, + }}, + {{ + {7.466540336608888, 46.94753769790697}, + {7.464609146118165, 46.946219320241674}, + {7.468342781066894, 46.94592634301753}, + {7.466540336608888, 46.94753769790697}, + }}, + }, + // point + {{{{7.449932098388673, 46.95817142366062}}}}, + // multipoint + {{{{7.479157447814942, 46.96370715518446}, {7.4532365798950195, 46.96657730900153}}}}, + }, + []string{"linestring", "polygon", "multipolygon", "point", "multipoint"}, + "geometry", + []string{"polygon2", "multipolygon4"}, + }, + } + + i := setupGeoJsonShapesIndexForGeometryCollectionQuery(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapeGeometryCollectionRelationQuery("contains", + indexReader, test.points, test.types, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", + n, test.want, got, test.points) + } + } +} + +func runGeoShapeGeometryCollectionRelationQuery(relation string, i index.IndexReader, + points [][][][][]float64, types []string, field string, +) ([]string, error) { + s, _, err := geo.NewGeometryCollection(points, types) + if err != nil { + return nil, err + } + return executeSearch(relation, i, s, field) +} + +func setupGeoJsonShapesIndexForGeometryCollectionQuery(t *testing.T) index.Index { + analysisQueue := index.NewAnalysisQueue(1) + i, err := scorch.NewScorch( + gtreap.Name, + map[string]interface{}{ + "path": "", + "spatialPlugin": "s2", + }, + analysisQueue) + if err != nil { + t.Fatal(err) + } + err = i.Open() + if err != nil { + t.Fatal(err) + } + + // document gc_polygon1_linestring1 + polygon1 := [][][][]float64{{{ + {-118.15246582031249, 34.876918445772084}, + {-118.46557617187499, 34.773203753940734}, + {-118.3172607421875, 34.50655662164561}, + {-117.91625976562499, 34.4793919710481}, + {-117.76245117187499, 34.76417891445512}, + {-118.15246582031249, 34.876918445772084}, + }}} + + linestring1 := [][][][]float64{{{ + {-120.78918457031251, 36.87522650673951}, + {-118.9215087890625, 34.95349314197422}, + }}} + + coordinates := [][][][][]float64{polygon1, linestring1} + types := []string{"polygon", "linestring"} + + doc := document.NewDocument("gc_polygon1_linestring1") + addGeoCollectionFieldV1V2(doc,"geometry", + []uint64{}, coordinates, types, document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + // document gc_multipolygon1_multilinestring1 + multipolygon1 := [][][][]float64{ + {{ + {-117.24609374999999, 37.67512527892127}, + {-117.61962890624999, 37.26530995561875}, + {-116.597900390625, 37.56199695314352}, + {-117.24609374999999, 37.67512527892127}, + }}, + {{ + {-117.60864257812501, 38.71123253895224}, + {-117.41638183593749, 38.36750215395045}, + {-117.66357421875, 37.93986540897977}, + {-116.6473388671875, 37.94852933714952}, + {-117.1307373046875, 38.363195134453846}, + {-116.75170898437501, 38.7283759182398}, + {-117.60864257812501, 38.71123253895224}, + }}, + } + multilinestring1 := [][][][]float64{{ + {{-118.9215087890625, 38.74123075381228}, {-118.78967285156249, 38.43207668538207}}, + {{-118.57543945312501, 38.8225909761771}, {-118.45458984375, 38.522384090200845}}, + {{-118.94897460937499, 38.788345355085625}, {-118.61938476562499, 38.86965182408357}}, + }} + + coordinates = [][][][][]float64{multipolygon1, multilinestring1} + types = []string{"multipolygon", "multilinestring"} + doc = document.NewDocument("gc_multipolygon1_multilinestring1") + addGeoCollectionFieldV1V2(doc,"geometry", + []uint64{}, coordinates, types, document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + // document gc_point1_multipoint1 + point1 := [][][][]float64{{{{-115.10925292968749, 36.20882309283712}}}} + multipoint1 := [][][][]float64{{{ + {-117.13623046874999, 36.474306755095235}, + {-118.57543945312501, 36.518465989675875}, + {-118.58642578124999, 36.90597988519294}, + {-119.5477294921875, 37.85316995894978}, + }}} + + coordinates = [][][][][]float64{point1, multipoint1} + types = []string{"point", "multipoint"} + + doc = document.NewDocument("gc_point1_multipoint1") + addGeoCollectionFieldV1V2(doc,"geometry", + []uint64{}, coordinates, types, document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + // document gc_multipoint2_multipolygon2_multiline2 + multipoint2 := [][][][]float64{{{ + {-122.4052906036377, 37.75626203719391}, + {-122.42091178894044, 37.74757548736071}, + }}} + multipolygon2 := [][][][]float64{ + {{ + {-122.46168136596681, 37.765151122096945}, + {-122.46168136596681, 37.754972691904946}, + {-122.45103836059569, 37.754972691904946}, + {-122.451810836792, 37.7624370109886}, + {-122.46168136596681, 37.765151122096945}, + }}, + {{ + {-122.41902351379395, 37.726194088705576}, + {-122.43533134460448, 37.71668926284967}, + {-122.40777969360353, 37.71634978222733}, + {-122.41902351379395, 37.726194088705576}, + }}, + } + multilinestring2 := [][][][]float64{{ + {{-122.41284370422362, 37.73155698786267}, {-122.40700721740721, 37.73338978839743}}, + {{-122.40434646606444, 37.73400071182758}, {-122.39730834960938, 37.73691949864062}}, + }} + + coordinates = [][][][][]float64{multipoint2, multipolygon2, multilinestring2} + types = []string{"multipoint", "multipolygon", "multiline"} + + doc = document.NewDocument("gc_multipoint2_multipolygon2_multiline2") + addGeoCollectionFieldV1V2(doc,"geometry", + []uint64{}, coordinates, types, document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + // document gc_multipolygon3 + multipolygon3 := [][][][]float64{ + {{ + {85.60546875, 57.20771009775018}, + {86.396484375, 55.99838095535963}, + {87.03369140625, 56.71656572651468}, + {85.60546875, 57.20771009775018}, + }}, + {{ + {79.56298828125, 55.3915921070334}, + {79.60693359375, 54.43171285946844}, + {80.39794921875, 54.85131525968606}, + {79.56298828125, 55.3915921070334}, + }}, + {{ + {74.35546875, 54.13669645687002}, + {74.1796875, 52.802761415419674}, + {75.87158203125, 53.44880683542759}, + {74.35546875, 54.13669645687002}, + }}, + } + + coordinates = [][][][][]float64{multipolygon3} + types = []string{"multipolygon"} + + doc = document.NewDocument("gc_multipolygon3") + addGeoCollectionFieldV1V2(doc,"geometry", + []uint64{}, coordinates, types, document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + polygon2 := [][][][]float64{{{ + {7.452635765075683, 46.96692874582506}, + {7.449803352355956, 46.95817142366062}, + {7.4573564529418945, 46.95149263607834}, + {7.462162971496582, 46.945955640812095}, + {7.483148574829102, 46.945311085627445}, + {7.487225532531738, 46.957029058564686}, + {7.4793291091918945, 46.96388288331302}, + {7.464480400085448, 46.96903731827891}, + {7.452635765075683, 46.96692874582506}, + }}} + + doc = document.NewDocument("polygon2") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + polygon2, "polygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + multipolygon4 := [][][][]float64{ + {{ + {7.452635765075683, 46.96692874582506}, + {7.449803352355956, 46.95817142366062}, + {7.4573564529418945, 46.95149263607834}, + {7.462162971496582, 46.945955640812095}, + {7.483148574829102, 46.945311085627445}, + {7.487225532531738, 46.957029058564686}, + {7.4793291091918945, 46.96388288331302}, + {7.464480400085448, 46.96903731827891}, + {7.452635765075683, 46.96692874582506}, + }}, + {{ + {7.4478721618652335, 47.00015837528636}, + {7.5110435485839835, 47.00015837528636}, + {7.5110435485839835, 47.00683108710118}, + {7.4478721618652335, 47.00683108710118}, + {7.4478721618652335, 47.00015837528636}, + }}, + } + + doc = document.NewDocument("multipolygon4") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + multipolygon4, "multipolygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + return i +} diff --git a/search/searcher/search_geoshape_linestring_test.go b/search/searcher/search_geoshape_linestring_test.go new file mode 100644 index 0000000..7fbb84c --- /dev/null +++ b/search/searcher/search_geoshape_linestring_test.go @@ -0,0 +1,704 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func TestGeoJsonLinestringIntersectsQuery(t *testing.T) { + tests := []struct { + line [][]float64 + field string + want []string + }{ + // test intersecting linestring query for polygon1. + { + [][]float64{ + {74.85860824584961, 22.407219759334023}, + {74.8663330078125, 22.382936446589863}, + }, + "geometry", + []string{"polygon1"}, + }, + + // test intersecting linestring query for polygon1 and polygon2. + { + [][]float64{ + {74.82461929321289, 22.393729553598526}, + {74.93671417236328, 22.356743809494784}, + }, + "geometry", + []string{"polygon1", "polygon2"}, + }, + + // test intersecting linestring query for envelope1. + { + [][]float64{ + {74.83938217163086, 22.325782524687973}, + {74.8692512512207, 22.311172762889516}, + }, + "geometry", + []string{"envelope1"}, + }, + + // test intersecting linestring query for circle. + { + [][]float64{ + {74.94546890258789, 22.310815439776572}, + {74.93276596069336, 22.303708490145645}, + }, + "geometry", + []string{"circle1"}, + }, + + // test intersecting linestring query for linestring1. + { + [][]float64{ + {74.938645362854, 22.321614134448936}, + {74.94070529937744, 22.320224643365446}, + }, + "geometry", + []string{"linestring1"}, + }, + + // test intersecting linestring query for multilinestring1. + { + [][]float64{ + {74.9241828918457, 22.307996525380194}, + {74.94100570678711, 22.293781977618558}, + }, + "geometry", + []string{"multilinestring1"}, + }, + + // test intersecting linestring query for multipolygon1. + { + [][]float64{ + {36.22072219848633, 50.007132228568786}, + {36.22218132019043, 49.99791917183082}, + }, + "geometry", + []string{"multipolygon1"}, + }, + + // test intersecting linestring query for envelope2, circle2, + // multipolygon1 and gc_polygonInGc_multipolygonInGc. + { + [][]float64{ + {36.19840621948242, 50.03834418692451}, + {36.25720024108887, 50.02136210283289}, + }, + "geometry", + []string{"envelope2", "circle2", "multipolygon1", "gc_polygonInGc_multipolygonInGc"}, + }, + } + i := setupGeoJsonShapesIndexForLinestringQuery(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapeLinestringQueryWithRelation("intersects", + indexReader, test.line, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", + n, test.want, got, test.line) + } + } +} + +func TestGeoJsonLinestringContainsQuery(t *testing.T) { + tests := []struct { + line [][]float64 + field string + want []string + }{ + // test a linestring query for multipolygon1. + { + [][]float64{ + {36.21668815612793, 50.040494087443996}, + {36.226301193237305, 50.03861982057644}, + }, + "geometry", + []string{"multipolygon1"}, + }, + + // test a linestring query with endspoints on two + // different polygons in a multipolygon. + { + [][]float64{ + {36.19746208190918, 50.038564693972646}, + {36.21565818786621, 50.03718650830641}, + }, + "geometry", nil, + }, + + // test a linestring query for envelope2. + { + [][]float64{ + {36.25290870666503, 50.03018471417061}, + {36.23110771179199, 50.01854955486945}, + }, + "geometry", + []string{"envelope2"}, + }, + + // test a linestring query for circle2. + { + [][]float64{ + {36.220550537109375, 50.02930252595981}, + {36.224327087402344, 50.02847545979485}, + }, + "geometry", + []string{"circle2"}, + }, + + // test a linestring query for polygonWithHole2. + { + [][]float64{ + {36.27367973327637, 49.89883638369706}, + {36.27445220947265, 49.89596137883285}, + }, + "geometry", + []string{"polygonWithHole2"}, + }, + + // test a linestring query within the hole of polygonWithHole2. + {[][]float64{ + {36.261234283447266, 49.89540847364305}, + {36.26243591308594, 49.89087441212101}, + }, "geometry", nil}, + } + i := setupGeoJsonShapesIndexForLinestringQuery(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapeLinestringQueryWithRelation("contains", + indexReader, test.line, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", + n, test.want, got, test.line) + } + } +} + +func TestGeoJsonMultiLinestringContainsQuery(t *testing.T) { + tests := []struct { + line [][][]float64 + field string + want []string + }{ + // test a multilinestring query for multipolygon1. + { + [][][]float64{ + { + {36.21668815612793, 50.040494087443996}, + {36.226301193237305, 50.03861982057644}, + }, + { + {36.226816177368164, 49.999463999158}, + {36.234025955200195, 50.00271900853649}, + }, + }, + "geometry", + []string{"multipolygon1"}, + }, + + // test a multilinestring query that is covered by the geometryCollection. + { + [][][]float64{{ + {36.28664016723633, 49.96574238290487}, + {36.30251884460449, 49.96369956194569}, + }, { + {36.19179725646973, 50.03983258984584}, + {36.19420051574707, 50.03801342445342}, + }}, + "geometry", + []string{"gc_polygonInGc_multipolygonInGc"}, + }, + + // test a multilinestring query for envelope2. + { + [][][]float64{ + { + {36.23213768005371, 50.02913711386621}, + {36.25187873840332, 50.02902683882067}, + }, + { + {36.231794357299805, 50.018935600613254}, + {36.2314510345459, 50.025883893582055}, + }, + }, + "geometry", + []string{"envelope2"}, + }, + + // test a multilinestring query with one linestring outside of envelope2. + { + [][][]float64{ + { + {36.23213768005371, 50.02913711386621}, + {36.25187873840332, 50.02902683882067}, + }, + {{36.231794357299805, 50.018935600613254}, {36.2314510345459, 50.025883893582055}}, + {{36.25659942626953, 50.024284772330844}, {36.24406814575195, 50.01518531066489}}, + }, + "geometry", nil, + }, + + // test a multilinestring query with one linestring + // inside the whole of a polygonWithHole2. + { + [][][]float64{ + { + {36.27367973327637, 49.89883638369706}, + {36.27445220947265, 49.89596137883285}, + }, + {{36.261234283447266, 49.89540847364305}, {36.26243591308594, 49.89087441212101}}, + }, + "geometry", nil, + }, + + // test a multilinestring query for polygonWithHole2. + { + [][][]float64{ + { + {36.27367973327637, 49.89883638369706}, + {36.27445220947265, 49.89596137883285}, + }, + {{36.279258728027344, 49.894302644257856}, {36.28166198730469, 49.887335336408235}}, + }, + "geometry", + []string{"polygonWithHole2"}, + }, + + // test a multilinestring query for polygonWithHole2 with last line cross the hole. + { + [][][]float64{ + { + {36.27367973327637, 49.89883638369706}, + {36.27445220947265, 49.89596137883285}, + }, + {{36.279258728027344, 49.894302644257856}, {36.28166198730469, 49.887335336408235}}, + {{36.254024505615234, 49.89839408640621}, {36.27016067504883, 49.90038439228633}}, + }, + "geometry", nil, + }, + } + i := setupGeoJsonShapesIndexForLinestringQuery(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapeMultiLinestringQueryWithRelation("contains", + indexReader, test.line, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", + n, test.want, got, test.line) + } + } +} + +func runGeoShapeMultiLinestringQueryWithRelation(relation string, i index.IndexReader, + points [][][]float64, field string, +) ([]string, error) { + s := geo.NewGeoJsonMultilinestring(points) + return executeSearch(relation, i, s, field) +} + +func runGeoShapeLinestringQueryWithRelation(relation string, i index.IndexReader, + points [][]float64, field string, +) ([]string, error) { + s := geo.NewGeoJsonLinestring(points) + return executeSearch(relation, i, s, field) +} + +// executeSearch runs the v1 geoshape searcher and, so that the same corpus +// also exercises the v2 implementation, requires the geoshape_v2 searcher to +// agree on the result set (see assertGeoShapeV2Agrees). It returns the v1 +// result in v1 order so existing order-sensitive assertions keep working. +func executeSearch(relation string, i index.IndexReader, + s index.GeoJSON, field string, +) ([]string, error) { + rv, err := executeGeoShapeV1Search(relation, i, s, field) + if err != nil { + return nil, err + } + if err := assertGeoShapeV2Agrees(relation, i, s, field, rv); err != nil { + return nil, err + } + return rv, nil +} + +func executeGeoShapeV1Search(relation string, i index.IndexReader, + s index.GeoJSON, field string, +) ([]string, error) { + var rv []string + gbs, err := NewGeoShapeSearcher(context.TODO(), i, s, relation, field, 1.0, search.SearcherOptions{}) + if err != nil { + return nil, err + } + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(gbs.DocumentMatchPoolSize(), 0), + } + docMatch, err := gbs.Next(ctx) + for docMatch != nil && err == nil { + docID, _ := i.ExternalID(docMatch.IndexInternalID) + rv = append(rv, docID) + docMatch, err = gbs.Next(ctx) + } + if err != nil { + return nil, err + } + return rv, nil +} + +func setupGeoJsonShapesIndexForLinestringQuery(t *testing.T) index.Index { + analysisQueue := index.NewAnalysisQueue(1) + i, err := scorch.NewScorch( + gtreap.Name, + map[string]interface{}{ + "path": "", + "spatialPlugin": "s2", + }, + analysisQueue) + if err != nil { + t.Fatal(err) + } + err = i.Open() + if err != nil { + t.Fatal(err) + } + + polygon1 := [][][][]float64{{{ + {74.84642028808594, 22.402776071459712}, + {74.83234405517578, 22.39039647758608}, + {74.86719131469727, 22.38801566009795}, + {74.85139846801758, 22.39103135536648}, + {74.86461639404297, 22.394840561182853}, + {74.8495101928711, 22.397697397065034}, + {74.86186981201172, 22.401982540816856}, + {74.84642028808594, 22.402776071459712}, + }}} + doc := document.NewDocument("polygon1") + addGeoShapeFieldV1V2(doc,"geometry", + []uint64{}, polygon1, "polygon", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + polygon2 := [][][][]float64{{{ + {74.93431091308592, 22.376428433285266}, + {74.92898941040039, 22.39103135536648}, + {74.9241828918457, 22.37722210974017}, + {74.90821838378906, 22.37388863821397}, + {74.92504119873047, 22.369920115637292}, + {74.92864608764648, 22.355632497760894}, + {74.93207931518555, 22.370396344320053}, + {74.94855880737305, 22.3743648533201}, + {74.93431091308592, 22.376428433285266}, + }}} + doc = document.NewDocument("polygon2") + addGeoShapeFieldV1V2(doc,"geometry", + []uint64{}, polygon2, "polygon", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + envelope1 := [][][][]float64{{{ + {74.86736297607422, 22.307361269208684}, + {74.87028121948242, 22.345471522338478}, + }}} + doc = document.NewDocument("envelope1") + addGeoShapeFieldV1V2(doc,"geometry", + []uint64{}, envelope1, "envelope", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + envelope2 := [][][][]float64{{{ + {36.23007774353027, 50.01810835593541}, + {36.25333786010742, 50.03068093791795}, + }}} + doc = document.NewDocument("envelope2") + addGeoShapeFieldV1V2(doc,"geometry", + []uint64{}, envelope2, "envelope", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("circle1") + addGeoCircleFieldV1V2(doc,"geometry", + []uint64{}, []float64{74.93671417236328, 22.308314152382284}, "300m", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("circle2") + addGeoCircleFieldV1V2(doc,"geometry", + []uint64{}, []float64{36.22243881225586, 50.02941280037234}, "600m", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + linestring := [][][][]float64{{{ + {74.92697238922119, 22.320343743143248}, + {74.94036197662354, 22.32054224254707}, + }}} + doc = document.NewDocument("linestring1") + addGeoShapeFieldV1V2(doc,"geometry", + []uint64{}, linestring, "linestring", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + linestring1 := [][][][]float64{{{ + {77.60188579559325, 12.982604078764705}, + {77.60557651519775, 12.987329508048184}, + }}} + doc = document.NewDocument("linestring2") + addGeoShapeFieldV1V2(doc,"geometry", + []uint64{}, linestring1, "linestring", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + multilinestring := [][][][]float64{{ + { + {74.92203712463379, 22.3113315728684}, + {74.92323875427246, 22.307798008137024}, + }, + {{74.92405414581299, 22.307559787072712}, {74.92735862731934, 22.310021385140573}}, + {{74.9223804473877, 22.311688894660474}, {74.92534160614014, 22.30930673210729}}, + }} + doc = document.NewDocument("multilinestring1") + addGeoShapeFieldV1V2(doc,"geometry", + []uint64{}, multilinestring, "multilinestring", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + multilinestring1 := [][][][]float64{{ + { + {77.6015853881836, 12.990089451715061}, + {77.60476112365723, 12.987747683302153}, + }, + {{77.59875297546387, 12.988751301039581}, {77.59446144104004, 12.98197680263484}}, + {{77.60188579559325, 12.982604078764705}, {77.60557651519775, 12.987329508048184}}, + }} + doc = document.NewDocument("multilinestring2") + addGeoShapeFieldV1V2(doc,"geometry", + []uint64{}, multilinestring1, "multilinestring", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + multipoint1 := [][][][]float64{{{ + {77.56618022918701, 12.958180959662695}, + {77.56407737731932, 12.951614746607163}, + {77.56922721862793, 12.956173473406446}, + }}} + doc = document.NewDocument("multipoint1") + addGeoShapeFieldV1V2(doc,"geometry", + []uint64{}, multipoint1, "multipoint", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + polygonWithHole1 := [][][][]float64{{ + { + {77.59991168975829, 12.972232910164502}, + {77.6039457321167, 12.97582941279006}, + {77.60424613952637, 12.98168407323241}, + {77.59974002838135, 12.985489528568463}, + {77.59321689605713, 12.979300406693417}, + {77.59991168975829, 12.972232910164502}, + }, + { + {77.59682178497314, 12.975787593290978}, + {77.60295867919922, 12.975787593290978}, + {77.60295867919922, 12.98143316204164}, + {77.59682178497314, 12.98143316204164}, + {77.59682178497314, 12.975787593290978}, + }, + }} + + doc = document.NewDocument("polygonWithHole1") + addGeoShapeFieldV1V2(doc,"geometry", + []uint64{}, polygonWithHole1, "polygon", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + polygonWithHole2 := [][][][]float64{{ + { + {36.261234283447266, 49.90712870720605}, + {36.2479305267334, 49.89480027061714}, + {36.254539489746094, 49.883408870659736}, + {36.280717849731445, 49.883408870659736}, + {36.28741264343262, 49.890432041848264}, + {36.27788543701172, 49.90276159448742}, + {36.261234283447266, 49.90712870720605}, + }, + + { + {36.264581680297844, 49.905249238801304}, + {36.25368118286133, 49.89673543545543}, + {36.253509521484375, 49.88578690918283}, + {36.270332336425774, 49.886174020645804}, + {36.27127647399902, 49.89579550794111}, + {36.264581680297844, 49.905249238801304}, + }, + }} + + doc = document.NewDocument("polygonWithHole2") + addGeoShapeFieldV1V2(doc,"geometry", + []uint64{}, polygonWithHole2, "polygon", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + multipolygon1 := [][][][]float64{{{ + {36.1875057220459, 50.04363607656457}, + {36.192398071289055, 50.034871067327856}, + {36.20218276977539, 50.03955696315653}, + {36.1875057220459, 50.04363607656457}, + }}, // polygon1 + {{ + {36.2123966217041, 50.03795829715335}, + {36.218318939208984, 50.0333273779768}, + {36.226558685302734, 50.03867494711694}, + {36.217031478881836, 50.04286437899031}, + {36.2123966217041, 50.03795829715335}, + }}, // polygon2 + {{ + {36.221065521240234, 50.00365685169585}, + {36.226301193237305, 49.998029518286025}, + {36.23342514038086, 49.9995743420677}, + {36.23531341552734, 50.002994846659156}, + {36.231021881103516, 50.00630478067617}, + {36.22810363769531, 50.00663576154257}, + {36.226043701171875, 50.004815338573046}, + {36.221065521240234, 50.00365685169585}, + }}} + doc = document.NewDocument("multipolygon1") + addGeoShapeFieldV1V2(doc,"geometry", + []uint64{}, multipolygon1, "multipolygon", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + polygonInGc := [][][][]float64{{{ + {36.1875057220459, 50.04363607656457}, + {36.192398071289055, 50.034871067327856}, + {36.20218276977539, 50.03955696315653}, + {36.1875057220459, 50.04363607656457}, + }}} + multipolygonInGc := [][][][]float64{{{ + {36.29015922546387, 49.980150089789376}, + {36.28337860107422, 49.961656654293485}, + {36.307411193847656, 49.96033147865059}, + {36.29015922546387, 49.980150089789376}, + }}, // polygon1 + {{ + {36.16106986999512, 50.00387751801547}, + {36.161842346191406, 49.9908012905034}, + {36.17900848388672, 49.99841572888488}, + {36.16106986999512, 50.00387751801547}, + }}} + coordinates := [][][][][]float64{polygonInGc, multipolygonInGc} + types := []string{"polygon", "multipolygon"} + doc = document.NewDocument("gc_polygonInGc_multipolygonInGc") + addGeoCollectionFieldV1V2(doc,"geometry", + []uint64{}, coordinates, types, + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + return i +} diff --git a/search/searcher/search_geoshape_points_test.go b/search/searcher/search_geoshape_points_test.go new file mode 100644 index 0000000..a56ef6d --- /dev/null +++ b/search/searcher/search_geoshape_points_test.go @@ -0,0 +1,580 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap" + index "github.com/blevesearch/bleve_index_api" +) + +func TestGeoJsonPointContainsQuery(t *testing.T) { + tests := []struct { + point []float64 + field string + want []string + }{ + // test points inside the polygon1. + { + []float64{77.58334636688232, 12.948268838994263}, + "geometry", + []string{"polygon1"}, + }, + + // test points inside the circle1. + { + []float64{77.58553504943848, 12.954040501528555}, + "geometry", + []string{"circle1"}, + }, + + // test points inside the polygon1 and the circle. + { + []float64{77.59293794631958, 12.948896200093982}, + "geometry", + []string{"polygon1", "circle1"}, + }, + + // test points outside the polygon1 and the circle1. + { + []float64{77.5614595413208, 12.953287683563568}, + "geometry", nil, + }, + + // test point within the envelope1. + { + []float64{81.28166198730469, 26.34203746601541}, + "geometry", + []string{"envelope1"}, + }, + + // test point on the linestring vertex. + { + []float64{77.57776737213135, 12.952074805390097}, + "geometry", + []string{"linestring1"}, + }, + + // test point on the multilinestring vertex. + { + []float64{77.5779390335083, 12.945006535817749}, + "geometry", + []string{"multilinestring1"}, + }, + + // test point on the multipoint vertex. + { + []float64{77.56407737731932, 12.951614746607163}, + "geometry", + []string{"multipoint1"}, + }, + + // test point within the polygonWithHole1. + { + []float64{77.60334491729736, 12.979844051951334}, + "geometry", + []string{"polygonWithHole1"}, + }, + + // test point within the hole of the polygonWithHole1. + { + []float64{77.60244369506836, 12.976247607394027}, + "geometry", nil, + }, + } + i := setupGeoJsonShapesIndex(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapePointRelationQuery("contains", + false, indexReader, [][]float64{test.point}, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", + n, test.want, got, test.point) + } + } +} + +func TestGeoJsonMultiPointWithInQuery(t *testing.T) { + tests := []struct { + multipoint [][]float64 + field string + want []string + }{ + // test multipoint inside the polygon1. + { + [][]float64{ + {77.58334636688232, 12.948268838994263}, + {77.58467674255371, 12.944295515355652}, + }, + "geometry", + []string{"polygon1"}, + }, + + // test multipoint inside the circle1. + { + [][]float64{ + {77.58553504943848, 12.954040501528555}, + {77.58643627166747, 12.956089827794571}, + }, + "geometry", + []string{"circle1"}, + }, + + // test multipoint inside the envelope1. + { + [][]float64{ + {81.28166198730469, 26.34203746601541}, + {80.94314575195312, 26.346960121309415}, + }, + "geometry", + []string{"envelope1"}, + }, + + // test multipoint inside the polygon1 and the circle. + { + [][]float64{ + {77.59293794631958, 12.948896200093982}, + {77.58532047271729, 12.953789562459688}, + }, + "geometry", + []string{"polygon1", "circle1"}, + }, + + // test multipoint (only 1 point outside) outside. + {[][]float64{ + {77.58334636688232, 12.948268838994263}, + {77.58643627166747, 12.956089827794571}, + {77.5615, 12.9533}, + }, "geometry", nil}, + + // test multipoint on the linestring vertex. + { + [][]float64{ + {77.5841188430786, 12.957093573282744}, + {77.57776737213135, 12.952074805390097}, + }, + "geometry", + []string{"linestring1"}, + }, + + // test multipoint outside the linestring vertex. + { + [][]float64{ + {77.5841188430786, 12.957093573282744}, + {77.57776737213135, 12.952074805390097}, + {77.58334636688232, 12.948268838994263}, + }, + "geometry", nil, + }, + + // test multipoint on the multilinestring vertex. + { + [][]float64{ + {77.5779390335083, 12.94471376293191}, + {77.57218837738037, 12.948268838994263}, + }, + "geometry", + []string{"multilinestring1"}, + }, + + // test multipoint outside the multilinestring vertex. + { + [][]float64{ + {77.5779390335083, 12.94471376293191}, + {77.57218837738037, 12.948268838994263}, + {77.58532047271729, 12.953789562459688}, + }, + "geometry", nil, + }, + + // test multipoint with one inside the hole within the polygonWithHole1. + { + [][]float64{ + {77.60334491729736, 12.979844051951334}, + {77.60244369506836, 12.976247607394027}, + }, + "geometry", nil, + }, + + // test multipoint with all inside the hole within the polygonWithHole1. + { + [][]float64{ + {77.59656429290771, 12.981767710239714}, + {77.59888172149658, 12.979969508380469}, + }, + "geometry", nil, + }, + + // test multipoint with all inside the polygonWithHole1. + { + [][]float64{ + {77.60334491729736, 12.979844051951334}, + {77.59656429290771, 12.981767710239714}, + {77.59802341461182, 12.9751602999608}, + }, + "geometry", + []string{"polygonWithHole1"}, + }, + } + i := setupGeoJsonShapesIndex(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapePointRelationQuery("contains", + true, indexReader, test.multipoint, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", + n, test.want, got, test.multipoint) + } + } +} + +func TestGeoJsonMultiPointIntersectsQuery(t *testing.T) { + tests := []struct { + multipoint [][]float64 + field string + want []string + }{ + // test multipoint inside the polygon1. + { + [][]float64{ + {77.58334636688232, 12.948268838994263}, + {77.58467674255371, 12.944295515355652}, + }, + "geometry", + []string{"polygon1"}, + }, + + // test multipoint inside the circle1. + { + [][]float64{ + {77.58553504943848, 12.954040501528555}, + {77.58643627166747, 12.956089827794571}, + }, + "geometry", + []string{"circle1"}, + }, + + // test multipoint inside the envelope1. (1 point outside) + { + [][]float64{ + {81.28166198730469, 26.34203746601541}, + {80.94314575195312, 26.346960121309415}, + {81.12716674804688, 26.353728430338332}, + }, + "geometry", + []string{"envelope1"}, + }, + + // test multipoint inside the polygon1 and the circle. + { + [][]float64{ + {77.59293794631958, 12.948896200093982}, + {77.58532047271729, 12.953789562459688}, + }, + "geometry", + []string{"polygon1", "circle1"}, + }, + + // test multipoint (only 1 point outside) intersects. + { + [][]float64{ + {77.58334636688232, 12.948268838994263}, + {77.58643627166747, 12.956089827794571}, + {77.5615, 12.9533}, + }, + "geometry", + []string{"polygon1", "circle1"}, + }, + + // test multipoint on the linestring vertex. + { + [][]float64{ + {77.5841188430786, 12.957093573282744}, + {77.57776737213135, 12.952074805390097}, + }, + "geometry", + []string{"linestring1"}, + }, + + // test multipoint outside the linestring vertex. + { + [][]float64{ + {77.5841188430786, 12.957093573282744}, + {77.57776737213135, 12.952074805390097}, + {77.58334636688232, 12.948268838994263}, + }, + "geometry", + []string{"polygon1", "linestring1"}, + }, + + // test multipoint on the multilinestring vertex. + { + [][]float64{ + {77.5779390335083, 12.94471376293191}, + {77.57218837738037, 12.948268838994263}, + }, + "geometry", + []string{"multilinestring1"}, + }, + + // test multipoint outside the multilinestring vertex. + { + [][]float64{ + {77.5779390335083, 12.94471376293191}, + {77.57218837738037, 12.948268838994263}, + {77.58532047271729, 12.953789562459688}, + }, + "geometry", + []string{"polygon1", "circle1", "multilinestring1"}, + }, + + // test multipoint with one inside the hole within the polygonWithHole1. + { + [][]float64{ + {77.60334491729736, 12.979844051951334}, + {77.60244369506836, 12.976247607394027}, + }, + "geometry", + []string{"polygonWithHole1"}, + }, + + // test multipoint with all inside the hole within the polygonWithHole1. + { + [][]float64{ + {77.60244369506836, 12.976247607394027}, + {77.59888172149658, 12.979969508380469}, + }, + "geometry", nil, + }, + + // test multipoint with all inside the polygonWithHole1. + { + [][]float64{ + {77.60334491729736, 12.979844051951334}, + {77.59656429290771, 12.981767710239714}, + {77.59802341461182, 12.9751602999608}, + }, + "geometry", + []string{"polygonWithHole1"}, + }, + } + i := setupGeoJsonShapesIndex(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapePointRelationQuery("intersects", + true, indexReader, test.multipoint, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", + n, test.want, got, test.multipoint) + } + } +} + +func runGeoShapePointRelationQuery(relation string, multi bool, + i index.IndexReader, points [][]float64, field string, +) ([]string, error) { + var s index.GeoJSON + if multi { + s = geo.NewGeoJsonMultiPoint(points) + } else { + s = geo.NewGeoJsonPoint(points[0]) + } + return executeSearch(relation, i, s, field) +} + +type Fatalfable interface { + Fatalf(format string, args ...interface{}) +} + +func setupGeoJsonShapesIndex(t *testing.T) index.Index { + analysisQueue := index.NewAnalysisQueue(1) + i, err := scorch.NewScorch( + gtreap.Name, + map[string]interface{}{ + "path": "", + "spatialPlugin": "s2", + }, + analysisQueue) + if err != nil { + t.Fatal(err) + } + err = i.Open() + if err != nil { + t.Fatal(err) + } + + polygon1 := [][][][]float64{{{ + {77.5853419303894, 12.953977766785052}, + {77.58405447006226, 12.95393594361393}, + {77.5819730758667, 12.9495026476557}, + {77.58068561553955, 12.94883346405509}, + {77.58019208908081, 12.948331575175299}, + {77.57991313934326, 12.943814529775414}, + {77.58497714996338, 12.94394000436408}, + {77.58517026901245, 12.9446301134728}, + {77.58572816848755, 12.945508431393435}, + {77.58785247802734, 12.946365833997325}, + {77.58967638015747, 12.946428570657417}, + {77.59070634841918, 12.947474179333993}, + {77.59317398071289, 12.948875288082773}, + {77.59167194366454, 12.949962710338657}, + {77.59077072143555, 12.950276388953625}, + {77.59098529815674, 12.951196510612728}, + {77.58729457855225, 12.952472128200755}, + {77.5853419303894, 12.953977766785052}, + }}} + doc := document.NewDocument("polygon1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + polygon1, "polygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + // not working envelope + envelope1 := [][][][]float64{{{ + {80.93696594238281, 26.33957605983274}, + {81.28440856933594, 26.351267272877074}, + }}} + doc = document.NewDocument("envelope1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + envelope1, "envelope", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("circle1") + addGeoCircleFieldV1V2(doc,"geometry", []uint64{}, + []float64{77.59137153625487, 12.952660333521468}, "900m", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + linestring := [][][][]float64{{{ + {77.5841188430786, 12.957093573282744}, + {77.57776737213135, 12.952074805390097}, + }}} + doc = document.NewDocument("linestring1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + linestring, "linestring", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + multilinestring := [][][][]float64{{{ + {77.57227420806883, 12.948687079902895}, + {77.57600784301758, 12.954165970968194}, + {77.5779390335083, 12.94471376293191}, + {77.57218837738037, 12.948268838994263}, + {77.57781028747559, 12.951740217268595}, + {77.5779390335083, 12.945006535817749}, + }}} + doc = document.NewDocument("multilinestring1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + multilinestring, "multilinestring", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + multipoint1 := [][][][]float64{{{ + {77.56618022918701, 12.958180959662695}, + {77.56407737731932, 12.951614746607163}, + {77.56922721862793, 12.956173473406446}, + }}} + doc = document.NewDocument("multipoint1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + multipoint1, "multipoint", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + polygonWithHole1 := [][][][]float64{{ + { + {77.59991168975829, 12.972232910164502}, + {77.6039457321167, 12.97582941279006}, + {77.60424613952637, 12.98168407323241}, + {77.59974002838135, 12.985489528568463}, + {77.59321689605713, 12.979300406693417}, + {77.59991168975829, 12.972232910164502}, + }, + { + {77.59682178497314, 12.975787593290978}, + {77.60295867919922, 12.975787593290978}, + {77.60295867919922, 12.98143316204164}, + {77.59682178497314, 12.98143316204164}, + {77.59682178497314, 12.975787593290978}, + }, + }} + + doc = document.NewDocument("polygonWithHole1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + polygonWithHole1, "polygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + return i +} diff --git a/search/searcher/search_geoshape_polygon_test.go b/search/searcher/search_geoshape_polygon_test.go new file mode 100644 index 0000000..4ba5ece --- /dev/null +++ b/search/searcher/search_geoshape_polygon_test.go @@ -0,0 +1,1184 @@ +// Copyright (c) 2022 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap" + index "github.com/blevesearch/bleve_index_api" +) + +func TestGeoJsonPolygonIntersectsQuery(t *testing.T) { + tests := []struct { + polygon [][][]float64 + field string + want []string + }{ + // test intersecting query polygon for polygon1. + {[][][]float64{{ + {77.57926940917969, 12.945257483731918}, + {77.57875442504883, 12.942036966318216}, + {77.58278846740721, 12.9424970427816}, + {77.57926940917969, 12.945257483731918}, + }}, "geometry", []string{"polygon1"}}, + + // test intersecting query polygon for polygon1, polygon2, circle1. + { + [][][]float64{{ + {77.59562015533446, 12.94099133483504}, + {77.59665012359619, 12.949356263896634}, + {77.59313106536865, 12.951321981484776}, + {77.59085655212402, 12.948477959536318}, + {77.59562015533446, 12.94099133483504}, + }}, + "geometry", + []string{"polygon1", "polygon2", "circle1"}, + }, + + // test intersecting query polygon for polygon1, polygon2 and polygon3. + {[][][]float64{{ + {77.5929594039917, 12.939151012774925}, + {77.58321762084961, 12.94546660680072}, + {77.59737968444824, 12.931998723107322}, + {77.60111331939697, 12.955169724209911}, + {77.59592056274414, 12.936265025833965}, + {77.5929594039917, 12.939151012774925}, + }}, "geometry", []string{"polygon1", "polygon2", "polygon3"}}, + + // test intersecting query polygon for polygon2 and the circle1. + { + [][][]float64{{ + {77.59012699127197, 12.959853852513307}, + {77.59836673736572, 12.959853852513307}, + {77.59836673736572, 12.965541604118611}, + {77.59012699127197, 12.965541604118611}, + {77.59012699127197, 12.959853852513307}, + }}, + "geometry", + []string{"polygon2", "circle1"}, + }, + + // test intersecting query polygon for linestring2 and multilinestring2. + { + [][][]float64{{ + {77.59669303894043, 12.989504011681609}, + {77.60699272155762, 12.983231353311314}, + {77.60115623474121, 12.993183897537897}, + {77.59669303894043, 12.989504011681609}, + }}, + "geometry", + []string{"linestring2", "multilinestring2"}, + }, + + // test intersecting query polygon for multilinestring2. + { + [][][]float64{{ + {77.60124206542969, 12.987162237749484}, + {77.60330200195312, 12.992849364713313}, + {77.59514808654785, 12.989671280403403}, + {77.60124206542969, 12.987162237749484}, + }}, + "geometry", + []string{"multilinestring2"}, + }, + + // test intersecting query polygon for multipoint1. + { + [][][]float64{{ + {77.56648063659668, 12.956382587313202}, + {77.56819725036621, 12.949523559614263}, + {77.5718879699707, 12.958222782120954}, + {77.56648063659668, 12.956382587313202}, + }}, + "geometry", + []string{"multipoint1"}, + }, + + // test intersecting query polygon for envelope1. + {[][][]float64{{ + {36.19986534118652, 50.00034673534484}, + {36.19351387023926, 50.00464984215712}, + {36.178321838378906, 49.991573824716205}, + {36.19986534118652, 50.00034673534484}, + }}, "geometry", []string{"envelope1"}}, + + // test intersecting query polygon for envelope1. + {[][][]float64{{ + {36.170082092285156, 49.99229116680205}, + {36.14982604980469, 49.99002874388075}, + {36.227073669433594, 49.98754547425633}, + {36.170082092285156, 49.99229116680205}, + }}, "geometry", []string{"envelope1"}}, + } + i := setupGeoJsonShapesIndexForPolygonQuery(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapePolygonQueryWithRelation("intersects", + indexReader, test.polygon, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", n, + test.want, got, test.polygon) + } + } +} + +func TestGeoJsonPolygonContainsQuery(t *testing.T) { + tests := []struct { + polygon [][][]float64 + field string + want []string + }{ + // test containment query polygon for polygon1. + { + [][][]float64{{ + {77.5843334197998, 12.952702156906767}, + {77.58510589599608, 12.952702156906767}, + {77.58510589599608, 12.953622269606669}, + {77.5843334197998, 12.953622269606669}, + {77.5843334197998, 12.952702156906767}, + }}, + "geometry", + []string{"polygon1"}, + }, + + // test containment query polygon for circle1. + { + [][][]float64{{ + {77.59025573730469, 12.953810474058429}, + {77.59145736694336, 12.953810474058429}, + {77.59145736694336, 12.954918786278716}, + {77.59025573730469, 12.954918786278716}, + {77.59025573730469, 12.953810474058429}, + }}, + "geometry", + []string{"circle1"}, + }, + + // test containment query polygon for polygon2, polygon3. + { + [][][]float64{{ + {77.60235786437988, 12.956884459972992}, + {77.60124206542969, 12.956800814599926}, + {77.6008129119873, 12.955713422193524}, + {77.60244369506836, 12.955211547173878}, + {77.60313034057617, 12.955880713641998}, + {77.60235786437988, 12.956884459972992}, + }}, + "geometry", + []string{"polygon2", "polygon3"}, + }, + + // test containment query polygon which resides within a hole in polygonWithHole1. + { + [][][]float64{{ + {77.60012626647949, 12.97963495776207}, + {77.5978946685791, 12.978213112610835}, + {77.60089874267577, 12.977962197916442}, + {77.60012626647949, 12.97963495776207}, + }}, + "geometry", nil, + }, + + // test containment query polygon which resides within polygonWithHole1. + { + [][][]float64{{ + {77.59978294372559, 12.984067716910454}, + {77.59780883789062, 12.982227713276774}, + {77.60089874267577, 12.982227713276774}, + {77.59978294372559, 12.984067716910454}, + }}, + "geometry", + []string{"polygonWithHole1"}, + }, + + // test with query polygon for polygon4 with a single vertex lying outside. + { + [][][]float64{{ + {-121.48138761520384, 38.50964107572585}, + {-121.48226737976073, 38.509238097766875}, + {-121.48115158081055, 38.50781086602439}, + {-121.48014307022095, 38.50806273250507}, + {-121.48138761520384, 38.50964107572585}, + }}, + "geometry", nil, + }, + + // test with query polygon for polygon4. + { + [][][]float64{{ + {-121.48381233215332, 38.507974579337045}, + {-121.48361384868622, 38.507869634948676}, + {-121.48361921310425, 38.50765135013098}, + {-121.48343682289122, 38.50797038156446}, + {-121.48381233215332, 38.507974579337045}, + }}, + "geometry", + []string{"polygon4"}, + }, + + // test with query polygon for multipolygon1. + {[][][]float64{{ + {-121.47578716278075, 38.51617236229197}, + {-121.47578716278075, 38.51566868518406}, + {-121.47546529769896, 38.516105205547866}, + {-121.47578716278075, 38.51617236229197}, + }}, "geometry", []string{"multipolygon1"}}, + + // test with query polygon for envelope1. + { + [][][]float64{{ + {36.197547912597656, 49.99642946989866}, + {36.18939399719238, 49.988649165474}, + {36.20201110839844, 49.98853879749191}, + {36.1970329284668, 49.980150089789376}, + { + 36.205787658691406, + 49.9885939815146, + }, + {36.197547912597656, 49.99642946989866}, + }}, + "geometry", + []string{"envelope1"}, + }, + + // test with query polygon for no hits. (envelope1 has one vertex outside the polygon) + {[][][]float64{{ + {36.19832038879394, 49.99626394461266}, + {36.19016647338867, 49.98439981533724}, + {36.20698928833008, 49.98158510403259}, + {36.19832038879394, 49.99626394461266}, + }}, "geometry", nil}, + } + i := setupGeoJsonShapesIndexForPolygonQuery(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapePolygonQueryWithRelation("contains", + indexReader, test.polygon, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", + n, test.want, got, test.polygon) + } + } +} + +func TestGeoJsonPolygonWithInQuery(t *testing.T) { + tests := []struct { + polygon [][][]float64 + field string + want []string + }{ + // test with query polygon for polygon1. + { + [][][]float64{{ + {77.58407592773438, 12.956382587313202}, + {77.57746696472168, 12.943249893344905}, + {77.5920581817627, 12.944086391304364}, + {77.59454727172852, 12.95353862313803}, + {77.58407592773438, 12.956382587313202}, + }}, + "geometry", + []string{"polygon1"}, + }, + + // test with query polygon for circle1 and polygon3. + { + [][][]float64{{ + {77.59248733520508, 12.967841760870071}, + {77.58261680603027, 12.968594534825176}, + {77.57789611816406, 12.957302686416881}, + {77.58896827697754, 12.945341132980488}, + {77.60450363159178, 12.947599652080394}, + {77.60673522949219, 12.96483064227584}, + {77.59248733520508, 12.967841760870071}, + }}, + "geometry", + []string{"polygon3", "circle1"}, + }, + + // test with query polygon for linestring2, multilinestring2. + { + [][][]float64{{ + {77.59909629821777, 12.998118204343788}, + {77.58931159973145, 12.978882217224443}, + {77.61128425598145, 12.983565899088745}, + {77.59909629821777, 12.998118204343788}, + }}, + "geometry", + []string{"linestring2", "multilinestring2"}, + }, + + // test with query polygon for multipoint1. + {[][][]float64{{ + {77.55703926086426, 12.964245142762644}, + {77.5631332397461, 12.944253690559432}, + {77.57429122924805, 12.957720912158363}, + {77.55703926086426, 12.964245142762644}, + }}, "geometry", []string{"multipoint1"}}, + + // test with query polygon with no results. + // (polygon4 has one vertex lying outside the query polygon). + { + [][][]float64{{ + {-121.48812532424927, 38.51058134885975}, + {-121.48258924484252, 38.500153704565065}, + {-121.47492885589598, 38.50799556819636}, + {-121.48630142211913, 38.51147123890908}, + {-121.48812532424927, 38.51058134885975}, + }}, + "geometry", nil, + }, + + // test with query polygon for polygon4. + {[][][]float64{{ + {-121.48366212844849, 38.510161585585045}, + {-121.48533582687377, 38.50841534409804}, + {-121.48376941680908, 38.507777283760426}, + {-121.48370504379272, 38.50250467407243}, + {-121.48010015487672, 38.50253825879518}, + {-121.48018598556519, 38.504502937819765}, + {-121.47756814956665, 38.50755899866278}, + {-121.48113012313843, 38.50866720846446}, + {-121.48115158081055, 38.51017837616302}, + {-121.48366212844849, 38.510161585585045}, + }}, "geometry", []string{"polygon4"}}, + + // test with query polygon for envelope1. + { + [][][]float64{{ + {36.20587348937988, 50.00470500769241}, + {36.17969512939453, 49.993946530777606}, + {36.19368553161621, 49.971870325635074}, + {36.21119499206543, 49.983075265826656}, + {36.20587348937988, 50.00470500769241}, + }}, + "geometry", + []string{"envelope1"}, + }, + + // test with query polygon for linestring2 which lies outside except the endpoints. + { + [][][]float64{{ + {8.515305519104004, 47.392597129887}, + {8.514232635498047, 47.38896544894171}, + {8.507537841796875, 47.38815191810328}, + {8.514318466186523, 47.38725120859953}, + {8.516035079956053, 47.383357642070706}, + {8.516979217529295, 47.38733837470806}, + {8.522472381591797, 47.38794853343167}, + {8.516507148742676, 47.388994503382285}, + {8.515305519104004, 47.392597129887}, + }}, + "geometry", nil, + }, + + // test with query polygon for all the shapes. + { + [][][]float64{{ + {-135.0, -38.0}, + {149.0, -38.0}, + {149.0, 77.0}, + {-135.0, 77.0}, + }}, + "geometry", + []string{ + "polygon1", "polygon2", "polygon3", "envelope1", "circle1", "linestring1", + "linestring2", "linestring3", "multilinestring1", "multilinestring2", "multipoint1", + "polygonWithHole1", "polygon4", "multipolygon1", + }, + }, + } + + i := setupGeoJsonShapesIndexForPolygonQuery(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapePolygonQueryWithRelation("within", + indexReader, test.polygon, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", + n, test.want, got, test.polygon) + } + } +} + +func runGeoShapePolygonQueryWithRelation(relation string, i index.IndexReader, + points [][][]float64, field string, +) ([]string, error) { + s := geo.NewGeoJsonPolygon(points) + return executeSearch(relation, i, s, field) +} + +// runGeoShapePolygonQueryV1Only runs only the v1 searcher. It is used by the +// specific-issue regression tests (e.g. the S2 loop porting issue) which have +// no v2 equivalent and therefore must not trigger the v1/v2 agreement check. +func runGeoShapePolygonQueryV1Only(relation string, i index.IndexReader, + points [][][]float64, field string, +) ([]string, error) { + s := geo.NewGeoJsonPolygon(points) + return executeGeoShapeV1Search(relation, i, s, field) +} + +func setupGeoJsonShapesIndexForPolygonQuery(t *testing.T) index.Index { + analysisQueue := index.NewAnalysisQueue(1) + i, err := scorch.NewScorch( + gtreap.Name, + map[string]interface{}{ + "path": "", + "spatialPlugin": "s2", + }, + analysisQueue) + if err != nil { + t.Fatal(err) + } + err = i.Open() + if err != nil { + t.Fatal(err) + } + + polygon1 := [][][][]float64{{{ + {77.5853419303894, 12.953977766785052}, + {77.58405447006226, 12.95393594361393}, + {77.5819730758667, 12.9495026476557}, + {77.58068561553955, 12.94883346405509}, + {77.58019208908081, 12.948331575175299}, + {77.57991313934326, 12.943814529775414}, + {77.58497714996338, 12.94394000436408}, + {77.58517026901245, 12.9446301134728}, + {77.58572816848755, 12.945508431393435}, + {77.58785247802734, 12.946365833997325}, + {77.58967638015747, 12.946428570657417}, + {77.59070634841918, 12.947474179333993}, + {77.59317398071289, 12.948875288082773}, + {77.59167194366454, 12.949962710338657}, + {77.59077072143555, 12.950276388953625}, + {77.59098529815674, 12.951196510612728}, + {77.58729457855225, 12.952472128200755}, + {77.5853419303894, 12.953977766785052}, + }}} + doc := document.NewDocument("polygon1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + polygon1, "polygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + polygon2 := [][][][]float64{{{ + {77.59527683258057, 12.951112863329588}, + {77.59420394897461, 12.947976069940545}, + {77.59579181671143, 12.946010325958518}, + {77.60347366333008, 12.950401860289055}, + {77.60673522949219, 12.95600618215462}, + {77.60107040405273, 12.96345053407734}, + {77.5984525680542, 12.961861309096507}, + {77.59527683258057, 12.951112863329588}, + }}} + doc = document.NewDocument("polygon2") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + polygon2, "polygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + polygon3 := [][][][]float64{{{ + {77.59974002838135, 12.953789562459688}, + {77.60347366333008, 12.953789562459688}, + {77.60347366333008, 12.957720912158363}, + {77.59974002838135, 12.957720912158363}, + {77.59974002838135, 12.953789562459688}, + }}} + doc = document.NewDocument("polygon3") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + polygon3, "polygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + /*polygon4 := [][][][]float64{{{{8.515305519104004, 47.392597129887}, + {8.514232635498047, 47.38896544894171}, {8.507537841796875, 47.38815191810328}, + {8.514318466186523, 47.38725120859953}, {8.516035079956053, 47.383357642070706}, + {8.516979217529295, 47.38733837470806}, {8.522472381591797, 47.38794853343167}, + {8.516507148742676, 47.388994503382285}, {8.515305519104004, 47.392597129887}}}} + doc = document.NewDocument("polygon4") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + polygon4, "polygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + }*/ + + // not working envelope + envelope1 := [][][][]float64{{{ + {36.18896484375, 49.9799293145682}, + {36.20613098144531, 49.99714673955337}, + }}} + doc = document.NewDocument("envelope1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + envelope1, "envelope", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + doc = document.NewDocument("circle1") + addGeoCircleFieldV1V2(doc,"geometry", + []uint64{}, []float64{77.59253025054932, 12.955587953533424}, "900m", + document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + linestring := [][][][]float64{{{ + {77.5841188430786, 12.957093573282744}, + {77.57776737213135, 12.952074805390097}, + }}} + doc = document.NewDocument("linestring1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + linestring, "linestring", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + linestring1 := [][][][]float64{{{ + {77.60188579559325, 12.982604078764705}, + {77.60557651519775, 12.987329508048184}, + }}} + doc = document.NewDocument("linestring2") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + linestring1, "linestring", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + linestring3 := [][][][]float64{{{ + {8.51539134979248, 47.390592472948434}, + {8.520884513854979, 47.388006643417924}, + }}} + doc = document.NewDocument("linestring3") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + linestring3, "linestring", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + multilinestring := [][][][]float64{{ + { + {77.57227420806883, 12.948687079902895}, + {77.57600784301758, 12.954165970968194}, + }, + {{77.5779390335083, 12.94471376293191}, {77.57218837738037, 12.948268838994263}}, + {{77.57781028747559, 12.951740217268595}, {77.5779390335083, 12.945006535817749}}, + }} + doc = document.NewDocument("multilinestring1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + multilinestring, "multilinestring", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + multilinestring1 := [][][][]float64{{ + { + {77.6015853881836, 12.990089451715061}, + {77.60476112365723, 12.987747683302153}, + }, + {{77.59875297546387, 12.988751301039581}, {77.59446144104004, 12.98197680263484}}, + {{77.60188579559325, 12.982604078764705}, {77.60557651519775, 12.987329508048184}}, + }} + doc = document.NewDocument("multilinestring2") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + multilinestring1, "multilinestring", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + multipoint1 := [][][][]float64{{{ + {77.56618022918701, 12.958180959662695}, + {77.56407737731932, 12.951614746607163}, + {77.56922721862793, 12.956173473406446}, + }}} + doc = document.NewDocument("multipoint1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + multipoint1, "multipoint", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + polygonWithHole1 := [][][][]float64{{ + { + {77.59991168975829, 12.972232910164502}, + {77.6039457321167, 12.97582941279006}, + {77.60424613952637, 12.98168407323241}, + {77.59974002838135, 12.985489528568463}, + {77.59321689605713, 12.979300406693417}, + {77.59991168975829, 12.972232910164502}, + }, + { + {77.59682178497314, 12.975787593290978}, + {77.60295867919922, 12.975787593290978}, + {77.60295867919922, 12.98143316204164}, + {77.59682178497314, 12.98143316204164}, + {77.59682178497314, 12.975787593290978}, + }, + }} + + doc = document.NewDocument("polygonWithHole1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + polygonWithHole1, "polygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + polygon4 := [][][][]float64{{{ + {-121.48125886917113, 38.51009442323401}, + {-121.48361921310425, 38.51012800441735}, + {-121.48497104644774, 38.50858325377352}, + {-121.48366212844849, 38.507861239391026}, + {-121.48353338241577, 38.50277335141579}, + {-121.4803147315979, 38.50267259752949}, + {-121.48033618927, 38.5046204810195}, + {-121.47771835327147, 38.50754220747402}, + {-121.48123741149902, 38.508616835661655}, + {-121.48125886917113, 38.51009442323401}, + }}} + + doc = document.NewDocument("polygon4") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + polygon4, "polygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + multipolygon1 := [][][][]float64{ + {{ + {-121.49104356765746, 38.52149433504263}, + {-121.47857666015625, 38.51592052417851}, + {-121.47688150405884, 38.515970891871696}, + {-121.4770746231079, 38.51714612804143}, + {-121.49033546447754, 38.52221621271097}, + {-121.49104356765746, 38.52149433504263}, + }}, + {{ + {-121.47647380828859, 38.51714612804143}, + {-121.47658109664916, 38.51477884701455}, + {-121.4741563796997, 38.5159876810949}, + {-121.47647380828859, 38.51714612804143}, + }}, + } + + doc = document.NewDocument("multipolygon1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + multipolygon1, "multipolygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + return i +} + +func TestGeoJsonMultiPolygonWithInQuery(t *testing.T) { + tests := []struct { + polygon [][][][]float64 + field string + want []string + }{ + // test within multipolygon query for multipolygon1. + // (where each query polygon contains each of the indexed polygons) + { + [][][][]float64{ + { + { + {-121.49458408355713, 38.53270780324851}, + {-121.48823261260985, 38.52533866992879}, + {-121.48048639297485, 38.53253994984147}, + {-121.49458408355713, 38.53270780324851}, + }, + }, + {{ + {-121.48700952529907, 38.53306029412857}, + {-121.48160219192505, 38.53306029412857}, + {-121.48160219192505, 38.53829709805414}, + {-121.48700952529907, 38.53829709805414}, + {-121.48700952529907, 38.53306029412857}, + }}, + {{ + {-121.47344827651976, 38.54475865436684}, + {-121.46396398544312, 38.54475865436684}, + {-121.46396398544312, 38.55366961462033}, + {-121.47344827651976, 38.55366961462033}, + {-121.47344827651976, 38.54475865436684}, + }}, + }, + "geometry", + []string{"multipolygon1"}, + }, + + // test within multipolygon query. (only partial containment of the three + // indexed polygons by the two query polygons) + { + [][][][]float64{ + { + { + {-121.49458408355713, 38.53270780324851}, + {-121.48823261260985, 38.52533866992879}, + {-121.48048639297485, 38.53253994984147}, + {-121.49458408355713, 38.53270780324851}, + }, + }, + {{ + {-121.48700952529907, 38.53306029412857}, + {-121.48160219192505, 38.53306029412857}, + {-121.48160219192505, 38.53829709805414}, + {-121.48700952529907, 38.53829709805414}, + {-121.48700952529907, 38.53306029412857}, + }}, + {{ + {-121.4734697341919, 38.544825784372485}, + {-121.4644145965576, 38.544825784372485}, + {-121.4644145965576, 38.5537199558913}, + {-121.4734697341919, 38.5537199558913}, + {-121.4734697341919, 38.544825784372485}, + }}, + }, + "geometry", nil, + }, + + // test within multipolygon query for multilinestring1. + {[][][][]float64{ + {{ + {-121.49876832962036, 38.551739839324334}, + {-121.49814605712889, 38.54553064564853}, + {-121.49158000946044, 38.54908841140355}, + {-121.49876832962036, 38.551739839324334}, + }}, + { + { + {-121.49258852005006, 38.54294612052762}, + {-121.49117231369017, 38.54294612052762}, + {-121.49117231369017, 38.54526212788182}, + {-121.49258852005006, 38.54526212788182}, + {-121.49258852005006, 38.54294612052762}, + }, + }, + }, "geometry", []string{"multilinestring1"}}, + + // test within multipolygon query for multipoint1. + {[][][][]float64{ + {{ + {-121.50286674499512, 38.564810956372185}, + {-121.49694442749023, 38.56226068115802}, + {-121.48406982421875, 38.5675624676039}, + {-121.4875030517578, 38.57514535565976}, + {-121.50286674499512, 38.564810956372185}, + }}, + {{ + {-121.48685932159422, 38.565163289911425}, + {-121.48623704910278, 38.56283114531348}, + {-121.48357629776001, 38.565129734410704}, + {-121.48685932159422, 38.565163289911425}, + }}, + { + { + {-121.49430513381958, 38.56195866888961}, + {-121.4899492263794, 38.5584518779682}, + {-121.48842573165892, 38.56194189039304}, + {-121.49430513381958, 38.56195866888961}, + }, + }, + }, "geometry", []string{"multipoint1"}}, + } + i := setupGeoJsonShapesIndexForMultiPolygonQuery(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapeMultiPolygonQueryWithRelation("within", + indexReader, test.polygon, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", + n, test.want, got, test.polygon) + } + } +} + +func runGeoShapeMultiPolygonQueryWithRelation(relation string, + i index.IndexReader, + points [][][][]float64, field string, +) ([]string, error) { + s := geo.NewGeoJsonMultiPolygon(points) + return executeSearch(relation, i, s, field) +} + +func setupGeoJsonShapesIndexForMultiPolygonQuery(t *testing.T) index.Index { + analysisQueue := index.NewAnalysisQueue(1) + i, err := scorch.NewScorch( + gtreap.Name, + map[string]interface{}{ + "path": "", + "spatialPlugin": "s2", + }, + analysisQueue) + if err != nil { + t.Fatal(err) + } + err = i.Open() + if err != nil { + t.Fatal(err) + } + + multipolygon1 := [][][][]float64{{{ + {-121.49140834808348, 38.5320028163074}, + {-121.49112939834593, 38.52916601331889}, + {-121.48889780044556, 38.52913244101627}, + {-121.4887261390686, 38.527655244193205}, + {-121.48559331893921, 38.52794061412457}, + {-121.48638725280762, 38.53213710006686}, + {-121.49140834808348, 38.5320028163074}, + }}, // polygon1 + {{ + {-121.48677349090575, 38.533194575914315}, + {-121.48179531097412, 38.533194575914315}, + {-121.48179531097412, 38.53814604174215}, + {-121.48677349090575, 38.53814604174215}, + {-121.48677349090575, 38.533194575914315}, + }}, // polygon2 + {{ + {-121.47334098815918, 38.553485029658475}, + {-121.47329807281494, 38.54485934935182}, + {-121.46415710449219, 38.54526212788182}, + {-121.47334098815918, 38.553485029658475}, + }}} + doc := document.NewDocument("multipolygon1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + multipolygon1, "multipolygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + multilinestring1 := [][][][]float64{{ + { + {-121.4983820915222, 38.55081688500274}, + {-121.49649381637572, 38.550447699956685}, + }, + {{-121.49655818939209, 38.548635309508775}, {-121.49370431900023, 38.54811507788636}}, + {{-121.49134397506714, 38.54490969679143}, {-121.4919662475586, 38.54304681805045}}, + }} + doc = document.NewDocument("multilinestring1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + multilinestring1, "multilinestring", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + multipoint1 := [][][][]float64{{{ + {-121.48960590362547, 38.56066671319285}, + {-121.4933180809021, 38.56157276247755}, + {-121.4973521232605, 38.56318348855919}, + {-121.48582935333252, 38.56736114108619}, + {-121.50104284286498, 38.56449217691959}, + {-121.4881682395935, 38.57158887950165}, + }}} + doc = document.NewDocument("multipoint1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + multipoint1, "multipoint", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + return i +} + +func setupGeoJsonPolygonS2LoopPortingIssue(t *testing.T) index.Index { + analysisQueue := index.NewAnalysisQueue(1) + i, err := scorch.NewScorch( + gtreap.Name, + map[string]interface{}{ + "path": "", + "spatialPlugin": "s2", + }, + analysisQueue) + if err != nil { + t.Fatal(err) + } + err = i.Open() + if err != nil { + t.Fatal(err) + } + + polygon1 := [][][][]float64{{{ + {-135.0, -38.0}, + {149.0, -38.0}, + {149.0, 77.0}, + {-135.0, 77.0}, + }}} + doc := document.NewDocument("polygon1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + polygon1, "polygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + return i +} + +func TestGeoJsonPolygonContainsQueryS2LoopPortingIssue(t *testing.T) { + tests := []struct { + polygon [][][]float64 + field string + want []string + }{ + // test containment query polygon for polygon1. + { + [][][]float64{{ + {13.007812500000002, 37.99616267972809}, + {13.559375000000002, 37.99616267972809}, + {13.559375000000002, 38.472819658516866}, + {13.007812500000002, 38.472819658516866}, + }}, + "geometry", + []string{"polygon1"}, + }, + + // test containment query polygon for polygon1. + { + [][][]float64{{ + {13.007812500000002, 37.99616267972809}, + {13.359375000000002, 37.99616267972809}, + {13.359375000000002, 38.272819658516866}, + {13.007812500000002, 38.272819658516866}, + }}, + "geometry", + []string{"polygon1"}, + }, + } + i := setupGeoJsonPolygonS2LoopPortingIssue(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapePolygonQueryV1Only("contains", + indexReader, test.polygon, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", + n, test.want, got, test.polygon) + } + } +} + +func TestGeoJsonPolygonIntersectsQuery1(t *testing.T) { + tests := []struct { + polygon [][][]float64 + field string + want []string + }{ + // test non-intersecting query polygon. + {[][][]float64{{ + { + 97.745361328125, + 68.21644657802169, + }, + { + 97.701416015625, + 67.97051353559428, + }, + { + 97.80029296875, + 67.97875365614591, + }, + { + 97.745361328125, + 68.21644657802169, + }, + }}, "geometry", nil}, + + // test intersecting query polygon. + {[][][]float64{{ + { + 77.59214401245117, + 12.966043458314124, + }, + { + 77.58853912353516, + 12.95232574618635, + }, + { + 77.60943889617919, + 12.956466232826733, + }, + { + 77.59214401245117, + 12.966043458314124, + }, + }}, "geometry", nil}, + + // test intersecting query polygon for polygon1. + {[][][]float64{{ + {97.0806884765625, 61.61423180712503}, + {96.7510986328125, 61.54625879879804}, + {97.305908203125, 61.367777577924}, + {97.0806884765625, 61.61423180712503}, + }}, "geometry", []string{"polygon1"}}, + } + i := setupGeoJsonShapesIndexForPolygonQuery1(t) + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err = indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + for n, test := range tests { + got, err := runGeoShapePolygonQueryV1Only("intersects", + indexReader, test.polygon, test.field) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("test %d, expected %v, got %v for polygon: %+v", n, + test.want, got, test.polygon) + } + } +} + +func setupGeoJsonShapesIndexForPolygonQuery1(t *testing.T) index.Index { + analysisQueue := index.NewAnalysisQueue(1) + i, err := scorch.NewScorch( + gtreap.Name, + map[string]interface{}{ + "path": "", + "spatialPlugin": "s2", + }, + analysisQueue) + if err != nil { + t.Fatal(err) + } + err = i.Open() + if err != nil { + t.Fatal(err) + } + + polygon1 := [][][][]float64{{{ + {96.69202458735312, 61.59480859768306}, + {96.79202458735311, 61.39480859768306}, + {96.79202458735311, 61.59480859768306}, + {96.69202458735312, 61.59480859768306}, + }}} + doc := document.NewDocument("polygon1") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + polygon1, "polygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + polygon2 := [][][][]float64{{{ + {91.35604953911839, 65.11164029408492}, + {91.45604953911838, 64.91164029408492}, + {91.45604953911838, 65.11164029408492}, + {91.35604953911839, 65.11164029408492}, + }}} + doc = document.NewDocument("polygon2") + addGeoShapeFieldV1V2(doc,"geometry", []uint64{}, + polygon2, "polygon", document.DefaultGeoShapeIndexingOptions) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + return i +} diff --git a/search/searcher/search_geoshape_v2.go b/search/searcher/search_geoshape_v2.go new file mode 100644 index 0000000..8a8bea0 --- /dev/null +++ b/search/searcher/search_geoshape_v2.go @@ -0,0 +1,128 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "fmt" + "reflect" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeGeoShapeV2Searcher int + +func init() { + var gsv2s GeoShapeV2Searcher + reflectStaticSizeGeoShapeV2Searcher = int(reflect.TypeOf(gsv2s).Size()) +} + +type GeoShapeV2Searcher struct { + geoShapeIndexReader index.GeoShapeV2FieldReader + scorer *scorer.ConstantScorer + + gd index.GeoShapeV2FieldDoc +} + +func NewGeoShapeV2Searcher(ctx context.Context, indexReader index.IndexReader, + shape index.GeoJSON, relation string, field string, boost float64, + options search.SearcherOptions, +) (search.Searcher, error) { + + if gr, ok := indexReader.(index.GeoShapeV2IndexReader); ok { + // get the GeoShapeV2FieldReader for the specified field + geoShapeIndexReader, err := gr.GeoShapeV2FieldReader(ctx, field) + if err != nil { + return nil, err + } + + // perform the search on the GeoShapeV2FieldReader with the specified + // shape and relation + err = geoShapeIndexReader.Search(shape, relation) + if err != nil { + return nil, err + } + + return &GeoShapeV2Searcher{ + geoShapeIndexReader: geoShapeIndexReader, + scorer: scorer.NewConstantScorer(1, boost, options), + gd: index.GeoShapeV2FieldDoc{}, + }, nil + } + + return nil, fmt.Errorf("indexReader does not support geoshape_v2 queries") +} + +// Next returns the next document match for the GeoShapeV2Searcher. +// It retrieves the next matching document from the GeoShapeV2FieldReader +// and scores it using the ConstantScorer. +func (g *GeoShapeV2Searcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + match, err := g.geoShapeIndexReader.Next(g.gd.Reset()) + if err != nil { + return nil, err + } + if match == nil { + return nil, nil + } + + docMatch := g.scorer.Score(ctx, match.ID) + return docMatch, nil +} + +// Advance moves the searcher to the first document with an ID greater than or equal to the specified ID. +// It retrieves the next matching document from the GeoShapeV2FieldReader and scores it using the ConstantScorer. +func (g *GeoShapeV2Searcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + match, err := g.geoShapeIndexReader.Advance(ID, g.gd.Reset()) + if err != nil { + return nil, err + } + if match == nil { + return nil, nil + } + + docMatch := g.scorer.Score(ctx, match.ID) + return docMatch, nil +} + +func (g *GeoShapeV2Searcher) Close() error { + return g.geoShapeIndexReader.Close() +} + +func (g *GeoShapeV2Searcher) Count() uint64 { + return g.geoShapeIndexReader.Count() +} + +func (g *GeoShapeV2Searcher) DocumentMatchPoolSize() int { + return 1 +} + +func (g *GeoShapeV2Searcher) Min() int { + return 0 +} + +func (g *GeoShapeV2Searcher) SetQueryNorm(n float64) { + g.scorer.SetQueryNorm(n) +} + +func (g *GeoShapeV2Searcher) Size() int { + return reflectStaticSizeGeoShapeV2Searcher + g.geoShapeIndexReader.Size() + + g.scorer.Size() + g.gd.Size() +} + +func (g *GeoShapeV2Searcher) Weight() float64 { + return g.scorer.Weight() +} diff --git a/search/searcher/search_geoshape_v2_test.go b/search/searcher/search_geoshape_v2_test.go new file mode 100644 index 0000000..2d276b9 --- /dev/null +++ b/search/searcher/search_geoshape_v2_test.go @@ -0,0 +1,455 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "fmt" + "sort" + "testing" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/geo/geojson" +) + +const geoV2TestField = "geometry" + +// box returns a single-polygon coordinate set (one CCW exterior ring) +// spanning the given lng/lat extents. +func box(minLng, minLat, maxLng, maxLat float64) [][][][]float64 { + return [][][][]float64{{{ + {minLng, minLat}, + {maxLng, minLat}, + {maxLng, maxLat}, + {minLng, maxLat}, + {minLng, minLat}, + }}} +} + +func addGeoShapeV2Doc(t *testing.T, i index.Index, id, typ string, + coords [][][][]float64) { + shape := &geojson.GeoShape{Type: typ, Coordinates: coords} + doc := document.NewDocument(id) + f := document.NewGeoShapeV2FieldFromShapeWithIndexingOptions( + geoV2TestField, shape, index.IndexField) + if f == nil { + t.Fatalf("failed to build geoshape_v2 field for %q", id) + } + doc.AddField(f) + if err := i.Update(doc); err != nil { + t.Fatal(err) + } +} + +// setupGeoShapeV2Index builds a scorch index with a spread of shapes whose +// relationships to the query box Q = [0,0]-[10,10] are unambiguous: +// +// smallInside : polygon fully inside Q +// pointInside : point inside Q +// overlapping : polygon straddling the corner of Q +// bigOutside : polygon fully disjoint from Q +// hugeContainer : polygon that fully contains Q +func setupGeoShapeV2Index(t *testing.T) index.Index { + analysisQueue := index.NewAnalysisQueue(1) + i, err := scorch.NewScorch(gtreap.Name, + map[string]interface{}{ + "path": "", + "spatialPlugin": "s2", + }, analysisQueue) + if err != nil { + t.Fatal(err) + } + if err = i.Open(); err != nil { + t.Fatal(err) + } + + addGeoShapeV2Doc(t, i, "smallInside", "polygon", box(2, 2, 4, 4)) + addGeoShapeV2Doc(t, i, "pointInside", "point", + [][][][]float64{{{{5, 5}}}}) + addGeoShapeV2Doc(t, i, "overlapping", "polygon", box(8, 8, 15, 15)) + addGeoShapeV2Doc(t, i, "bigOutside", "polygon", box(20, 20, 25, 25)) + addGeoShapeV2Doc(t, i, "hugeContainer", "polygon", box(-10, -10, 20, 20)) + + return i +} + +func executeGeoShapeV2Search(relation string, i index.IndexReader, + s index.GeoJSON, field string) ([]string, error) { + var rv []string + gbs, err := NewGeoShapeV2Searcher(context.TODO(), i, s, relation, field, + 1.0, search.SearcherOptions{}) + if err != nil { + return nil, err + } + defer func() { _ = gbs.Close() }() + + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(gbs.DocumentMatchPoolSize(), 0), + } + docMatch, err := gbs.Next(ctx) + for docMatch != nil && err == nil { + docID, _ := i.ExternalID(docMatch.IndexInternalID) + rv = append(rv, docID) + docMatch, err = gbs.Next(ctx) + } + if err != nil { + return nil, err + } + sort.Strings(rv) + return rv, nil +} + +func TestGeoShapeV2Relations(t *testing.T) { + i := setupGeoShapeV2Index(t) + defer func() { _ = i.Close() }() + + indexReader, err := i.Reader() + if err != nil { + t.Fatal(err) + } + defer func() { _ = indexReader.Close() }() + + // query box Q = [0,0]-[10,10] + queryShape, _, err := geo.NewGeoJsonShape(box(0, 0, 10, 10), "polygon") + if err != nil { + t.Fatal(err) + } + + tests := []struct { + relation string + want []string + }{ + {"within", []string{"pointInside", "smallInside"}}, + {"intersects", []string{"hugeContainer", "overlapping", "pointInside", "smallInside"}}, + {"contains", []string{"hugeContainer"}}, + {"disjoint", []string{"bigOutside"}}, + } + + for _, test := range tests { + got, err := executeGeoShapeV2Search(test.relation, indexReader, + queryShape, geoV2TestField) + if err != nil { + t.Fatalf("relation %q: %v", test.relation, err) + } + want := append([]string(nil), test.want...) + sort.Strings(want) + if !equalStrings(got, want) { + t.Errorf("relation %q: expected %v, got %v", test.relation, want, got) + } + } +} + +func TestGeoShapeV2Count(t *testing.T) { + i := setupGeoShapeV2Index(t) + defer func() { _ = i.Close() }() + + indexReader, err := i.Reader() + if err != nil { + t.Fatal(err) + } + defer func() { _ = indexReader.Close() }() + + queryShape, _, err := geo.NewGeoJsonShape(box(0, 0, 10, 10), "polygon") + if err != nil { + t.Fatal(err) + } + + gbs, err := NewGeoShapeV2Searcher(context.TODO(), indexReader, queryShape, + "intersects", geoV2TestField, 1.0, search.SearcherOptions{}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = gbs.Close() }() + + if got := gbs.Count(); got != 4 { + t.Fatalf("expected Count 4 for intersects, got %d", got) + } +} + +func TestGeoShapeV2UnsupportedReaderErrors(t *testing.T) { + // a nil / non-GeoShapeV2 index reader must produce an error rather than + // a nil searcher, so that callers which do not nil-check (e.g. the + // conjunction searcher) do not panic + queryShape, _, err := geo.NewGeoJsonShape(box(0, 0, 10, 10), "polygon") + if err != nil { + t.Fatal(err) + } + _, err = NewGeoShapeV2Searcher(context.TODO(), nil, queryShape, + "intersects", geoV2TestField, 1.0, search.SearcherOptions{}) + if err == nil { + t.Fatal("expected an error for an index reader that does not support geoshape_v2") + } +} + +// TestGeoShapeV2Advance drives Advance across the per-segment iterators and +// confirms it lands on the requested document and continues correctly. It +// also guards the ConstantScorer's per-match ID copy: the internal IDs +// collected across successive Next calls must stay distinct rather than all +// aliasing the reader's reused buffer. +func TestGeoShapeV2Advance(t *testing.T) { + i := setupGeoShapeV2Index(t) + defer func() { _ = i.Close() }() + + indexReader, err := i.Reader() + if err != nil { + t.Fatal(err) + } + defer func() { _ = indexReader.Close() }() + + queryShape, _, err := geo.NewGeoJsonShape(box(0, 0, 10, 10), "polygon") + if err != nil { + t.Fatal(err) + } + + // first pass: collect the ordered internal IDs of all intersects hits, + // copying each one so a reused buffer cannot corrupt the collection + newSearcher := func() search.Searcher { + s, serr := NewGeoShapeV2Searcher(context.TODO(), indexReader, queryShape, + "intersects", geoV2TestField, 1.0, search.SearcherOptions{}) + if serr != nil { + t.Fatal(serr) + } + return s + } + + s1 := newSearcher() + defer func() { _ = s1.Close() }() + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(s1.DocumentMatchPoolSize(), 0), + } + + var ids []index.IndexInternalID + dm, err := s1.Next(ctx) + for dm != nil && err == nil { + ids = append(ids, append(index.IndexInternalID(nil), dm.IndexInternalID...)) + dm, err = s1.Next(ctx) + } + if err != nil { + t.Fatal(err) + } + if len(ids) < 3 { + t.Fatalf("expected at least 3 intersects hits to test Advance, got %d", len(ids)) + } + + // the collected IDs must be strictly increasing and distinct; if the + // scorer aliased the reader's buffer they would all be equal + for k := 1; k < len(ids); k++ { + if ids[k].Compare(ids[k-1]) <= 0 { + t.Fatalf("collected internal IDs are not strictly increasing: %v", ids) + } + } + + // second pass: Advance directly to the second hit and confirm we land on it + s2 := newSearcher() + defer func() { _ = s2.Close() }() + ctx2 := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(s2.DocumentMatchPoolSize(), 0), + } + got, err := s2.Advance(ctx2, ids[1]) + if err != nil { + t.Fatal(err) + } + if got == nil || !got.IndexInternalID.Equals(ids[1]) { + t.Fatalf("Advance did not land on the requested doc: want %v, got %v", + ids[1], got) + } + // continuing with Next must yield the following hit + next, err := s2.Next(ctx2) + if err != nil { + t.Fatal(err) + } + if next == nil || !next.IndexInternalID.Equals(ids[2]) { + t.Fatalf("Next after Advance: want %v, got %v", ids[2], next) + } +} + +func TestGeoShapeV2CircleAndCollection(t *testing.T) { + analysisQueue := index.NewAnalysisQueue(1) + i, err := scorch.NewScorch(gtreap.Name, + map[string]interface{}{"path": "", "spatialPlugin": "s2"}, analysisQueue) + if err != nil { + t.Fatal(err) + } + if err = i.Open(); err != nil { + t.Fatal(err) + } + defer func() { _ = i.Close() }() + + // a circle centred inside Q + circle := &geojson.GeoShape{Type: geo.CircleType, Center: []float64{5, 5}, Radius: "100m"} + doc := document.NewDocument("circleInside") + cf := document.NewGeoShapeV2FieldFromShapeWithIndexingOptions(geoV2TestField, + circle, index.IndexField) + if cf == nil { + t.Fatal("failed to build circle geoshape_v2 field") + } + doc.AddField(cf) + if err = i.Update(doc); err != nil { + t.Fatal(err) + } + + // a geometry collection with one polygon inside Q and one far outside + shapes := []*geojson.GeoShape{ + {Type: "polygon", Coordinates: box(1, 1, 3, 3)}, + {Type: "polygon", Coordinates: box(40, 40, 42, 42)}, + } + gcDoc := document.NewDocument("collection") + gf := document.NewGeometryCollectionV2FieldFromShapesWithIndexingOptions( + geoV2TestField, shapes, index.IndexField) + if gf == nil { + t.Fatal("failed to build geometry collection geoshape_v2 field") + } + gcDoc.AddField(gf) + if err = i.Update(gcDoc); err != nil { + t.Fatal(err) + } + + indexReader, err := i.Reader() + if err != nil { + t.Fatal(err) + } + defer func() { _ = indexReader.Close() }() + + queryShape, _, err := geo.NewGeoJsonShape(box(0, 0, 10, 10), "polygon") + if err != nil { + t.Fatal(err) + } + + // both the circle and the collection (via its inside polygon) intersect Q + got, err := executeGeoShapeV2Search("intersects", indexReader, queryShape, geoV2TestField) + if err != nil { + t.Fatal(err) + } + want := []string{"circleInside", "collection"} + if !equalStrings(got, want) { + t.Errorf("intersects: expected %v, got %v", want, got) + } +} + +func equalStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// geoV2FieldSuffix is appended to a v1 geoshape field name to produce the +// parallel geoshape_v2 field name. The shared v1 test corpus indexes each +// shape under both names so every v1 test also exercises the v2 path. +const geoV2FieldSuffix = "_v2" + +// addGeoShapeV2Parallel adds only the geoshape_v2 field, under "_v2". +func addGeoShapeV2Parallel(doc *document.Document, name, typ string, + coords [][][][]float64) { + shape := &geojson.GeoShape{Type: typ, Coordinates: coords} + if f := document.NewGeoShapeV2FieldFromShapeWithIndexingOptions( + name+geoV2FieldSuffix, shape, index.IndexField); f != nil { + doc.AddField(f) + } +} + +// addGeoCircleV2Parallel adds only the geoshape_v2 circle field, under "_v2". +func addGeoCircleV2Parallel(doc *document.Document, name string, + center []float64, radius string) { + shape := &geojson.GeoShape{Type: geo.CircleType, Center: center, Radius: radius} + if f := document.NewGeoShapeV2FieldFromShapeWithIndexingOptions( + name+geoV2FieldSuffix, shape, index.IndexField); f != nil { + doc.AddField(f) + } +} + +// addGeoCollectionV2Parallel adds only the geoshape_v2 geometry collection +// field, under "_v2". +func addGeoCollectionV2Parallel(doc *document.Document, name string, + coords [][][][][]float64, types []string) { + shapes := make([]*geojson.GeoShape, 0, len(coords)) + for idx := range coords { + shapes = append(shapes, &geojson.GeoShape{Type: types[idx], Coordinates: coords[idx]}) + } + if f := document.NewGeometryCollectionV2FieldFromShapesWithIndexingOptions( + name+geoV2FieldSuffix, shapes, index.IndexField); f != nil { + doc.AddField(f) + } +} + +// addGeoShapeFieldV1V2 adds the v1 geoshape field and, under "_v2", +// the equivalent geoshape_v2 field carrying the same shape. +func addGeoShapeFieldV1V2(doc *document.Document, name string, ap []uint64, + coords [][][][]float64, typ string, opts index.FieldIndexingOptions) { + doc.AddField(document.NewGeoShapeFieldWithIndexingOptions(name, ap, coords, typ, opts)) + addGeoShapeV2Parallel(doc, name, typ, coords) +} + +// addGeoCircleFieldV1V2 mirrors addGeoShapeFieldV1V2 for circle shapes. +func addGeoCircleFieldV1V2(doc *document.Document, name string, ap []uint64, + center []float64, radius string, opts index.FieldIndexingOptions) { + doc.AddField(document.NewGeoCircleFieldWithIndexingOptions(name, ap, center, radius, opts)) + addGeoCircleV2Parallel(doc, name, center, radius) +} + +// addGeoCollectionFieldV1V2 mirrors addGeoShapeFieldV1V2 for geometry collections. +func addGeoCollectionFieldV1V2(doc *document.Document, name string, ap []uint64, + coords [][][][][]float64, types []string, opts index.FieldIndexingOptions) { + doc.AddField(document.NewGeometryCollectionFieldWithIndexingOptions(name, ap, coords, types, opts)) + addGeoCollectionV2Parallel(doc, name, coords, types) +} + +// sameStringSet reports whether a and b contain the same elements, ignoring +// order and treating nil and empty as equal. +func sameStringSet(a, b []string) bool { + if len(a) != len(b) { + return false + } + seen := make(map[string]int, len(a)) + for _, v := range a { + seen[v]++ + } + for _, v := range b { + seen[v]-- + } + for _, c := range seen { + if c != 0 { + return false + } + } + return true +} + +// assertGeoShapeV2Agrees runs the geoshape_v2 searcher for the same shape and +// relation against the parallel "_v2" field, and returns an error when +// its result set differs from the v1 result set. This is how each v1 searcher +// test transparently also validates the v2 implementation. +func assertGeoShapeV2Agrees(relation string, i index.IndexReader, + s index.GeoJSON, field string, v1 []string) error { + v2, err := executeGeoShapeV2Search(relation, i, s, field+geoV2FieldSuffix) + if err != nil { + return fmt.Errorf("geoshape_v2 search error for relation %q: %w", relation, err) + } + if !sameStringSet(v1, v2) { + return fmt.Errorf("geoshape_v2 mismatch for relation %q on field %q: "+ + "v1=%v v2=%v", relation, field, v1, v2) + } + return nil +} diff --git a/search/searcher/search_ip_range.go b/search/searcher/search_ip_range.go new file mode 100644 index 0000000..3826620 --- /dev/null +++ b/search/searcher/search_ip_range.go @@ -0,0 +1,68 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "net" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +// netLimits returns the lo and hi bounds inside the network. +func netLimits(n *net.IPNet) (lo net.IP, hi net.IP) { + ones, bits := n.Mask.Size() + netNum := n.IP + if bits == net.IPv4len*8 { + netNum = netNum.To16() + ones += 8 * (net.IPv6len - net.IPv4len) + } + mask := net.CIDRMask(ones, 8*net.IPv6len) + lo = make(net.IP, net.IPv6len) + hi = make(net.IP, net.IPv6len) + for i := 0; i < net.IPv6len; i++ { + lo[i] = netNum[i] & mask[i] + hi[i] = lo[i] | ^mask[i] + } + return lo, hi +} + +func NewIPRangeSearcher(ctx context.Context, indexReader index.IndexReader, ipNet *net.IPNet, + field string, boost float64, options search.SearcherOptions) ( + search.Searcher, error) { + + lo, hi := netLimits(ipNet) + fieldDict, err := indexReader.FieldDictRange(field, lo, hi) + if err != nil { + return nil, err + } + defer fieldDict.Close() + + var terms []string + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + terms = append(terms, tfd.Term) + if tooManyClauses(len(terms)) { + return nil, tooManyClausesErr(field, len(terms)) + } + tfd, err = fieldDict.Next() + } + if err != nil { + return nil, err + } + + return NewMultiTermSearcher(ctx, indexReader, terms, field, boost, options, true) +} diff --git a/search/searcher/search_ip_range_test.go b/search/searcher/search_ip_range_test.go new file mode 100644 index 0000000..7abbe1b --- /dev/null +++ b/search/searcher/search_ip_range_test.go @@ -0,0 +1,52 @@ +// Copyright (c) 2021 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "net" + "testing" +) + +func Test_netLimits(t *testing.T) { + tests := []struct { + arg string + lo string + hi string + }{ + {"128.0.0.0/1", "128.0.0.0", "255.255.255.255"}, + {"128.0.0.0/7", "128.0.0.0", "129.255.255.255"}, + {"1.1.1.1/8", "1.0.0.0", "1.255.255.255"}, + {"1.2.3.0/24", "1.2.3.0", "1.2.3.255"}, + {"1.2.2.0/23", "1.2.2.0", "1.2.3.255"}, + {"1.2.3.128/25", "1.2.3.128", "1.2.3.255"}, + {"1.2.3.0/25", "1.2.3.0", "1.2.3.127"}, + {"1.2.3.4/31", "1.2.3.4", "1.2.3.5"}, + {"1.2.3.4/32", "1.2.3.4", "1.2.3.4"}, + {"2a00:23c8:7283:ff00:1fa8:0:0:0/80", "2a00:23c8:7283:ff00:1fa8::", "2a00:23c8:7283:ff00:1fa8:ffff:ffff:ffff"}, + } + for _, tt := range tests { + t.Run(tt.arg, func(t *testing.T) { + _, net, err := net.ParseCIDR(tt.arg) + if err != nil { + t.Fatal(err) + } + lo, hi := netLimits(net) + if lo.String() != tt.lo || hi.String() != tt.hi { + t.Errorf("netLimits(%q) = %s %s, want %s %s", tt.arg, lo, hi, tt.lo, tt.hi) + } + + }) + } +} diff --git a/search/searcher/search_knn.go b/search/searcher/search_knn.go new file mode 100644 index 0000000..26d5a2a --- /dev/null +++ b/search/searcher/search_knn.go @@ -0,0 +1,145 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package searcher + +import ( + "context" + "encoding/json" + "reflect" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeKNNSearcher int + +func init() { + var ks KNNSearcher + reflectStaticSizeKNNSearcher = int(reflect.TypeOf(ks).Size()) +} + +type KNNSearcher struct { + field string + vector []float32 + k int64 + indexReader index.IndexReader + vectorReader index.VectorReader + scorer *scorer.KNNQueryScorer + count uint64 + vd index.VectorDoc +} + +func NewKNNSearcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, + options search.SearcherOptions, field string, vector []float32, k int64, + boost float64, similarityMetric string, searchParams json.RawMessage, + eligibleSelector index.EligibleDocumentSelector) ( + search.Searcher, error) { + + if vr, ok := i.(index.VectorIndexReader); ok { + vectorReader, err := vr.VectorReader(ctx, vector, field, k, searchParams, eligibleSelector) + if err != nil { + return nil, err + } + knnScorer := scorer.NewKNNQueryScorer(vector, field, boost, + options, similarityMetric) + return &KNNSearcher{ + indexReader: i, + vectorReader: vectorReader, + field: field, + vector: vector, + k: k, + scorer: knnScorer, + }, nil + } + return nil, nil +} + +func (s *KNNSearcher) VectorOptimize(ctx context.Context, octx index.VectorOptimizableContext) ( + index.VectorOptimizableContext, error) { + o, ok := s.vectorReader.(index.VectorOptimizable) + if ok { + return o.VectorOptimize(ctx, octx) + } + + return nil, nil +} + +func (s *KNNSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) ( + *search.DocumentMatch, error) { + knnMatch, err := s.vectorReader.Advance(ID, s.vd.Reset()) + if err != nil { + return nil, err + } + + if knnMatch == nil { + return nil, nil + } + + docMatch := s.scorer.Score(ctx, knnMatch) + + return docMatch, nil +} + +func (s *KNNSearcher) Close() error { + return s.vectorReader.Close() +} + +func (s *KNNSearcher) Count() uint64 { + return s.vectorReader.Count() +} + +func (s *KNNSearcher) DocumentMatchPoolSize() int { + return 1 +} + +func (s *KNNSearcher) Min() int { + return 0 +} + +func (s *KNNSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + knnMatch, err := s.vectorReader.Next(s.vd.Reset()) + if err != nil { + return nil, err + } + + if knnMatch == nil { + return nil, nil + } + + docMatch := s.scorer.Score(ctx, knnMatch) + + return docMatch, nil +} + +func (s *KNNSearcher) SetQueryNorm(qnorm float64) { + s.scorer.SetQueryNorm(qnorm) +} + +func (s *KNNSearcher) Size() int { + return reflectStaticSizeKNNSearcher + size.SizeOfPtr + + s.vectorReader.Size() + + s.vd.Size() + + s.scorer.Size() +} + +func (s *KNNSearcher) Weight() float64 { + return s.scorer.Weight() +} diff --git a/search/searcher/search_match_all.go b/search/searcher/search_match_all.go new file mode 100644 index 0000000..57d8d07 --- /dev/null +++ b/search/searcher/search_match_all.go @@ -0,0 +1,123 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "reflect" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeMatchAllSearcher int + +func init() { + var mas MatchAllSearcher + reflectStaticSizeMatchAllSearcher = int(reflect.TypeOf(mas).Size()) +} + +type MatchAllSearcher struct { + indexReader index.IndexReader + reader index.DocIDReader + scorer *scorer.ConstantScorer + count uint64 +} + +func NewMatchAllSearcher(ctx context.Context, indexReader index.IndexReader, boost float64, options search.SearcherOptions) (*MatchAllSearcher, error) { + reader, err := indexReader.DocIDReaderAll() + if err != nil { + return nil, err + } + count, err := indexReader.DocCount() + if err != nil { + _ = reader.Close() + return nil, err + } + scorer := scorer.NewConstantScorer(1.0, boost, options) + + return &MatchAllSearcher{ + indexReader: indexReader, + reader: reader, + scorer: scorer, + count: count, + }, nil +} + +func (s *MatchAllSearcher) Size() int { + return reflectStaticSizeMatchAllSearcher + size.SizeOfPtr + + s.reader.Size() + + s.scorer.Size() +} + +func (s *MatchAllSearcher) Count() uint64 { + return s.count +} + +func (s *MatchAllSearcher) Weight() float64 { + return s.scorer.Weight() +} + +func (s *MatchAllSearcher) SetQueryNorm(qnorm float64) { + s.scorer.SetQueryNorm(qnorm) +} + +func (s *MatchAllSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + id, err := s.reader.Next() + if err != nil { + return nil, err + } + + if id == nil { + return nil, nil + } + + // score match + docMatch := s.scorer.Score(ctx, id) + // return doc match + return docMatch, nil + +} + +func (s *MatchAllSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + id, err := s.reader.Advance(ID) + if err != nil { + return nil, err + } + + if id == nil { + return nil, nil + } + + // score match + docMatch := s.scorer.Score(ctx, id) + + // return doc match + return docMatch, nil +} + +func (s *MatchAllSearcher) Close() error { + return s.reader.Close() +} + +func (s *MatchAllSearcher) Min() int { + return 0 +} + +func (s *MatchAllSearcher) DocumentMatchPoolSize() int { + return 1 +} diff --git a/search/searcher/search_match_all_test.go b/search/searcher/search_match_all_test.go new file mode 100644 index 0000000..f3b3d88 --- /dev/null +++ b/search/searcher/search_match_all_test.go @@ -0,0 +1,146 @@ +// Copyright (c) 2013 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "testing" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func TestMatchAllSearch(t *testing.T) { + twoDocIndexReader, err := twoDocIndex.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := twoDocIndexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + explainTrue := search.SearcherOptions{Explain: true} + + allSearcher, err := NewMatchAllSearcher(context.TODO(), twoDocIndexReader, 1.0, explainTrue) + if err != nil { + t.Fatal(err) + } + + allSearcher2, err := NewMatchAllSearcher(context.TODO(), twoDocIndexReader, 1.2, explainTrue) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + searcher search.Searcher + queryNorm float64 + results []*search.DocumentMatch + }{ + { + searcher: allSearcher, + queryNorm: 1.0, + results: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("1"), + Score: 1.0, + }, + { + IndexInternalID: index.IndexInternalID("2"), + Score: 1.0, + }, + { + IndexInternalID: index.IndexInternalID("3"), + Score: 1.0, + }, + { + IndexInternalID: index.IndexInternalID("4"), + Score: 1.0, + }, + { + IndexInternalID: index.IndexInternalID("5"), + Score: 1.0, + }, + }, + }, + { + searcher: allSearcher2, + queryNorm: 0.8333333, + results: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("1"), + Score: 1.0, + }, + { + IndexInternalID: index.IndexInternalID("2"), + Score: 1.0, + }, + { + IndexInternalID: index.IndexInternalID("3"), + Score: 1.0, + }, + { + IndexInternalID: index.IndexInternalID("4"), + Score: 1.0, + }, + { + IndexInternalID: index.IndexInternalID("5"), + Score: 1.0, + }, + }, + }, + } + + for testIndex, test := range tests { + + if test.queryNorm != 1.0 { + test.searcher.SetQueryNorm(test.queryNorm) + } + defer func() { + err := test.searcher.Close() + if err != nil { + t.Fatal(err) + } + }() + + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0), + } + next, err := test.searcher.Next(ctx) + i := 0 + for err == nil && next != nil { + if i < len(test.results) { + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) + } + if !scoresCloseEnough(next.Score, test.results[i].Score) { + t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) + t.Logf("scoring explanation: %s", next.Expl) + } + } + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) + i++ + } + if err != nil { + t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) + } + if len(test.results) != i { + t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) + } + } +} diff --git a/search/searcher/search_match_none.go b/search/searcher/search_match_none.go new file mode 100644 index 0000000..b7f7694 --- /dev/null +++ b/search/searcher/search_match_none.go @@ -0,0 +1,76 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "reflect" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeMatchNoneSearcher int + +func init() { + var mns MatchNoneSearcher + reflectStaticSizeMatchNoneSearcher = int(reflect.TypeOf(mns).Size()) +} + +type MatchNoneSearcher struct { + indexReader index.IndexReader +} + +func NewMatchNoneSearcher(indexReader index.IndexReader) (*MatchNoneSearcher, error) { + return &MatchNoneSearcher{ + indexReader: indexReader, + }, nil +} + +func (s *MatchNoneSearcher) Size() int { + return reflectStaticSizeMatchNoneSearcher + size.SizeOfPtr +} + +func (s *MatchNoneSearcher) Count() uint64 { + return uint64(0) +} + +func (s *MatchNoneSearcher) Weight() float64 { + return 0.0 +} + +func (s *MatchNoneSearcher) SetQueryNorm(qnorm float64) { + +} + +func (s *MatchNoneSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + return nil, nil +} + +func (s *MatchNoneSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + return nil, nil +} + +func (s *MatchNoneSearcher) Close() error { + return nil +} + +func (s *MatchNoneSearcher) Min() int { + return 0 +} + +func (s *MatchNoneSearcher) DocumentMatchPoolSize() int { + return 0 +} diff --git a/search/searcher/search_match_none_test.go b/search/searcher/search_match_none_test.go new file mode 100644 index 0000000..c3a370d --- /dev/null +++ b/search/searcher/search_match_none_test.go @@ -0,0 +1,85 @@ +// Copyright (c) 2013 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "testing" + + "github.com/blevesearch/bleve/v2/search" +) + +func TestMatchNoneSearch(t *testing.T) { + + twoDocIndexReader, err := twoDocIndex.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := twoDocIndexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + noneSearcher, err := NewMatchNoneSearcher(twoDocIndexReader) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + searcher search.Searcher + results []*search.DocumentMatch + }{ + { + searcher: noneSearcher, + results: []*search.DocumentMatch{}, + }, + } + + for testIndex, test := range tests { + defer func() { + err := test.searcher.Close() + if err != nil { + t.Fatal(err) + } + }() + + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0), + } + next, err := test.searcher.Next(ctx) + i := 0 + for err == nil && next != nil { + if i < len(test.results) { + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) + } + if !scoresCloseEnough(next.Score, test.results[i].Score) { + t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) + t.Logf("scoring explanation: %s", next.Expl) + } + } + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) + i++ + } + if err != nil { + t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) + } + if len(test.results) != i { + t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) + } + } +} diff --git a/search/searcher/search_multi_term.go b/search/searcher/search_multi_term.go new file mode 100644 index 0000000..98f8f92 --- /dev/null +++ b/search/searcher/search_multi_term.go @@ -0,0 +1,268 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "fmt" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func NewMultiTermSearcher(ctx context.Context, indexReader index.IndexReader, terms []string, + field string, boost float64, options search.SearcherOptions, limit bool) ( + search.Searcher, error) { + + if tooManyClauses(len(terms)) { + if optionsDisjunctionOptimizable(options) { + return optimizeMultiTermSearcher(ctx, indexReader, terms, field, boost, options) + } + if limit { + return nil, tooManyClausesErr(field, len(terms)) + } + } + + qsearchers, err := makeBatchSearchers(ctx, indexReader, terms, field, boost, options) + if err != nil { + return nil, err + } + + // build disjunction searcher of these ranges + return newMultiTermSearcherInternal(ctx, indexReader, qsearchers, field, boost, + options, limit) +} + +// Works similarly to the multi term searcher but additionally boosts individual terms based on +// their edit distance from the query terms +func NewMultiTermSearcherBoosted(ctx context.Context, indexReader index.IndexReader, terms []string, + field string, boost float64, editDistances []uint8, options search.SearcherOptions, limit bool) ( + search.Searcher, error) { + + if tooManyClauses(len(terms)) { + if optionsDisjunctionOptimizable(options) { + return optimizeMultiTermSearcher(ctx, indexReader, terms, field, boost, options) + } + if limit { + return nil, tooManyClausesErr(field, len(terms)) + } + } + + qsearchers, err := makeBatchSearchersBoosted(ctx, indexReader, terms, field, boost, editDistances, options) + if err != nil { + return nil, err + } + + // build disjunction searcher of these ranges + return newMultiTermSearcherInternal(ctx, indexReader, qsearchers, field, boost, + options, limit) +} + +func NewMultiTermSearcherBytes(ctx context.Context, indexReader index.IndexReader, terms [][]byte, + field string, boost float64, options search.SearcherOptions, limit bool) ( + search.Searcher, error) { + + if tooManyClauses(len(terms)) { + if optionsDisjunctionOptimizable(options) { + return optimizeMultiTermSearcherBytes(ctx, indexReader, terms, field, boost, options) + } + + if limit { + return nil, tooManyClausesErr(field, len(terms)) + } + } + + qsearchers, err := makeBatchSearchersBytes(ctx, indexReader, terms, field, boost, options) + if err != nil { + return nil, err + } + + // build disjunction searcher of these ranges + return newMultiTermSearcherInternal(ctx, indexReader, qsearchers, field, boost, + options, limit) +} + +func newMultiTermSearcherInternal(ctx context.Context, indexReader index.IndexReader, + searchers []search.Searcher, field string, boost float64, + options search.SearcherOptions, limit bool) ( + search.Searcher, error) { + + // build disjunction searcher of these ranges + searcher, err := newDisjunctionSearcher(ctx, indexReader, searchers, 0, options, + limit) + if err != nil { + for _, s := range searchers { + _ = s.Close() + } + return nil, err + } + + return searcher, nil +} + +func optimizeMultiTermSearcher(ctx context.Context, indexReader index.IndexReader, terms []string, + field string, boost float64, options search.SearcherOptions) ( + search.Searcher, error) { + var finalSearcher search.Searcher + for len(terms) > 0 { + var batchTerms []string + if len(terms) > DisjunctionMaxClauseCount { + batchTerms = terms[:DisjunctionMaxClauseCount] + terms = terms[DisjunctionMaxClauseCount:] + } else { + batchTerms = terms + terms = nil + } + batch, err := makeBatchSearchers(ctx, indexReader, batchTerms, field, boost, options) + if err != nil { + return nil, err + } + if finalSearcher != nil { + batch = append(batch, finalSearcher) + } + cleanup := func() { + for _, searcher := range batch { + if searcher != nil { + _ = searcher.Close() + } + } + } + finalSearcher, err = optimizeCompositeSearcher(ctx, "disjunction:unadorned", + indexReader, batch, options) + // all searchers in batch should be closed, regardless of error or optimization failure + // either we're returning, or continuing and only finalSearcher is needed for next loop + cleanup() + if err != nil { + return nil, err + } + if finalSearcher == nil { + return nil, fmt.Errorf("unable to optimize") + } + } + return finalSearcher, nil +} + +func makeBatchSearchers(ctx context.Context, indexReader index.IndexReader, terms []string, field string, + boost float64, options search.SearcherOptions) ([]search.Searcher, error) { + + qsearchers := make([]search.Searcher, len(terms)) + qsearchersClose := func() { + for _, searcher := range qsearchers { + if searcher != nil { + _ = searcher.Close() + } + } + } + for i, term := range terms { + var err error + qsearchers[i], err = NewTermSearcher(ctx, indexReader, term, field, boost, options) + if err != nil { + qsearchersClose() + return nil, err + } + } + return qsearchers, nil +} + +func makeBatchSearchersBoosted(ctx context.Context, indexReader index.IndexReader, terms []string, field string, + boost float64, editDistances []uint8, options search.SearcherOptions) ([]search.Searcher, error) { + + qsearchers := make([]search.Searcher, len(terms)) + qsearchersClose := func() { + for _, searcher := range qsearchers { + if searcher != nil { + _ = searcher.Close() + } + } + } + for i, term := range terms { + var err error + var editMultiplier float64 + if editDistances != nil { + editMultiplier = 1 / float64(editDistances[i]+1) + } + qsearchers[i], err = NewTermSearcher(ctx, indexReader, term, field, boost*editMultiplier, options) + if err != nil { + qsearchersClose() + return nil, err + } + } + return qsearchers, nil +} + +func optimizeMultiTermSearcherBytes(ctx context.Context, indexReader index.IndexReader, terms [][]byte, + field string, boost float64, options search.SearcherOptions) ( + search.Searcher, error) { + + var finalSearcher search.Searcher + for len(terms) > 0 { + var batchTerms [][]byte + if len(terms) > DisjunctionMaxClauseCount { + batchTerms = terms[:DisjunctionMaxClauseCount] + terms = terms[DisjunctionMaxClauseCount:] + } else { + batchTerms = terms + terms = nil + } + batch, err := makeBatchSearchersBytes(ctx, indexReader, batchTerms, field, boost, options) + if err != nil { + return nil, err + } + if finalSearcher != nil { + batch = append(batch, finalSearcher) + } + cleanup := func() { + for _, searcher := range batch { + if searcher != nil { + _ = searcher.Close() + } + } + } + finalSearcher, err = optimizeCompositeSearcher(ctx, "disjunction:unadorned", + indexReader, batch, options) + // all searchers in batch should be closed, regardless of error or optimization failure + // either we're returning, or continuing and only finalSearcher is needed for next loop + cleanup() + if err != nil { + return nil, err + } + if finalSearcher == nil { + return nil, fmt.Errorf("unable to optimize") + } + } + return finalSearcher, nil +} + +func makeBatchSearchersBytes(ctx context.Context, indexReader index.IndexReader, terms [][]byte, field string, + boost float64, options search.SearcherOptions) ([]search.Searcher, error) { + + qsearchers := make([]search.Searcher, len(terms)) + qsearchersClose := func() { + for _, searcher := range qsearchers { + if searcher != nil { + _ = searcher.Close() + } + } + } + for i, term := range terms { + var err error + qsearchers[i], err = NewTermSearcherBytes(ctx, indexReader, term, field, boost, options) + if err != nil { + qsearchersClose() + return nil, err + } + } + return qsearchers, nil +} diff --git a/search/searcher/search_multi_term_test.go b/search/searcher/search_multi_term_test.go new file mode 100644 index 0000000..7368c9b --- /dev/null +++ b/search/searcher/search_multi_term_test.go @@ -0,0 +1,101 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "testing" + + "github.com/blevesearch/bleve/v2/search" +) + +// ctxWithTermSearcherLimit caps the number of leaf term searchers, giving us a +// deterministic way to make term searcher construction fail on purpose. +func ctxWithTermSearcherLimit(limit int) context.Context { + ctx := context.WithValue(context.Background(), search.MaxTermSearchersKey, limit) + return search.ContextWithTermSearchersCounter(ctx) +} + +// A term searcher that fails to build must come back as a nil search.Searcher +// interface, never as an interface holding a nil *TermSearcher. Callers guard +// their cleanup with `if searcher != nil`, and a typed nil slips past that guard +// and panics on a nil receiver in (*TermSearcher).Close. +func TestTermSearcherErrorReturnsNilSearcher(t *testing.T) { + twoDocIndexReader, err := twoDocIndex.Reader() + if err != nil { + t.Fatal(err) + } + defer func() { + if err := twoDocIndexReader.Close(); err != nil { + t.Fatal(err) + } + }() + + // Room for exactly one leaf term searcher, so the second one fails inside + // newTermSearcherFromReader. + ctx := ctxWithTermSearcherLimit(1) + + first, err := NewTermSearcher(ctx, twoDocIndexReader, "beer", "desc", 1.0, + search.SearcherOptions{}) + if err != nil { + t.Fatalf("first term searcher should be within the limit, got: %v", err) + } + defer func() { + if err := first.Close(); err != nil { + t.Fatal(err) + } + }() + + second, err := NewTermSearcher(ctx, twoDocIndexReader, "angst", "desc", 1.0, + search.SearcherOptions{}) + if err == nil { + _ = second.Close() + t.Fatal("second term searcher should have exceeded the limit of 1") + } + if second != nil { + t.Fatalf("expected a nil search.Searcher alongside the error, got an "+ + "interface holding %T (typed nil: %v)", second, second == search.Searcher(nil)) + } +} + +// Regression test for the panic in makeBatchSearchers' cleanup: when one term in +// the batch fails to build, closing the already-built searchers must not trip +// over the failed entry. +func TestMultiTermSearcherCleanupAfterError(t *testing.T) { + twoDocIndexReader, err := twoDocIndex.Reader() + if err != nil { + t.Fatal(err) + } + defer func() { + if err := twoDocIndexReader.Close(); err != nil { + t.Fatal(err) + } + }() + + // Two of the three terms build; the third trips the limit, sending + // makeBatchSearchers down its qsearchersClose path. + ctx := ctxWithTermSearcherLimit(2) + terms := []string{"beer", "angst", "couch"} + + searcher, err := NewMultiTermSearcher(ctx, twoDocIndexReader, terms, "desc", + 1.0, search.SearcherOptions{}, true) + if err == nil { + _ = searcher.Close() + t.Fatalf("%d terms should have exceeded the limit of 2", len(terms)) + } + if searcher != nil { + t.Fatalf("expected a nil search.Searcher alongside the error, got %T", searcher) + } +} diff --git a/search/searcher/search_numeric_range.go b/search/searcher/search_numeric_range.go new file mode 100644 index 0000000..cd8f007 --- /dev/null +++ b/search/searcher/search_numeric_range.go @@ -0,0 +1,258 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "bytes" + "context" + "math" + "sort" + + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func NewNumericRangeSearcher(ctx context.Context, indexReader index.IndexReader, + min *float64, max *float64, inclusiveMin, inclusiveMax *bool, field string, + boost float64, options search.SearcherOptions) (search.Searcher, error) { + // account for unbounded edges + if min == nil { + negInf := math.Inf(-1) + min = &negInf + } + if max == nil { + Inf := math.Inf(1) + max = &Inf + } + if inclusiveMin == nil { + defaultInclusiveMin := true + inclusiveMin = &defaultInclusiveMin + } + if inclusiveMax == nil { + defaultInclusiveMax := false + inclusiveMax = &defaultInclusiveMax + } + // find all the ranges + minInt64 := numeric.Float64ToInt64(*min) + if !*inclusiveMin && minInt64 != math.MaxInt64 { + minInt64++ + } + maxInt64 := numeric.Float64ToInt64(*max) + if !*inclusiveMax && maxInt64 != math.MinInt64 { + maxInt64-- + } + + var fieldDict index.FieldDictContains + var dictBytesRead uint64 + var isIndexed filterFunc + var err error + if irr, ok := indexReader.(index.IndexReaderContains); ok { + fieldDict, err = irr.FieldDictContains(field) + if err != nil { + return nil, err + } + + isIndexed = func(term []byte) bool { + found, err := fieldDict.Contains(term) + return err == nil && found + } + + dictBytesRead = fieldDict.BytesRead() + } + + // FIXME hard-coded precision, should match field declaration + termRanges := splitInt64Range(minInt64, maxInt64, 4) + terms := termRanges.Enumerate(isIndexed) + if fieldDict != nil { + if fd, ok := fieldDict.(index.FieldDict); ok { + if err = fd.Close(); err != nil { + return nil, err + } + } + } + + if len(terms) < 1 { + // reporting back the IO stats with respect to the dictionary + // loaded, using the context + if ctx != nil { + reportIOStats(ctx, dictBytesRead) + search.RecordSearchCost(ctx, search.AddM, dictBytesRead) + } + + // cannot return MatchNoneSearcher because of interaction with + // commit f391b991c20f02681bacd197afc6d8aed444e132 + return NewMultiTermSearcherBytes(ctx, indexReader, terms, field, + boost, options, true) + } + + // for upside_down + if isIndexed == nil { + terms, err = filterCandidateTerms(indexReader, terms, field) + if err != nil { + return nil, err + } + } + + if tooManyClauses(len(terms)) { + return nil, tooManyClausesErr(field, len(terms)) + } + + if ctx != nil { + reportIOStats(ctx, dictBytesRead) + search.RecordSearchCost(ctx, search.AddM, dictBytesRead) + } + + return NewMultiTermSearcherBytes(ctx, indexReader, terms, field, + boost, options, true) +} + +func filterCandidateTerms(indexReader index.IndexReader, + terms [][]byte, field string) (rv [][]byte, err error) { + + fieldDict, err := indexReader.FieldDictRange(field, terms[0], terms[len(terms)-1]) + if err != nil { + return nil, err + } + + // enumerate the terms and check against list of terms + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + termBytes := []byte(tfd.Term) + i := sort.Search(len(terms), func(i int) bool { return bytes.Compare(terms[i], termBytes) >= 0 }) + if i < len(terms) && bytes.Equal(terms[i], termBytes) { + rv = append(rv, terms[i]) + } + terms = terms[i:] + tfd, err = fieldDict.Next() + } + + if cerr := fieldDict.Close(); cerr != nil && err == nil { + err = cerr + } + + return rv, err +} + +type termRange struct { + startTerm []byte + endTerm []byte +} + +func (t *termRange) Enumerate(filter filterFunc) [][]byte { + var rv [][]byte + next := t.startTerm + for bytes.Compare(next, t.endTerm) <= 0 { + if filter != nil { + if filter(next) { + rv = append(rv, next) + } + } else { + rv = append(rv, next) + } + next = incrementBytes(next) + } + return rv +} + +func incrementBytes(in []byte) []byte { + rv := make([]byte, len(in)) + copy(rv, in) + for i := len(rv) - 1; i >= 0; i-- { + rv[i] = rv[i] + 1 + if rv[i] != 0 { + // didn't overflow, so stop + break + } + } + return rv +} + +type termRanges []*termRange + +func (tr termRanges) Enumerate(filter filterFunc) [][]byte { + var rv [][]byte + for _, tri := range tr { + trie := tri.Enumerate(filter) + rv = append(rv, trie...) + } + return rv +} + +func splitInt64Range(minBound, maxBound int64, precisionStep uint) termRanges { + rv := make(termRanges, 0) + if minBound > maxBound { + return rv + } + + for shift := uint(0); ; shift += precisionStep { + + diff := int64(1) << (shift + precisionStep) + mask := ((int64(1) << precisionStep) - int64(1)) << shift + hasLower := (minBound & mask) != int64(0) + hasUpper := (maxBound & mask) != mask + + var nextMinBound int64 + if hasLower { + nextMinBound = (minBound + diff) &^ mask + } else { + nextMinBound = minBound &^ mask + } + var nextMaxBound int64 + if hasUpper { + nextMaxBound = (maxBound - diff) &^ mask + } else { + nextMaxBound = maxBound &^ mask + } + + lowerWrapped := nextMinBound < minBound + upperWrapped := nextMaxBound > maxBound + + if shift+precisionStep >= 64 || nextMinBound > nextMaxBound || + lowerWrapped || upperWrapped { + // We are in the lowest precision or the next precision is not available. + rv = append(rv, newRange(minBound, maxBound, shift)) + // exit the split recursion loop + break + } + + if hasLower { + rv = append(rv, newRange(minBound, minBound|mask, shift)) + } + if hasUpper { + rv = append(rv, newRange(maxBound&^mask, maxBound, shift)) + } + + // recurse to next precision + minBound = nextMinBound + maxBound = nextMaxBound + } + + return rv +} + +func newRange(minBound, maxBound int64, shift uint) *termRange { + maxBound |= (int64(1) << shift) - int64(1) + minBytes := numeric.MustNewPrefixCodedInt64(minBound, shift) + maxBytes := numeric.MustNewPrefixCodedInt64(maxBound, shift) + return newRangeBytes(minBytes, maxBytes) +} + +func newRangeBytes(minBytes, maxBytes []byte) *termRange { + return &termRange{ + startTerm: minBytes, + endTerm: maxBytes, + } +} diff --git a/search/searcher/search_numeric_range_test.go b/search/searcher/search_numeric_range_test.go new file mode 100644 index 0000000..83611cd --- /dev/null +++ b/search/searcher/search_numeric_range_test.go @@ -0,0 +1,60 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/numeric" +) + +func TestSplitRange(t *testing.T) { + min := numeric.Float64ToInt64(1.0) + max := numeric.Float64ToInt64(5.0) + ranges := splitInt64Range(min, max, 4) + enumerated := ranges.Enumerate(nil) + if len(enumerated) != 135 { + t.Errorf("expected 135 terms, got %d", len(enumerated)) + } + +} + +func TestIncrementBytes(t *testing.T) { + tests := []struct { + in []byte + out []byte + }{ + { + in: []byte{0}, + out: []byte{1}, + }, + { + in: []byte{0, 0}, + out: []byte{0, 1}, + }, + { + in: []byte{0, 255}, + out: []byte{1, 0}, + }, + } + + for _, test := range tests { + actual := incrementBytes(test.in) + if !reflect.DeepEqual(actual, test.out) { + t.Errorf("expected %#v, got %#v", test.out, actual) + } + } +} diff --git a/search/searcher/search_phrase.go b/search/searcher/search_phrase.go new file mode 100644 index 0000000..6cd0342 --- /dev/null +++ b/search/searcher/search_phrase.go @@ -0,0 +1,554 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "fmt" + "math" + "reflect" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizePhraseSearcher int + +func init() { + var ps PhraseSearcher + reflectStaticSizePhraseSearcher = int(reflect.TypeOf(ps).Size()) +} + +type PhraseSearcher struct { + mustSearcher search.Searcher + queryNorm float64 + currMust *search.DocumentMatch + terms [][]string + path phrasePath + paths []phrasePath + locations []search.Location + initialized bool + // map a term to a list of fuzzy terms that match it + fuzzyTermMatches map[string][]string +} + +func (s *PhraseSearcher) Size() int { + sizeInBytes := reflectStaticSizePhraseSearcher + size.SizeOfPtr + + if s.mustSearcher != nil { + sizeInBytes += s.mustSearcher.Size() + } + + if s.currMust != nil { + sizeInBytes += s.currMust.Size() + } + + for _, entry := range s.terms { + sizeInBytes += size.SizeOfSlice + for _, entry1 := range entry { + sizeInBytes += size.SizeOfString + len(entry1) + } + } + + return sizeInBytes +} + +func NewPhraseSearcher(ctx context.Context, indexReader index.IndexReader, terms []string, + fuzziness int, autoFuzzy bool, field string, boost float64, options search.SearcherOptions) (*PhraseSearcher, error) { + + // turn flat terms []string into [][]string + mterms := make([][]string, len(terms)) + for i, term := range terms { + mterms[i] = []string{term} + } + return NewMultiPhraseSearcher(ctx, indexReader, mterms, fuzziness, autoFuzzy, field, boost, options) +} + +func NewMultiPhraseSearcher(ctx context.Context, indexReader index.IndexReader, terms [][]string, + fuzziness int, autoFuzzy bool, field string, boost float64, options search.SearcherOptions) (*PhraseSearcher, error) { + + options.IncludeTermVectors = true + var termPositionSearchers []search.Searcher + var err error + var ts search.Searcher + // The following logic checks if fuzziness is enabled. + // Fuzziness is considered enabled if either: + // a. `fuzziness` is greater than 0, or + // b. `autoFuzzy` is set to true. + // if both conditions are true, `autoFuzzy` takes precedence. + // If enabled, a map will be created to store the matches for fuzzy terms. + fuzzinessEnabled := autoFuzzy || fuzziness > 0 + var fuzzyTermMatches map[string][]string + if fuzzinessEnabled { + fuzzyTermMatches = make(map[string][]string) + ctx = context.WithValue(ctx, search.FuzzyMatchPhraseKey, fuzzyTermMatches) + } + // in case of fuzzy multi-phrase, phrase and match-phrase queries we hardcode the + // prefix length to 0, as setting a per word matching prefix length would not + // make sense from a user perspective. + for _, termPos := range terms { + if len(termPos) == 1 && termPos[0] != "" { + // single term + if fuzzinessEnabled { + // fuzzy + if autoFuzzy { + // auto fuzzy + ts, err = NewAutoFuzzySearcher(ctx, indexReader, termPos[0], 0, field, boost, options) + } else { + // non-auto fuzzy + ts, err = NewFuzzySearcher(ctx, indexReader, termPos[0], 0, fuzziness, field, boost, options) + } + } else { + // non-fuzzy + ts, err = NewTermSearcher(ctx, indexReader, termPos[0], field, boost, options) + } + if err != nil { + // close any searchers already opened + for _, ts := range termPositionSearchers { + _ = ts.Close() + } + return nil, fmt.Errorf("phrase searcher error building term searcher: %v", err) + } + termPositionSearchers = append(termPositionSearchers, ts) + } else if len(termPos) > 1 { + // multiple terms + var termSearchers []search.Searcher + for _, term := range termPos { + if term == "" { + continue + } + if fuzzinessEnabled { + // fuzzy + if autoFuzzy { + // auto fuzzy + ts, err = NewAutoFuzzySearcher(ctx, indexReader, term, 0, field, boost, options) + } else { + // non-auto fuzzy + ts, err = NewFuzzySearcher(ctx, indexReader, term, 0, fuzziness, field, boost, options) + } + } else { + // non-fuzzy + ts, err = NewTermSearcher(ctx, indexReader, term, field, boost, options) + } + if err != nil { + // close any searchers already opened + for _, ts := range termPositionSearchers { + _ = ts.Close() + } + return nil, fmt.Errorf("phrase searcher error building term searcher: %v", err) + } + termSearchers = append(termSearchers, ts) + } + disjunction, err := NewDisjunctionSearcher(ctx, indexReader, termSearchers, 1, options) + if err != nil { + // close any searchers already opened + for _, ts := range termPositionSearchers { + _ = ts.Close() + } + return nil, fmt.Errorf("phrase searcher error building term position disjunction searcher: %v", err) + } + termPositionSearchers = append(termPositionSearchers, disjunction) + } + } + + if ctx != nil { + if fts, ok := ctx.Value(search.FieldTermSynonymMapKey).(search.FieldTermSynonymMap); ok { + if ts, exists := fts[field]; exists { + if fuzzinessEnabled { + for term, fuzzyTerms := range fuzzyTermMatches { + fuzzySynonymTerms := make([]string, 0, len(fuzzyTerms)) + if s, found := ts[term]; found { + fuzzySynonymTerms = append(fuzzySynonymTerms, s...) + } + for _, fuzzyTerm := range fuzzyTerms { + if fuzzyTerm == term { + continue + } + if s, found := ts[fuzzyTerm]; found { + fuzzySynonymTerms = append(fuzzySynonymTerms, s...) + } + } + if len(fuzzySynonymTerms) > 0 { + fuzzyTermMatches[term] = append(fuzzyTermMatches[term], fuzzySynonymTerms...) + } + } + } else { + for _, termPos := range terms { + for _, term := range termPos { + if s, found := ts[term]; found { + if fuzzyTermMatches == nil { + fuzzyTermMatches = make(map[string][]string) + } + fuzzyTermMatches[term] = s + } + } + } + } + } + } + } + mustSearcher, err := NewConjunctionSearcher(ctx, indexReader, termPositionSearchers, options) + if err != nil { + // close any searchers already opened + for _, ts := range termPositionSearchers { + _ = ts.Close() + } + return nil, fmt.Errorf("phrase searcher error building conjunction searcher: %v", err) + } + + // build our searcher + rv := PhraseSearcher{ + mustSearcher: mustSearcher, + terms: terms, + fuzzyTermMatches: fuzzyTermMatches, + } + rv.computeQueryNorm() + return &rv, nil +} + +func (s *PhraseSearcher) computeQueryNorm() { + // first calculate sum of squared weights + sumOfSquaredWeights := 0.0 + if s.mustSearcher != nil { + sumOfSquaredWeights += s.mustSearcher.Weight() + } + + // now compute query norm from this + s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) + // finally tell all the downstream searchers the norm + if s.mustSearcher != nil { + s.mustSearcher.SetQueryNorm(s.queryNorm) + } +} + +func (s *PhraseSearcher) initSearchers(ctx *search.SearchContext) error { + err := s.advanceNextMust(ctx) + if err != nil { + return err + } + + s.initialized = true + return nil +} + +func (s *PhraseSearcher) advanceNextMust(ctx *search.SearchContext) error { + var err error + + if s.mustSearcher != nil { + if s.currMust != nil { + ctx.DocumentMatchPool.Put(s.currMust) + } + s.currMust, err = s.mustSearcher.Next(ctx) + if err != nil { + return err + } + } + + return nil +} + +func (s *PhraseSearcher) Weight() float64 { + return s.mustSearcher.Weight() +} + +func (s *PhraseSearcher) SetQueryNorm(qnorm float64) { + s.mustSearcher.SetQueryNorm(qnorm) +} + +func (s *PhraseSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + + for s.currMust != nil { + // check this match against phrase constraints + rv := s.checkCurrMustMatch(ctx) + + // prepare for next iteration (either loop or subsequent call to Next()) + err := s.advanceNextMust(ctx) + if err != nil { + return nil, err + } + + // if match satisfied phrase constraints return it as a hit + if rv != nil { + return rv, nil + } + } + + return nil, nil +} + +// checkCurrMustMatch is solely concerned with determining if the DocumentMatch +// pointed to by s.currMust (which satisfies the pre-condition searcher) +// also satisfies the phrase constraints. if so, it returns a DocumentMatch +// for this document, otherwise nil +func (s *PhraseSearcher) checkCurrMustMatch(ctx *search.SearchContext) *search.DocumentMatch { + s.locations = s.currMust.Complete(s.locations) + + locations := s.currMust.Locations + s.currMust.Locations = nil + + ftls := s.currMust.FieldTermLocations + + // typically we would expect there to only actually be results in + // one field, but we allow for this to not be the case + // but, we note that phrase constraints can only be satisfied within + // a single field, so we can check them each independently + for field, tlm := range locations { + ftls = s.checkCurrMustMatchField(ctx, field, tlm, ftls) + } + + if len(ftls) > 0 { + // return match + rv := s.currMust + s.currMust = nil + rv.FieldTermLocations = ftls + return rv + } + + return nil +} + +// checkCurrMustMatchField is solely concerned with determining if one +// particular field within the currMust DocumentMatch Locations +// satisfies the phrase constraints (possibly more than once). if so, +// the matching field term locations are appended to the provided +// slice +func (s *PhraseSearcher) checkCurrMustMatchField(ctx *search.SearchContext, + field string, tlm search.TermLocationMap, + ftls []search.FieldTermLocation) []search.FieldTermLocation { + if s.path == nil { + s.path = make(phrasePath, 0, len(s.terms)) + } + var tlmPtr *search.TermLocationMap = &tlm + if s.fuzzyTermMatches != nil { + // if fuzzy search, we need to expand the tlm to include all the fuzzy matches + // Example - term is "foo" and fuzzy matches are "foo", "fool", "food" + // the non expanded tlm will be: + // foo -> Locations[foo] + // fool -> Locations[fool] + // food -> Locations[food] + // the expanded tlm will be: + // foo -> [Locations[foo], Locations[fool], Locations[food]] + expandedTlm := make(search.TermLocationMap) + s.expandFuzzyMatches(tlm, expandedTlm) + tlmPtr = &expandedTlm + } + s.paths = findPhrasePaths(0, nil, s.terms, *tlmPtr, s.path[:0], 0, s.paths[:0]) + for _, p := range s.paths { + for _, pp := range p { + ftls = append(ftls, search.FieldTermLocation{ + Field: field, + Term: pp.term, + Location: search.Location{ + Pos: pp.loc.Pos, + Start: pp.loc.Start, + End: pp.loc.End, + ArrayPositions: pp.loc.ArrayPositions, + }, + }) + } + } + return ftls +} + +func (s *PhraseSearcher) expandFuzzyMatches(tlm search.TermLocationMap, expandedTlm search.TermLocationMap) { + for term, fuzzyMatches := range s.fuzzyTermMatches { + locations := tlm[term] + for _, fuzzyMatch := range fuzzyMatches { + if fuzzyMatch == term { + continue + } + locations = append(locations, tlm[fuzzyMatch]...) + } + expandedTlm[term] = locations + } +} + +type phrasePart struct { + term string + loc *search.Location +} + +func (p *phrasePart) String() string { + return fmt.Sprintf("[%s %v]", p.term, p.loc) +} + +type phrasePath []phrasePart + +func (p phrasePath) MergeInto(in search.TermLocationMap) { + for _, pp := range p { + in[pp.term] = append(in[pp.term], pp.loc) + } +} + +func (p phrasePath) String() string { + rv := "[" + for i, pp := range p { + if i > 0 { + rv += ", " + } + rv += pp.String() + } + rv += "]" + return rv +} + +// findPhrasePaths is a function to identify phrase matches from a set +// of known term locations. it recursive so care must be taken with +// arguments and return values. +// +// prevPos - the previous location, 0 on first invocation +// +// ap - array positions of the first candidate phrase part to +// which further recursive phrase parts must match, +// nil on initial invocation or when there are no array positions +// +// phraseTerms - slice containing the phrase terms, +// may contain empty string as placeholder (don't care) +// +// tlm - the Term Location Map containing all relevant term locations +// +// p - the current path being explored (appended to in recursive calls) +// this is the primary state being built during the traversal +// +// remainingSlop - amount of sloppiness that's allowed, which is the +// sum of the editDistances from each matching phrase part, where 0 means no +// sloppiness allowed (all editDistances must be 0), decremented during recursion +// +// rv - the final result being appended to by all the recursive calls +// +// returns slice of paths, or nil if invocation did not find any successful paths +func findPhrasePaths(prevPos uint64, ap search.ArrayPositions, phraseTerms [][]string, + tlm search.TermLocationMap, p phrasePath, remainingSlop int, rv []phrasePath) []phrasePath { + // no more terms + if len(phraseTerms) < 1 { + // snapshot or copy the recursively built phrasePath p and + // append it to the rv, also optimizing by checking if next + // phrasePath item in the rv (which we're about to overwrite) + // is available for reuse + var pcopy phrasePath + if len(rv) < cap(rv) { + pcopy = rv[:len(rv)+1][len(rv)][:0] + } + return append(rv, append(pcopy, p...)) + } + + car := phraseTerms[0] + cdr := phraseTerms[1:] + + // empty term is treated as match (continue) + if len(car) == 0 || (len(car) == 1 && car[0] == "") { + nextPos := prevPos + 1 + if prevPos == 0 { + // if prevPos was 0, don't set it to 1 (as that's not a real abs pos) + nextPos = 0 // don't advance nextPos if prevPos was 0 + } + return findPhrasePaths(nextPos, ap, cdr, tlm, p, remainingSlop, rv) + } + + // locations for this term + for _, carTerm := range car { + locations := tlm[carTerm] + LOCATIONS_LOOP: + for _, loc := range locations { + if prevPos != 0 && !loc.ArrayPositions.Equals(ap) { + // if the array positions are wrong, can't match, try next location + continue + } + + // compute distance from previous phrase term + dist := 0 + if prevPos != 0 { + dist = editDistance(prevPos+1, loc.Pos) + } + + // if enough slop remaining, continue recursively + if prevPos == 0 || (remainingSlop-dist) >= 0 { + // skip if we've already used this term+loc already + for _, ppart := range p { + if ppart.term == carTerm && ppart.loc == loc { + continue LOCATIONS_LOOP + } + } + + // this location works, add it to the path (but not for empty term) + px := append(p, phrasePart{term: carTerm, loc: loc}) + rv = findPhrasePaths(loc.Pos, loc.ArrayPositions, cdr, tlm, px, remainingSlop-dist, rv) + } + } + } + return rv +} + +func editDistance(p1, p2 uint64) int { + dist := int(p1 - p2) + if dist < 0 { + return -dist + } + return dist +} + +func (s *PhraseSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + if s.currMust != nil { + if s.currMust.IndexInternalID.Compare(ID) >= 0 { + return s.Next(ctx) + } + ctx.DocumentMatchPool.Put(s.currMust) + } + if s.currMust == nil { + return nil, nil + } + var err error + s.currMust, err = s.mustSearcher.Advance(ctx, ID) + if err != nil { + return nil, err + } + return s.Next(ctx) +} + +func (s *PhraseSearcher) Count() uint64 { + // for now return a worst case + return s.mustSearcher.Count() +} + +func (s *PhraseSearcher) Close() error { + if s.mustSearcher != nil { + err := s.mustSearcher.Close() + if err != nil { + return err + } + } + return nil +} + +func (s *PhraseSearcher) Min() int { + return 0 +} + +func (s *PhraseSearcher) DocumentMatchPoolSize() int { + return s.mustSearcher.DocumentMatchPoolSize() + 1 +} diff --git a/search/searcher/search_phrase_test.go b/search/searcher/search_phrase_test.go new file mode 100644 index 0000000..178919d --- /dev/null +++ b/search/searcher/search_phrase_test.go @@ -0,0 +1,818 @@ +// Copyright (c) 2013 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func TestPhraseSearch(t *testing.T) { + twoDocIndexReader, err := twoDocIndex.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := twoDocIndexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + soptions := search.SearcherOptions{Explain: true, IncludeTermVectors: true} + phraseSearcher, err := NewPhraseSearcher(context.TODO(), twoDocIndexReader, []string{"angst", "beer"}, 0, false, "desc", 1.0, soptions) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + searcher search.Searcher + results []*search.DocumentMatch + locations map[string]map[string][]search.Location + fieldterms [][2]string + }{ + { + searcher: phraseSearcher, + results: []*search.DocumentMatch{ + { + IndexInternalID: index.IndexInternalID("2"), + Score: 1.0807601687084403, + }, + }, + locations: map[string]map[string][]search.Location{"desc": {"beer": {{Pos: 2, Start: 6, End: 10}}, "angst": {{Pos: 1, Start: 0, End: 5}}}}, + fieldterms: [][2]string{{"desc", "beer"}, {"desc", "angst"}}, + }, + } + + for testIndex, test := range tests { + defer func() { + err := test.searcher.Close() + if err != nil { + t.Fatal(err) + } + }() + + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0), + } + next, err := test.searcher.Next(ctx) + i := 0 + for err == nil && next != nil { + next.Complete(nil) + if i < len(test.results) { + if !next.IndexInternalID.Equals(test.results[i].IndexInternalID) { + t.Errorf("expected result %d to have id %s got %s for test %d\n", i, test.results[i].IndexInternalID, next.IndexInternalID, testIndex) + } + if next.Score != test.results[i].Score { + t.Errorf("expected result %d to have score %v got %v for test %d\n", i, test.results[i].Score, next.Score, testIndex) + t.Logf("scoring explanation: %s\n", next.Expl) + } + for _, ft := range test.fieldterms { + locs := next.Locations[ft[0]][ft[1]] + explocs := test.locations[ft[0]][ft[1]] + if len(explocs) != len(locs) { + t.Fatalf("expected result %d to have %d Locations (%#v) but got %d (%#v) for test %d with field %q and term %q\n", i, len(explocs), explocs, len(locs), locs, testIndex, ft[0], ft[1]) + } + for ind, exploc := range explocs { + if !reflect.DeepEqual(*locs[ind], exploc) { + t.Errorf("expected result %d to have Location %v got %v for test %d\n", i, exploc, locs[ind], testIndex) + } + } + } + } + + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) + i++ + } + if err != nil { + t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) + } + if len(test.results) != i { + t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) + } + } +} + +func TestMultiPhraseSearch(t *testing.T) { + soptions := search.SearcherOptions{Explain: true, IncludeTermVectors: true} + + tests := []struct { + phrase [][]string + docids [][]byte + }{ + { + phrase: [][]string{{"angst", "what"}, {"beer"}}, + docids: [][]byte{[]byte("2")}, + }, + } + + for i, test := range tests { + + reader, err := twoDocIndex.Reader() + if err != nil { + t.Error(err) + } + searcher, err := NewMultiPhraseSearcher(context.TODO(), reader, test.phrase, 0, false, "desc", 1.0, soptions) + if err != nil { + t.Error(err) + } + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(searcher.DocumentMatchPoolSize(), 0), + } + next, err := searcher.Next(ctx) + var actualIds [][]byte + for err == nil && next != nil { + actualIds = append(actualIds, next.IndexInternalID) + ctx.DocumentMatchPool.Put(next) + next, err = searcher.Next(ctx) + } + if err != nil { + t.Fatalf("error iterating searcher: %v for test %d", err, i) + } + if !reflect.DeepEqual(test.docids, actualIds) { + t.Fatalf("expected ids: %v, got %v", test.docids, actualIds) + } + + err = searcher.Close() + if err != nil { + t.Error(err) + } + + err = reader.Close() + if err != nil { + t.Error(err) + } + } +} + +func TestFuzzyMultiPhraseSearch(t *testing.T) { + soptions := search.SearcherOptions{Explain: true, IncludeTermVectors: true} + + tests := []struct { + mphrase [][]string + docids [][]byte + fuzziness int + prefix int + }{ + { + mphrase: [][]string{{"pale", "anger"}, {"best"}, {"colon", "porch"}}, + docids: [][]byte{[]byte("2"), []byte("3")}, + fuzziness: 2, + }, + { + mphrase: [][]string{{"pale", "anger"}, {}, {"colon", "porch", "could"}}, + docids: nil, + fuzziness: 1, + }, + { + mphrase: [][]string{{"app"}, {"best"}, {"volume"}}, + docids: [][]byte{[]byte("3")}, + fuzziness: 2, + }, + { + mphrase: [][]string{{"anger", "pale", "bar"}, {"beard"}, {}, {}}, + docids: [][]byte{[]byte("1"), []byte("2"), []byte("3"), []byte("4")}, + fuzziness: 2, + }, + { + mphrase: [][]string{{"anger", "pale", "bar"}, {}, {"beard"}, {}}, + docids: [][]byte{[]byte("1"), []byte("4")}, + fuzziness: 2, + }, + } + + for i, test := range tests { + + reader, err := twoDocIndex.Reader() + if err != nil { + t.Error(err) + } + searcher, err := NewMultiPhraseSearcher(context.TODO(), reader, test.mphrase, test.fuzziness, false, "desc", 1.0, soptions) + if err != nil { + t.Error(err) + } + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(searcher.DocumentMatchPoolSize(), 0), + } + next, err := searcher.Next(ctx) + var actualIds [][]byte + for err == nil && next != nil { + actualIds = append(actualIds, next.IndexInternalID) + ctx.DocumentMatchPool.Put(next) + next, err = searcher.Next(ctx) + } + if err != nil { + t.Fatalf("error iterating searcher: %v for test %d", err, i) + } + if !reflect.DeepEqual(test.docids, actualIds) { + t.Fatalf("expected ids: %v, got %v", test.docids, actualIds) + } + + err = searcher.Close() + if err != nil { + t.Error(err) + } + + err = reader.Close() + if err != nil { + t.Error(err) + } + } +} + +func TestFindPhrasePaths(t *testing.T) { + tests := []struct { + phrase [][]string + tlm search.TermLocationMap + paths []phrasePath + }{ + // simplest matching case + { + phrase: [][]string{{"cat"}, {"dog"}}, + tlm: search.TermLocationMap{ + "cat": search.Locations{ + &search.Location{ + Pos: 1, + }, + }, + "dog": search.Locations{ + &search.Location{ + Pos: 2, + }, + }, + }, + paths: []phrasePath{ + { + phrasePart{"cat", &search.Location{Pos: 1}}, + phrasePart{"dog", &search.Location{Pos: 2}}, + }, + }, + }, + // second term missing, no match + { + phrase: [][]string{{"cat"}, {"dog"}}, + tlm: search.TermLocationMap{ + "cat": search.Locations{ + &search.Location{ + Pos: 1, + }, + }, + }, + paths: nil, + }, + // second term exists but in wrong position + { + phrase: [][]string{{"cat"}, {"dog"}}, + tlm: search.TermLocationMap{ + "cat": search.Locations{ + &search.Location{ + Pos: 1, + }, + }, + "dog": search.Locations{ + &search.Location{ + Pos: 3, + }, + }, + }, + paths: nil, + }, + // matches multiple times + { + phrase: [][]string{{"cat"}, {"dog"}}, + tlm: search.TermLocationMap{ + "cat": search.Locations{ + &search.Location{ + Pos: 1, + }, + &search.Location{ + Pos: 8, + }, + }, + "dog": search.Locations{ + &search.Location{ + Pos: 2, + }, + &search.Location{ + Pos: 9, + }, + }, + }, + paths: []phrasePath{ + { + phrasePart{"cat", &search.Location{Pos: 1}}, + phrasePart{"dog", &search.Location{Pos: 2}}, + }, + { + phrasePart{"cat", &search.Location{Pos: 8}}, + phrasePart{"dog", &search.Location{Pos: 9}}, + }, + }, + }, + // match over gaps + { + phrase: [][]string{{"cat"}, {""}, {"dog"}}, + tlm: search.TermLocationMap{ + "cat": search.Locations{ + &search.Location{ + Pos: 1, + }, + }, + "dog": search.Locations{ + &search.Location{ + Pos: 3, + }, + }, + }, + paths: []phrasePath{ + { + phrasePart{"cat", &search.Location{Pos: 1}}, + phrasePart{"dog", &search.Location{Pos: 3}}, + }, + }, + }, + // match with leading "" + { + phrase: [][]string{{""}, {"cat"}, {"dog"}}, + tlm: search.TermLocationMap{ + "cat": search.Locations{ + &search.Location{ + Pos: 2, + }, + }, + "dog": search.Locations{ + &search.Location{ + Pos: 3, + }, + }, + }, + paths: []phrasePath{ + { + phrasePart{"cat", &search.Location{Pos: 2}}, + phrasePart{"dog", &search.Location{Pos: 3}}, + }, + }, + }, + // match with trailing "" + { + phrase: [][]string{{"cat"}, {"dog"}, {""}}, + tlm: search.TermLocationMap{ + "cat": search.Locations{ + &search.Location{ + Pos: 2, + }, + }, + "dog": search.Locations{ + &search.Location{ + Pos: 3, + }, + }, + }, + paths: []phrasePath{ + { + phrasePart{"cat", &search.Location{Pos: 2}}, + phrasePart{"dog", &search.Location{Pos: 3}}, + }, + }, + }, + } + + for i, test := range tests { + actualPaths := findPhrasePaths(0, nil, test.phrase, test.tlm, nil, 0, nil) + if !reflect.DeepEqual(actualPaths, test.paths) { + t.Fatalf("expected: %v got %v for test %d", test.paths, actualPaths, i) + } + } +} + +func TestFindPhrasePathsSloppy(t *testing.T) { + tlm := search.TermLocationMap{ + "one": search.Locations{ + &search.Location{ + Pos: 1, + }, + }, + "two": search.Locations{ + &search.Location{ + Pos: 2, + }, + }, + "three": search.Locations{ + &search.Location{ + Pos: 3, + }, + }, + "four": search.Locations{ + &search.Location{ + Pos: 4, + }, + }, + "five": search.Locations{ + &search.Location{ + Pos: 5, + }, + }, + } + + tests := []struct { + phrase [][]string + paths []phrasePath + slop int + tlm search.TermLocationMap + }{ + // no match + { + phrase: [][]string{{"one"}, {"five"}}, + slop: 2, + }, + // should match + { + phrase: [][]string{{"one"}, {"five"}}, + slop: 3, + paths: []phrasePath{ + { + phrasePart{"one", &search.Location{Pos: 1}}, + phrasePart{"five", &search.Location{Pos: 5}}, + }, + }, + }, + // slop 0 finds exact match + { + phrase: [][]string{{"four"}, {"five"}}, + slop: 0, + paths: []phrasePath{ + { + phrasePart{"four", &search.Location{Pos: 4}}, + phrasePart{"five", &search.Location{Pos: 5}}, + }, + }, + }, + // slop 0 does not find exact match (reversed) + { + phrase: [][]string{{"two"}, {"one"}}, + slop: 0, + }, + // slop 1 finds exact match + { + phrase: [][]string{{"one"}, {"two"}}, + slop: 1, + paths: []phrasePath{ + { + phrasePart{"one", &search.Location{Pos: 1}}, + phrasePart{"two", &search.Location{Pos: 2}}, + }, + }, + }, + // slop 1 *still* does not find exact match (reversed) requires at least 2 + { + phrase: [][]string{{"two"}, {"one"}}, + slop: 1, + }, + // slop 2 does finds exact match reversed + { + phrase: [][]string{{"two"}, {"one"}}, + slop: 2, + paths: []phrasePath{ + { + phrasePart{"two", &search.Location{Pos: 2}}, + phrasePart{"one", &search.Location{Pos: 1}}, + }, + }, + }, + // slop 2 not enough for this + { + phrase: [][]string{{"three"}, {"one"}}, + slop: 2, + }, + // slop should be cumulative + { + phrase: [][]string{{"one"}, {"three"}, {"five"}}, + slop: 2, + paths: []phrasePath{ + { + phrasePart{"one", &search.Location{Pos: 1}}, + phrasePart{"three", &search.Location{Pos: 3}}, + phrasePart{"five", &search.Location{Pos: 5}}, + }, + }, + }, + // should require 6 + { + phrase: [][]string{{"five"}, {"three"}, {"one"}}, + slop: 5, + }, + // so lets try 6 + { + phrase: [][]string{{"five"}, {"three"}, {"one"}}, + slop: 6, + paths: []phrasePath{ + { + phrasePart{"five", &search.Location{Pos: 5}}, + phrasePart{"three", &search.Location{Pos: 3}}, + phrasePart{"one", &search.Location{Pos: 1}}, + }, + }, + }, + // test an append() related edge case, where append()'s + // current behavior needs to be called 3 times starting from a + // nil slice before it grows to a slice with extra capacity -- + // hence, 3 initial terms of ark, bat, cat + { + phrase: [][]string{ + {"ark"}, {"bat"}, {"cat"}, {"dog"}, + }, + slop: 1, + paths: []phrasePath{ + { + phrasePart{"ark", &search.Location{Pos: 1}}, + phrasePart{"bat", &search.Location{Pos: 2}}, + phrasePart{"cat", &search.Location{Pos: 3}}, + phrasePart{"dog", &search.Location{Pos: 4}}, + }, + { + phrasePart{"ark", &search.Location{Pos: 1}}, + phrasePart{"bat", &search.Location{Pos: 2}}, + phrasePart{"cat", &search.Location{Pos: 3}}, + phrasePart{"dog", &search.Location{Pos: 5}}, + }, + }, + tlm: search.TermLocationMap{ // ark bat cat dog dog + "ark": search.Locations{ + &search.Location{Pos: 1}, + }, + "bat": search.Locations{ + &search.Location{Pos: 2}, + }, + "cat": search.Locations{ + &search.Location{Pos: 3}, + }, + "dog": search.Locations{ + &search.Location{Pos: 4}, + &search.Location{Pos: 5}, + }, + }, + }, + // test that we don't see multiple hits from the same location + { + phrase: [][]string{ + {"cat"}, {"dog"}, {"dog"}, + }, + slop: 1, + paths: []phrasePath{ + { + phrasePart{"cat", &search.Location{Pos: 1}}, + phrasePart{"dog", &search.Location{Pos: 2}}, + phrasePart{"dog", &search.Location{Pos: 3}}, + }, + }, + tlm: search.TermLocationMap{ // cat dog dog + "cat": search.Locations{ + &search.Location{Pos: 1}, + }, + "dog": search.Locations{ + &search.Location{Pos: 2}, + &search.Location{Pos: 3}, + }, + }, + }, + // test that we don't see multiple hits from the same location + { + phrase: [][]string{ + {"cat"}, {"dog"}, + }, + slop: 10, + paths: []phrasePath{ + { + phrasePart{"cat", &search.Location{Pos: 1}}, + phrasePart{"dog", &search.Location{Pos: 2}}, + }, + { + phrasePart{"cat", &search.Location{Pos: 1}}, + phrasePart{"dog", &search.Location{Pos: 4}}, + }, + { + phrasePart{"cat", &search.Location{Pos: 3}}, + phrasePart{"dog", &search.Location{Pos: 2}}, + }, + { + phrasePart{"cat", &search.Location{Pos: 3}}, + phrasePart{"dog", &search.Location{Pos: 4}}, + }, + }, + tlm: search.TermLocationMap{ // cat dog cat dog + "cat": search.Locations{ + &search.Location{Pos: 1}, + &search.Location{Pos: 3}, + }, + "dog": search.Locations{ + &search.Location{Pos: 2}, + &search.Location{Pos: 4}, + }, + }, + }, + } + + for i, test := range tests { + tlmToUse := test.tlm + if tlmToUse == nil { + tlmToUse = tlm + } + actualPaths := findPhrasePaths(0, nil, test.phrase, tlmToUse, nil, test.slop, nil) + if !reflect.DeepEqual(actualPaths, test.paths) { + t.Fatalf("expected: %v got %v for test %d", test.paths, actualPaths, i) + } + } +} + +func TestFindPhrasePathsSloppyPalyndrome(t *testing.T) { + tlm := search.TermLocationMap{ + "one": search.Locations{ + &search.Location{ + Pos: 1, + }, + &search.Location{ + Pos: 5, + }, + }, + "two": search.Locations{ + &search.Location{ + Pos: 2, + }, + &search.Location{ + Pos: 4, + }, + }, + "three": search.Locations{ + &search.Location{ + Pos: 3, + }, + }, + } + + tests := []struct { + phrase [][]string + paths []phrasePath + slop int + }{ + // search non palyndrone, exact match + { + phrase: [][]string{{"two"}, {"three"}}, + slop: 0, + paths: []phrasePath{ + { + phrasePart{"two", &search.Location{Pos: 2}}, + phrasePart{"three", &search.Location{Pos: 3}}, + }, + }, + }, + // same with slop 2 (not required) (find it twice) + { + phrase: [][]string{{"two"}, {"three"}}, + slop: 2, + paths: []phrasePath{ + { + phrasePart{"two", &search.Location{Pos: 2}}, + phrasePart{"three", &search.Location{Pos: 3}}, + }, + { + phrasePart{"two", &search.Location{Pos: 4}}, + phrasePart{"three", &search.Location{Pos: 3}}, + }, + }, + }, + // palyndrone reversed + { + phrase: [][]string{{"three"}, {"two"}}, + slop: 2, + paths: []phrasePath{ + { + phrasePart{"three", &search.Location{Pos: 3}}, + phrasePart{"two", &search.Location{Pos: 2}}, + }, + { + phrasePart{"three", &search.Location{Pos: 3}}, + phrasePart{"two", &search.Location{Pos: 4}}, + }, + }, + }, + } + + for i, test := range tests { + actualPaths := findPhrasePaths(0, nil, test.phrase, tlm, nil, test.slop, nil) + if !reflect.DeepEqual(actualPaths, test.paths) { + t.Fatalf("expected: %v got %v for test %d", test.paths, actualPaths, i) + } + } +} + +func TestFindMultiPhrasePaths(t *testing.T) { + tlm := search.TermLocationMap{ + "cat": search.Locations{ + &search.Location{ + Pos: 1, + }, + }, + "dog": search.Locations{ + &search.Location{ + Pos: 2, + }, + }, + "frog": search.Locations{ + &search.Location{ + Pos: 3, + }, + }, + } + + tests := []struct { + phrase [][]string + paths []phrasePath + }{ + // simplest, one of two possible terms matches + { + phrase: [][]string{{"cat", "rat"}, {"dog"}}, + paths: []phrasePath{ + { + phrasePart{"cat", &search.Location{Pos: 1}}, + phrasePart{"dog", &search.Location{Pos: 2}}, + }, + }, + }, + // two possible terms, neither work + { + phrase: [][]string{{"cat", "rat"}, {"chicken"}}, + }, + // two possible terms, one works, but out of position with next + { + phrase: [][]string{{"cat", "rat"}, {"frog"}}, + }, + // matches multiple times, with different pairing + { + phrase: [][]string{{"cat", "dog"}, {"dog", "frog"}}, + paths: []phrasePath{ + { + phrasePart{"cat", &search.Location{Pos: 1}}, + phrasePart{"dog", &search.Location{Pos: 2}}, + }, + { + phrasePart{"dog", &search.Location{Pos: 2}}, + phrasePart{"frog", &search.Location{Pos: 3}}, + }, + }, + }, + // multi-match over a gap + { + phrase: [][]string{{"cat", "rat"}, {""}, {"frog"}}, + paths: []phrasePath{ + { + phrasePart{"cat", &search.Location{Pos: 1}}, + phrasePart{"frog", &search.Location{Pos: 3}}, + }, + }, + }, + // multi-match over a gap (same as before, but with empty term list) + { + phrase: [][]string{{"cat", "rat"}, {}, {"frog"}}, + paths: []phrasePath{ + { + phrasePart{"cat", &search.Location{Pos: 1}}, + phrasePart{"frog", &search.Location{Pos: 3}}, + }, + }, + }, + // multi-match over a gap (same once again, but nil term list) + { + phrase: [][]string{{"cat", "rat"}, nil, {"frog"}}, + paths: []phrasePath{ + { + phrasePart{"cat", &search.Location{Pos: 1}}, + phrasePart{"frog", &search.Location{Pos: 3}}, + }, + }, + }, + } + + for i, test := range tests { + actualPaths := findPhrasePaths(0, nil, test.phrase, tlm, nil, 0, nil) + if !reflect.DeepEqual(actualPaths, test.paths) { + t.Fatalf("expected: %v got %v for test %d", test.paths, actualPaths, i) + } + } +} diff --git a/search/searcher/search_regexp.go b/search/searcher/search_regexp.go new file mode 100644 index 0000000..bab779c --- /dev/null +++ b/search/searcher/search_regexp.go @@ -0,0 +1,169 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "regexp" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +// The Regexp interface defines the subset of the regexp.Regexp API +// methods that are used by bleve indexes, allowing callers to pass in +// alternate implementations. +type Regexp interface { + FindStringIndex(s string) (loc []int) + + LiteralPrefix() (prefix string, complete bool) + + String() string +} + +// NewRegexpStringSearcher is similar to NewRegexpSearcher, but +// additionally optimizes for index readers that handle regexp's. +func NewRegexpStringSearcher(ctx context.Context, indexReader index.IndexReader, pattern string, + field string, boost float64, options search.SearcherOptions) ( + search.Searcher, error) { + ir, ok := indexReader.(index.IndexReaderRegexp) + if !ok { + r, err := regexp.Compile(pattern) + if err != nil { + return nil, err + } + + return NewRegexpSearcher(ctx, indexReader, r, field, boost, options) + } + + fieldDict, a, err := ir.FieldDictRegexpAutomaton(field, pattern) + if err != nil { + return nil, err + } + defer func() { + if cerr := fieldDict.Close(); cerr != nil && err == nil { + err = cerr + } + }() + + var termSet = make(map[string]struct{}) + var candidateTerms []string + + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + if _, exists := termSet[tfd.Term]; !exists { + termSet[tfd.Term] = struct{}{} + candidateTerms = append(candidateTerms, tfd.Term) + tfd, err = fieldDict.Next() + } + } + if err != nil { + return nil, err + } + + if ctx != nil { + if fts, ok := ctx.Value(search.FieldTermSynonymMapKey).(search.FieldTermSynonymMap); ok { + if ts, exists := fts[field]; exists { + for term := range ts { + if _, exists := termSet[term]; exists { + continue + } + if a.MatchesRegex(term) { + termSet[term] = struct{}{} + candidateTerms = append(candidateTerms, term) + } + } + } + } + } + + return NewMultiTermSearcher(ctx, indexReader, candidateTerms, field, boost, + options, true) +} + +// NewRegexpSearcher creates a searcher which will match documents that +// contain terms which match the pattern regexp. The match must be EXACT +// matching the entire term. The provided regexp SHOULD NOT start with ^ +// or end with $ as this can interfere with the implementation. Separately, +// matches will be checked to ensure they match the entire term. +func NewRegexpSearcher(ctx context.Context, indexReader index.IndexReader, pattern Regexp, + field string, boost float64, options search.SearcherOptions) ( + search.Searcher, error) { + var candidateTerms []string + var regexpCandidates *regexpCandidates + prefixTerm, complete := pattern.LiteralPrefix() + if complete { + // there is no pattern + candidateTerms = []string{prefixTerm} + } else { + var err error + regexpCandidates, err = findRegexpCandidateTerms(indexReader, pattern, field, + prefixTerm) + if err != nil { + return nil, err + } + } + var dictBytesRead uint64 + if regexpCandidates != nil { + candidateTerms = regexpCandidates.candidates + dictBytesRead = regexpCandidates.bytesRead + } + + if ctx != nil { + reportIOStats(ctx, dictBytesRead) + search.RecordSearchCost(ctx, search.AddM, dictBytesRead) + } + + return NewMultiTermSearcher(ctx, indexReader, candidateTerms, field, boost, + options, true) +} + +type regexpCandidates struct { + candidates []string + bytesRead uint64 +} + +func findRegexpCandidateTerms(indexReader index.IndexReader, + pattern Regexp, field, prefixTerm string) (rv *regexpCandidates, err error) { + rv = ®expCandidates{ + candidates: make([]string, 0), + } + var fieldDict index.FieldDict + if len(prefixTerm) > 0 { + fieldDict, err = indexReader.FieldDictPrefix(field, []byte(prefixTerm)) + } else { + fieldDict, err = indexReader.FieldDict(field) + } + defer func() { + if cerr := fieldDict.Close(); cerr != nil && err == nil { + err = cerr + } + }() + + // enumerate the terms and check against regexp + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + matchPos := pattern.FindStringIndex(tfd.Term) + if matchPos != nil && matchPos[0] == 0 && matchPos[1] == len(tfd.Term) { + rv.candidates = append(rv.candidates, tfd.Term) + if tooManyClauses(len(rv.candidates)) { + return rv, tooManyClausesErr(field, len(rv.candidates)) + } + } + tfd, err = fieldDict.Next() + } + rv.bytesRead = fieldDict.BytesRead() + return rv, err +} diff --git a/search/searcher/search_regexp_test.go b/search/searcher/search_regexp_test.go new file mode 100644 index 0000000..02131bb --- /dev/null +++ b/search/searcher/search_regexp_test.go @@ -0,0 +1,171 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "encoding/binary" + "fmt" + "os" + "regexp" + "testing" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func TestRegexpSearchUpsideDown(t *testing.T) { + twoDocIndex := initTwoDocUpsideDown() + testRegexpSearch(t, twoDocIndex, internalIDMakerUpsideDown, searcherMaker) + _ = twoDocIndex.Close() +} + +func TestRegexpStringSearchUpsideDown(t *testing.T) { + twoDocIndex := initTwoDocUpsideDown() + testRegexpSearch(t, twoDocIndex, internalIDMakerUpsideDown, searcherStringMaker) + _ = twoDocIndex.Close() +} + +func TestRegexpSearchScorch(t *testing.T) { + dir, _ := os.MkdirTemp("", "scorchTwoDoc") + defer func() { + _ = os.RemoveAll(dir) + }() + + twoDocIndex := initTwoDocScorch(dir) + testRegexpSearch(t, twoDocIndex, internalIDMakerScorch, searcherMaker) + _ = twoDocIndex.Close() +} + +func TestRegexpStringSearchScorch(t *testing.T) { + dir, _ := os.MkdirTemp("", "scorchTwoDoc") + defer func() { + _ = os.RemoveAll(dir) + }() + + twoDocIndex := initTwoDocScorch(dir) + testRegexpSearch(t, twoDocIndex, internalIDMakerScorch, searcherStringMaker) + _ = twoDocIndex.Close() +} + +func internalIDMakerUpsideDown(id int) index.IndexInternalID { + return index.IndexInternalID(fmt.Sprintf("%d", id)) +} + +func internalIDMakerScorch(id int) index.IndexInternalID { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, uint64(id)) + return index.IndexInternalID(buf) +} + +func searcherMaker(t *testing.T, ir index.IndexReader, re, field string) search.Searcher { + pattern, err := regexp.Compile(re) + if err != nil { + t.Fatal(err) + } + + regexpSearcher, err := NewRegexpSearcher(context.TODO(), ir, pattern, field, 1.0, + search.SearcherOptions{Explain: true}) + if err != nil { + t.Fatal(err) + } + + return regexpSearcher +} + +func searcherStringMaker(t *testing.T, ir index.IndexReader, re, field string) search.Searcher { + regexpSearcher, err := NewRegexpStringSearcher(context.TODO(), ir, re, field, 1.0, + search.SearcherOptions{Explain: true}) + if err != nil { + t.Fatal(err) + } + + return regexpSearcher +} + +func testRegexpSearch(t *testing.T, twoDocIndex index.Index, + internalIDMaker func(int) index.IndexInternalID, + searcherMaker func(t *testing.T, ir index.IndexReader, re, field string) search.Searcher, +) { + twoDocIndexReader, err := twoDocIndex.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := twoDocIndexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + regexpSearcher := searcherMaker(t, twoDocIndexReader, "ma.*", "name") + regexpSearcherCo := searcherMaker(t, twoDocIndexReader, "co.*", "desc") + + tests := []struct { + searcher search.Searcher + id2score map[string]float64 + }{ + { + searcher: regexpSearcher, + id2score: map[string]float64{ + "1": 1.916290731874155, + }, + }, + { + searcher: regexpSearcherCo, + id2score: map[string]float64{ + "2": 0.33875554280828685, + "3": 0.33875554280828685, + }, + }, + } + + for testIndex, test := range tests { + defer func() { + err := test.searcher.Close() + if err != nil { + t.Fatal(err) + } + }() + + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(test.searcher.DocumentMatchPoolSize(), 0), + } + next, err := test.searcher.Next(ctx) + i := 0 + for err == nil && next != nil { + exID, _ := twoDocIndexReader.ExternalID(next.IndexInternalID) + if _, ok := test.id2score[exID]; !ok { + t.Errorf("test %d, found unexpected docID = %v, next = %v", testIndex, exID, next) + } else { + score := test.id2score[exID] + if next.Score != score { + t.Errorf("test %d, expected result %d to have score %v got %v,next: %#v", + testIndex, i, score, next.Score, next) + t.Logf("scoring explanation: %s", next.Expl) + } + } + ctx.DocumentMatchPool.Put(next) + next, err = test.searcher.Next(ctx) + i++ + } + if err != nil { + t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) + } + if len(test.id2score) != i { + t.Errorf("expected %d results got %d for test %d", len(test.id2score), i, testIndex) + } + } +} diff --git a/search/searcher/search_term.go b/search/searcher/search_term.go new file mode 100644 index 0000000..d9d0124 --- /dev/null +++ b/search/searcher/search_term.go @@ -0,0 +1,298 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "fmt" + "math" + "reflect" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/scorer" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeTermSearcher int + +func init() { + var ts TermSearcher + reflectStaticSizeTermSearcher = int(reflect.TypeOf(ts).Size()) +} + +type TermSearcher struct { + indexReader index.IndexReader + reader index.TermFieldReader + scorer *scorer.TermQueryScorer + tfd index.TermFieldDoc +} + +func NewTermSearcher(ctx context.Context, indexReader index.IndexReader, + term string, field string, boost float64, options search.SearcherOptions) (search.Searcher, error) { + if isTermQuery(ctx) { + ctx = context.WithValue(ctx, search.QueryTypeKey, search.Term) + } + return NewTermSearcherBytes(ctx, indexReader, []byte(term), field, boost, options) +} + +func NewTermSearcherBytes(ctx context.Context, indexReader index.IndexReader, + term []byte, field string, boost float64, options search.SearcherOptions) (search.Searcher, error) { + if ctx != nil { + if fts, ok := ctx.Value(search.FieldTermSynonymMapKey).(search.FieldTermSynonymMap); ok { + if ts, exists := fts[field]; exists { + if s, found := ts[string(term)]; found { + return NewSynonymSearcher(ctx, indexReader, term, s, field, boost, options) + } + } + } + } + needFreqNorm := options.Score != "none" + reader, err := indexReader.TermFieldReader(ctx, term, field, needFreqNorm, needFreqNorm, options.IncludeTermVectors) + if err != nil { + return nil, err + } + termSearcher, err := newTermSearcherFromReader(ctx, indexReader, reader, term, field, boost, options) + if err != nil || termSearcher == nil { + return nil, err + } + + return termSearcher, err +} + +func tfIDFScoreMetrics(indexReader index.IndexReader) (uint64, error) { + // default tf-idf stats + count, err := indexReader.DocCount() + if err != nil { + return 0, err + } + + if count == 0 { + return 0, nil + } + return count, nil +} + +func bm25ScoreMetrics(ctx context.Context, field string, + indexReader index.IndexReader) (uint64, float64, error) { + var count uint64 + var fieldCardinality int + var err error + + bm25Stats, ok := ctx.Value(search.BM25StatsKey).(*search.BM25Stats) + if !ok { + count, err = indexReader.DocCount() + if err != nil { + return 0, 0, err + } + if bm25Reader, ok := indexReader.(index.BM25Reader); ok { + fieldCardinality, err = bm25Reader.FieldCardinality(field) + if err != nil { + return 0, 0, err + } + } + } else { + count = uint64(bm25Stats.DocCount) + fieldCardinality, ok = bm25Stats.FieldCardinality[field] + if !ok { + return 0, 0, fmt.Errorf("field stat for bm25 not present %s", field) + } + } + + if count == 0 && fieldCardinality == 0 { + return 0, 0, nil + } + return count, math.Ceil(float64(fieldCardinality) / float64(count)), nil +} + +func newTermSearcherFromReader(ctx context.Context, indexReader index.IndexReader, + reader index.TermFieldReader, term []byte, field string, boost float64, + options search.SearcherOptions) (*TermSearcher, error) { + // Every leaf term searcher is built here, so this is where we count against + // the limit. Close the reader on failure since the caller won't get it back. + if err := search.RecordTermSearcher(ctx); err != nil { + _ = reader.Close() + return nil, err + } + var count uint64 + var avgDocLength float64 + var err error + var similarityModel string + + // as a fallback case we track certain stats for tf-idf scoring + if ctx != nil { + if similarityModelCallback, ok := ctx.Value(search. + GetScoringModelCallbackKey).(search.GetScoringModelCallbackFn); ok { + similarityModel = similarityModelCallback() + } + } + switch similarityModel { + case index.BM25Scoring: + count, avgDocLength, err = bm25ScoreMetrics(ctx, field, indexReader) + if err != nil { + _ = reader.Close() + return nil, err + } + case index.TFIDFScoring: + fallthrough + default: + count, err = tfIDFScoreMetrics(indexReader) + if err != nil { + _ = reader.Close() + return nil, err + } + } + scorer := scorer.NewTermQueryScorer(term, field, boost, count, reader.Count(), avgDocLength, options) + return &TermSearcher{ + indexReader: indexReader, + reader: reader, + scorer: scorer, + }, nil +} + +func NewSynonymSearcher(ctx context.Context, indexReader index.IndexReader, term []byte, synonyms []string, field string, boost float64, options search.SearcherOptions) (search.Searcher, error) { + createTermSearcher := func(term []byte, boostVal float64) (search.Searcher, error) { + needFreqNorm := options.Score != "none" + reader, err := indexReader.TermFieldReader(ctx, term, field, needFreqNorm, needFreqNorm, options.IncludeTermVectors) + if err != nil { + return nil, err + } + termSearcher, err := newTermSearcherFromReader(ctx, indexReader, reader, term, field, boostVal, options) + if err != nil || termSearcher == nil { + return nil, err + } + + return termSearcher, err + } + // create a searcher for the term itself + termSearcher, err := createTermSearcher(term, boost) + if err != nil { + return nil, err + } + // constituent searchers of the disjunction + qsearchers := make([]search.Searcher, 0, len(synonyms)+1) + // helper method to close all the searchers we've created + // in case of an error + qsearchersClose := func() { + for _, searcher := range qsearchers { + if searcher != nil { + _ = searcher.Close() + } + } + } + qsearchers = append(qsearchers, termSearcher) + // create a searcher for each synonym + for _, synonym := range synonyms { + synonymSearcher, err := createTermSearcher([]byte(synonym), boost/2.0) + if err != nil { + qsearchersClose() + return nil, err + } + qsearchers = append(qsearchers, synonymSearcher) + } + // create a disjunction searcher + rv, err := NewDisjunctionSearcher(ctx, indexReader, qsearchers, 0, options) + if err != nil { + qsearchersClose() + return nil, err + } + return rv, nil +} + +func (s *TermSearcher) Size() int { + return reflectStaticSizeTermSearcher + size.SizeOfPtr + + s.reader.Size() + + s.tfd.Size() + + s.scorer.Size() +} + +func (s *TermSearcher) Count() uint64 { + return s.reader.Count() +} + +func (s *TermSearcher) Weight() float64 { + return s.scorer.Weight() +} + +func (s *TermSearcher) SetQueryNorm(qnorm float64) { + s.scorer.SetQueryNorm(qnorm) +} + +func (s *TermSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + termMatch, err := s.reader.Next(s.tfd.Reset()) + if err != nil { + return nil, err + } + + if termMatch == nil { + return nil, nil + } + + // score match + docMatch := s.scorer.Score(ctx, termMatch) + // return doc match + return docMatch, nil + +} + +func (s *TermSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + termMatch, err := s.reader.Advance(ID, s.tfd.Reset()) + if err != nil { + return nil, err + } + + if termMatch == nil { + return nil, nil + } + + // score match + docMatch := s.scorer.Score(ctx, termMatch) + + // return doc match + return docMatch, nil +} + +func (s *TermSearcher) Close() error { + return s.reader.Close() +} + +func (s *TermSearcher) Min() int { + return 0 +} + +func (s *TermSearcher) DocumentMatchPoolSize() int { + return 1 +} + +func (s *TermSearcher) Optimize(kind string, octx index.OptimizableContext) ( + index.OptimizableContext, error) { + o, ok := s.reader.(index.Optimizable) + if ok { + return o.Optimize(kind, octx) + } + + return nil, nil +} + +func isTermQuery(ctx context.Context) bool { + if ctx != nil { + // if the ctx already has a value set for query type + // it would've been done at a non term searcher level. + _, ok := ctx.Value(search.QueryTypeKey).(string) + return !ok + } + // if the context is nil, then don't set the query type + return false +} diff --git a/search/searcher/search_term_prefix.go b/search/searcher/search_term_prefix.go new file mode 100644 index 0000000..3d98cd2 --- /dev/null +++ b/search/searcher/search_term_prefix.go @@ -0,0 +1,86 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "strings" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func NewTermPrefixSearcher(ctx context.Context, indexReader index.IndexReader, prefix string, + field string, boost float64, options search.SearcherOptions) ( + search.Searcher, error) { + // find the terms with this prefix + fieldDict, err := indexReader.FieldDictPrefix(field, []byte(prefix)) + if err != nil { + return nil, err + } + defer func() { + if cerr := fieldDict.Close(); cerr != nil && err == nil { + err = cerr + } + }() + + var terms []string + var termSet = make(map[string]struct{}) + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + if _, exists := termSet[tfd.Term]; !exists { + termSet[tfd.Term] = struct{}{} + terms = append(terms, tfd.Term) + if tooManyClauses(len(terms)) { + return nil, tooManyClausesErr(field, len(terms)) + } + tfd, err = fieldDict.Next() + } + } + if err != nil { + return nil, err + } + + if ctx != nil { + reportIOStats(ctx, fieldDict.BytesRead()) + search.RecordSearchCost(ctx, search.AddM, fieldDict.BytesRead()) + } + + if ctx != nil { + if fts, ok := ctx.Value(search.FieldTermSynonymMapKey).(search.FieldTermSynonymMap); ok { + if ts, exists := fts[field]; exists { + for term := range ts { + if _, exists := termSet[term]; exists { + continue + } + if strings.HasPrefix(term, prefix) { + termSet[term] = struct{}{} + terms = append(terms, term) + if tooManyClauses(len(terms)) { + return nil, tooManyClausesErr(field, len(terms)) + } + } + } + } + } + } + + // check if the terms are empty or have one term which is the prefix itself + if len(terms) == 0 || (len(terms) == 1 && terms[0] == prefix) { + return NewTermSearcher(ctx, indexReader, prefix, field, boost, options) + } + + return NewMultiTermSearcher(ctx, indexReader, terms, field, boost, options, true) +} diff --git a/search/searcher/search_term_range.go b/search/searcher/search_term_range.go new file mode 100644 index 0000000..990c738 --- /dev/null +++ b/search/searcher/search_term_range.go @@ -0,0 +1,92 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func NewTermRangeSearcher(ctx context.Context, indexReader index.IndexReader, + min, max []byte, inclusiveMin, inclusiveMax *bool, field string, + boost float64, options search.SearcherOptions) (search.Searcher, error) { + + if inclusiveMin == nil { + defaultInclusiveMin := true + inclusiveMin = &defaultInclusiveMin + } + if inclusiveMax == nil { + defaultInclusiveMax := false + inclusiveMax = &defaultInclusiveMax + } + + if min == nil { + min = []byte{} + } + + rangeMax := max + if rangeMax != nil { + // the term dictionary range end has an unfortunate implementation + rangeMax = append(rangeMax, 0) + } + + // find the terms with this prefix + fieldDict, err := indexReader.FieldDictRange(field, min, rangeMax) + if err != nil { + return nil, err + } + + defer func() { + if cerr := fieldDict.Close(); cerr != nil && err == nil { + err = cerr + } + }() + + var terms []string + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + terms = append(terms, tfd.Term) + tfd, err = fieldDict.Next() + } + if err != nil { + return nil, err + } + + if len(terms) < 1 { + return NewMatchNoneSearcher(indexReader) + } + + if !*inclusiveMin && min != nil && string(min) == terms[0] { + terms = terms[1:] + // check again, as we might have removed only entry + if len(terms) < 1 { + return NewMatchNoneSearcher(indexReader) + } + } + + // if our term list included the max, it would be the last item + if !*inclusiveMax && max != nil && string(max) == terms[len(terms)-1] { + terms = terms[:len(terms)-1] + } + + if ctx != nil { + reportIOStats(ctx, fieldDict.BytesRead()) + search.RecordSearchCost(ctx, search.AddM, fieldDict.BytesRead()) + } + + return NewMultiTermSearcher(ctx, indexReader, terms, field, boost, options, true) +} diff --git a/search/searcher/search_term_range_test.go b/search/searcher/search_term_range_test.go new file mode 100644 index 0000000..3e3e3f8 --- /dev/null +++ b/search/searcher/search_term_range_test.go @@ -0,0 +1,285 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "os" + "reflect" + "sort" + "testing" + + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/search" +) + +func TestTermRangeSearch(t *testing.T) { + twoDocIndexReader, err := twoDocIndex.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := twoDocIndexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + tests := []struct { + min []byte + max []byte + inclusiveMin bool + inclusiveMax bool + field string + want []string + }{ + { + min: []byte("marty"), + max: []byte("marty"), + field: "name", + inclusiveMin: true, + inclusiveMax: true, + want: []string{"1"}, + }, + { + min: []byte("marty"), + max: []byte("ravi"), + field: "name", + inclusiveMin: true, + inclusiveMax: true, + want: []string{"1", "4"}, + }, + // inclusive max false should exclude ravi + { + min: []byte("marty"), + max: []byte("ravi"), + field: "name", + inclusiveMin: true, + inclusiveMax: false, + want: []string{"1"}, + }, + // inclusive max false should remove last/only item + { + min: []byte("martz"), + max: []byte("ravi"), + field: "name", + inclusiveMin: true, + inclusiveMax: false, + want: nil, + }, + // inclusive min false should remove marty + { + min: []byte("marty"), + max: []byte("ravi"), + field: "name", + inclusiveMin: false, + inclusiveMax: true, + want: []string{"4"}, + }, + // inclusive min false should remove first/only item + { + min: []byte("marty"), + max: []byte("rav"), + field: "name", + inclusiveMin: false, + inclusiveMax: true, + want: nil, + }, + // max nil sees everything after marty + { + min: []byte("marty"), + max: nil, + field: "name", + inclusiveMin: true, + inclusiveMax: true, + want: []string{"1", "2", "4"}, + }, + // min nil sees everything before ravi + { + min: nil, + max: []byte("ravi"), + field: "name", + inclusiveMin: true, + inclusiveMax: true, + want: []string{"1", "3", "4", "5"}, + }, + // min and max nil sees everything + { + min: nil, + max: nil, + field: "name", + inclusiveMin: true, + inclusiveMax: true, + want: []string{"1", "2", "3", "4", "5"}, + }, + // min and max nil sees everything, even with inclusiveMin false + { + min: nil, + max: nil, + field: "name", + inclusiveMin: false, + inclusiveMax: true, + want: []string{"1", "2", "3", "4", "5"}, + }, + // min and max nil sees everything, even with inclusiveMax false + { + min: nil, + max: nil, + field: "name", + inclusiveMin: true, + inclusiveMax: false, + want: []string{"1", "2", "3", "4", "5"}, + }, + // min and max nil sees everything, even with both false + { + min: nil, + max: nil, + field: "name", + inclusiveMin: false, + inclusiveMax: false, + want: []string{"1", "2", "3", "4", "5"}, + }, + // min and max non-nil, but match 0 terms + { + min: []byte("martz"), + max: []byte("rav"), + field: "name", + inclusiveMin: true, + inclusiveMax: true, + want: nil, + }, + // min and max same (and term exists), both exclusive + { + min: []byte("marty"), + max: []byte("marty"), + field: "name", + inclusiveMin: false, + inclusiveMax: false, + want: nil, + }, + } + + for _, test := range tests { + + searcher, err := NewTermRangeSearcher(context.TODO(), twoDocIndexReader, test.min, test.max, + &test.inclusiveMin, &test.inclusiveMax, test.field, 1.0, search.SearcherOptions{Explain: true}) + if err != nil { + t.Fatal(err) + } + + var got []string + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool( + searcher.DocumentMatchPoolSize(), 0), + } + next, err := searcher.Next(ctx) + i := 0 + for err == nil && next != nil { + got = append(got, string(next.IndexInternalID)) + ctx.DocumentMatchPool.Put(next) + next, err = searcher.Next(ctx) + i++ + } + if err != nil { + t.Fatalf("error iterating searcher: %v", err) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("expected: %v, got %v for test %#v", test.want, got, test) + } + + } +} + +func TestTermRangeSearchTooManyTerms(t *testing.T) { + dir, _ := os.MkdirTemp("", "scorchTwoDoc") + defer func() { + _ = os.RemoveAll(dir) + }() + + scorchIndex := initTwoDocScorch(dir) + + // use lower limit for this test + origLimit := DisjunctionMaxClauseCount + DisjunctionMaxClauseCount = 2 + defer func() { + DisjunctionMaxClauseCount = origLimit + }() + + scorchReader, err := scorchIndex.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := scorchReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + want := []string{"1", "3", "4", "5"} + truth := true + searcher, err := NewTermRangeSearcher(context.TODO(), scorchReader, []byte("bobert"), []byte("ravi"), + &truth, &truth, "name", 1.0, search.SearcherOptions{Score: "none", IncludeTermVectors: false}) + if err != nil { + t.Fatal(err) + } + + var got []string + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool( + searcher.DocumentMatchPoolSize(), 0), + } + next, err := searcher.Next(ctx) + i := 0 + for err == nil && next != nil { + extId, err := scorchReader.ExternalID(next.IndexInternalID) + if err != nil { + t.Fatal(err) + } + got = append(got, extId) + ctx.DocumentMatchPool.Put(next) + next, err = searcher.Next(ctx) + if err != nil { + break + } + + i++ + } + if err != nil { + t.Fatalf("error iterating searcher: %v", err) + } + err = searcher.Close() + if err != nil { + t.Fatal(err) + } + + // check that the expected number of term searchers were started + // 6 = 4 original terms, 1 optimized after first round, then final searcher + // from the last round + statsMap := scorchIndex.(*scorch.Scorch).StatsMap() + if statsMap["term_searchers_started"].(uint64) != 6 { + t.Errorf("expected 6 term searchers started, got %d", statsMap["term_searchers_started"]) + } + // check that all started searchers were closed + if statsMap["term_searchers_started"] != statsMap["term_searchers_finished"] { + t.Errorf("expected all term searchers closed, %d started %d closed", + statsMap["term_searchers_started"], statsMap["term_searchers_finished"]) + } + + sort.Strings(got) + if !reflect.DeepEqual(got, want) { + t.Errorf("expected: %#v, got %#v", want, got) + } +} diff --git a/search/searcher/search_term_test.go b/search/searcher/search_term_test.go new file mode 100644 index 0000000..e87db9d --- /dev/null +++ b/search/searcher/search_term_test.go @@ -0,0 +1,185 @@ +// Copyright (c) 2013 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "context" + "math" + "testing" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/index/upsidedown" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func TestTermSearcher(t *testing.T) { + queryTerm := "beer" + queryField := "desc" + queryBoost := 3.0 + queryExplain := search.SearcherOptions{Explain: true} + + analysisQueue := index.NewAnalysisQueue(1) + i, err := upsidedown.NewUpsideDownCouch( + gtreap.Name, + map[string]interface{}{ + "path": "", + }, + analysisQueue) + if err != nil { + t.Fatal(err) + } + err = i.Open() + if err != nil { + t.Fatal(err) + } + doc := document.NewDocument("a") + doc.AddField(document.NewTextField("desc", []uint64{}, []byte("beer"))) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + doc = document.NewDocument("b") + doc.AddField(document.NewTextField("desc", []uint64{}, []byte("beer"))) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + doc = document.NewDocument("c") + doc.AddField(document.NewTextField("desc", []uint64{}, []byte("beer"))) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + doc = document.NewDocument("d") + doc.AddField(document.NewTextField("desc", []uint64{}, []byte("beer"))) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + doc = document.NewDocument("e") + doc.AddField(document.NewTextField("desc", []uint64{}, []byte("beer"))) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + doc = document.NewDocument("f") + doc.AddField(document.NewTextField("desc", []uint64{}, []byte("beer"))) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + doc = document.NewDocument("g") + doc.AddField(document.NewTextField("desc", []uint64{}, []byte("beer"))) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + doc = document.NewDocument("h") + doc.AddField(document.NewTextField("desc", []uint64{}, []byte("beer"))) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + doc = document.NewDocument("i") + doc.AddField(document.NewTextField("desc", []uint64{}, []byte("beer"))) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + doc = document.NewDocument("j") + doc.AddField(document.NewTextField("title", []uint64{}, []byte("cat"))) + err = i.Update(doc) + if err != nil { + t.Fatal(err) + } + + indexReader, err := i.Reader() + if err != nil { + t.Error(err) + } + defer func() { + err := indexReader.Close() + if err != nil { + t.Fatal(err) + } + }() + + searcher, err := NewTermSearcher(context.TODO(), indexReader, queryTerm, queryField, queryBoost, queryExplain) + if err != nil { + t.Fatal(err) + } + defer func() { + err := searcher.Close() + if err != nil { + t.Fatal(err) + } + }() + + searcher.SetQueryNorm(2.0) + docCount, err := indexReader.DocCount() + if err != nil { + t.Fatal(err) + } + idf := 1.0 + math.Log(float64(docCount)/float64(searcher.Count()+1.0)) + expectedQueryWeight := 3 * idf * 3 * idf + if expectedQueryWeight != searcher.Weight() { + t.Errorf("expected weight %v got %v", expectedQueryWeight, searcher.Weight()) + } + + if searcher.Count() != 9 { + t.Errorf("expected count of 9, got %d", searcher.Count()) + } + + ctx := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(1, 0), + } + docMatch, err := searcher.Next(ctx) + if err != nil { + t.Errorf("expected result, got %v", err) + } + if !docMatch.IndexInternalID.Equals(index.IndexInternalID("a")) { + t.Errorf("expected result ID to be 'a', got '%s", docMatch.IndexInternalID) + } + ctx.DocumentMatchPool.Put(docMatch) + docMatch, err = searcher.Advance(ctx, index.IndexInternalID("c")) + if err != nil { + t.Errorf("expected result, got %v", err) + } + if !docMatch.IndexInternalID.Equals(index.IndexInternalID("c")) { + t.Errorf("expected result ID to be 'c' got '%s'", docMatch.IndexInternalID) + } + + // try advancing past end + ctx.DocumentMatchPool.Put(docMatch) + docMatch, err = searcher.Advance(ctx, index.IndexInternalID("z")) + if err != nil { + t.Fatal(err) + } + if docMatch != nil { + t.Errorf("expected nil, got %v", docMatch) + } + + // try pushing next past end + ctx.DocumentMatchPool.Put(docMatch) + docMatch, err = searcher.Next(ctx) + if err != nil { + t.Fatal(err) + } + if docMatch != nil { + t.Errorf("expected nil, got %v", docMatch) + } +} diff --git a/search/sort.go b/search/sort.go new file mode 100644 index 0000000..43bfcc3 --- /dev/null +++ b/search/sort.go @@ -0,0 +1,814 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import ( + "bytes" + "encoding/json" + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" + + "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/util" +) + +var ( + HighTerm = strings.Repeat(string(utf8.MaxRune), 3) + LowTerm = string([]byte{0x00}) +) + +type SearchSort interface { + UpdateVisitor(field string, term []byte) + Value(a *DocumentMatch) string + DecodeValue(value string) string + Descending() bool + + RequiresDocID() bool + RequiresScoring() bool + RequiresFields() []string + + Reverse() + + Copy() SearchSort +} + +func ParseSearchSortObj(input map[string]interface{}) (SearchSort, error) { + descending, ok := input["desc"].(bool) + if !ok { + descending = false + } + + by, ok := input["by"].(string) + if !ok { + return nil, fmt.Errorf("search sort must specify by") + } + + switch by { + case "id": + return &SortDocID{ + Desc: descending, + }, nil + case "score": + return &SortScore{ + Desc: descending, + }, nil + case "geo_distance": + field, ok := input["field"].(string) + if !ok { + return nil, fmt.Errorf("search sort mode geo_distance must specify field") + } + lon, lat, foundLocation := geo.ExtractGeoPoint(input["location"]) + if !foundLocation { + return nil, fmt.Errorf("unable to parse geo_distance location") + } + rvd := &SortGeoDistance{ + Field: field, + Desc: descending, + Lon: lon, + Lat: lat, + unitMult: 1.0, + } + if distUnit, ok := input["unit"].(string); ok { + var err error + rvd.unitMult, err = geo.ParseDistanceUnit(distUnit) + if err != nil { + return nil, err + } + rvd.Unit = distUnit + } + return rvd, nil + case "field": + field, ok := input["field"].(string) + if !ok { + return nil, fmt.Errorf("search sort mode field must specify field") + } + rv := &SortField{ + Field: field, + Desc: descending, + } + typ, ok := input["type"].(string) + if ok { + switch typ { + case "auto": + rv.Type = SortFieldAuto + case "string": + rv.Type = SortFieldAsString + case "number": + rv.Type = SortFieldAsNumber + case "date": + rv.Type = SortFieldAsDate + default: + return nil, fmt.Errorf("unknown sort field type: %s", typ) + } + } + mode, ok := input["mode"].(string) + if ok { + switch mode { + case "default": + rv.Mode = SortFieldDefault + case "min": + rv.Mode = SortFieldMin + case "max": + rv.Mode = SortFieldMax + default: + return nil, fmt.Errorf("unknown sort field mode: %s", mode) + } + } + missing, ok := input["missing"].(string) + if ok { + switch missing { + case "first": + rv.Missing = SortFieldMissingFirst + case "last": + rv.Missing = SortFieldMissingLast + default: + return nil, fmt.Errorf("unknown sort field missing: %s", missing) + } + } + return rv, nil + } + + return nil, fmt.Errorf("unknown search sort by: %s", by) +} + +func ParseSearchSortString(input string) SearchSort { + descending := false + if strings.HasPrefix(input, "-") { + descending = true + input = input[1:] + } else if strings.HasPrefix(input, "+") { + input = input[1:] + } + + switch input { + case "_id": + return &SortDocID{ + Desc: descending, + } + case "_score": + return &SortScore{ + Desc: descending, + } + } + + return &SortField{ + Field: input, + Desc: descending, + } +} + +func ParseSearchSortJSON(input json.RawMessage) (SearchSort, error) { + // first try to parse it as string + var sortString string + err := util.UnmarshalJSON(input, &sortString) + if err != nil { + var sortObj map[string]interface{} + err = util.UnmarshalJSON(input, &sortObj) + if err != nil { + return nil, err + } + return ParseSearchSortObj(sortObj) + } + return ParseSearchSortString(sortString), nil +} + +func ParseSortOrderStrings(in []string) SortOrder { + rv := make(SortOrder, 0, len(in)) + for _, i := range in { + ss := ParseSearchSortString(i) + rv = append(rv, ss) + } + return rv +} + +func ParseSortOrderJSON(in []json.RawMessage) (SortOrder, error) { + rv := make(SortOrder, 0, len(in)) + for _, i := range in { + ss, err := ParseSearchSortJSON(i) + if err != nil { + return nil, err + } + rv = append(rv, ss) + } + return rv, nil +} + +type SortOrder []SearchSort + +func (so SortOrder) Value(doc *DocumentMatch) { + for _, soi := range so { + value := soi.Value(doc) + doc.Sort = append(doc.Sort, value) + doc.DecodedSort = append(doc.DecodedSort, soi.DecodeValue(value)) + } +} + +func (so SortOrder) UpdateVisitor(field string, term []byte) { + for _, soi := range so { + soi.UpdateVisitor(field, term) + } +} + +func (so SortOrder) Copy() SortOrder { + rv := make(SortOrder, len(so)) + for i, soi := range so { + rv[i] = soi.Copy() + } + return rv +} + +// Compare will compare two document matches using the specified sort order +// if both are numbers, we avoid converting back to term +func (so SortOrder) Compare(cachedScoring, cachedDesc []bool, i, j *DocumentMatch) int { + // compare the documents on all search sorts until a differences is found + for x := range so { + c := 0 + if cachedScoring[x] { + if i.Score < j.Score { + c = -1 + } else if i.Score > j.Score { + c = 1 + } + } else { + iVal := i.Sort[x] + jVal := j.Sort[x] + if iVal < jVal { + c = -1 + } else if iVal > jVal { + c = 1 + } + } + + if c == 0 { + continue + } + if cachedDesc[x] { + c = -c + } + return c + } + // if they are the same at this point, impose order based on index natural sort order + if i.HitNumber == j.HitNumber { + return 0 + } else if i.HitNumber > j.HitNumber { + return 1 + } + return -1 +} + +func CompareScoreDescending(i, j *DocumentMatch) int { + // first try to sort the two hits based on their score value + if i.Score < j.Score { + return 1 + } + if i.Score > j.Score { + return -1 + } + // tie-break on natural index order: earlier hit sorts first + if i.HitNumber > j.HitNumber { + return 1 + } + if i.HitNumber < j.HitNumber { + return -1 + } + return 0 +} + +func (so SortOrder) RequiresScore() bool { + for _, soi := range so { + if soi.RequiresScoring() { + return true + } + } + return false +} + +func (so SortOrder) RequiresDocID() bool { + for _, soi := range so { + if soi.RequiresDocID() { + return true + } + } + return false +} + +func (so SortOrder) RequiredFields() []string { + var rv []string + for _, soi := range so { + rv = append(rv, soi.RequiresFields()...) + } + return rv +} + +func (so SortOrder) CacheIsScore() []bool { + rv := make([]bool, 0, len(so)) + for _, soi := range so { + rv = append(rv, soi.RequiresScoring()) + } + return rv +} + +func (so SortOrder) CacheDescending() []bool { + rv := make([]bool, 0, len(so)) + for _, soi := range so { + rv = append(rv, soi.Descending()) + } + return rv +} + +func (so SortOrder) Reverse() { + for _, soi := range so { + soi.Reverse() + } +} + +// SortFieldType lets you control some internal sort behavior +// normally leaving this to the zero-value of SortFieldAuto is fine +type SortFieldType int + +const ( + // SortFieldAuto applies heuristics attempt to automatically sort correctly + SortFieldAuto SortFieldType = iota + // SortFieldAsString forces sort as string (no prefix coded terms removed) + SortFieldAsString + // SortFieldAsNumber forces sort as string (prefix coded terms with shift > 0 removed) + SortFieldAsNumber + // SortFieldAsDate forces sort as string (prefix coded terms with shift > 0 removed) + SortFieldAsDate +) + +// SortFieldMode describes the behavior if the field has multiple values +type SortFieldMode int + +const ( + // SortFieldDefault uses the first (or only) value, this is the default zero-value + SortFieldDefault SortFieldMode = iota // FIXME name is confusing + // SortFieldMin uses the minimum value + SortFieldMin + // SortFieldMax uses the maximum value + SortFieldMax +) + +// SortFieldMissing controls where documents missing a field value should be sorted +type SortFieldMissing int + +const ( + // SortFieldMissingLast sorts documents missing a field at the end + SortFieldMissingLast SortFieldMissing = iota + + // SortFieldMissingFirst sorts documents missing a field at the beginning + SortFieldMissingFirst +) + +// SortField will sort results by the value of a stored field +// +// Field is the name of the field +// Descending reverse the sort order (default false) +// Type allows forcing of string/number/date behavior (default auto) +// Mode controls behavior for multi-values fields (default first) +// Missing controls behavior of missing values (default last) +type SortField struct { + Field string + Desc bool + Type SortFieldType + Mode SortFieldMode + Missing SortFieldMissing + values [][]byte + tmp [][]byte +} + +// UpdateVisitor notifies this sort field that in this document +// this field has the specified term +func (s *SortField) UpdateVisitor(field string, term []byte) { + if field == s.Field { + s.values = append(s.values, term) + } +} + +// Value returns the sort value of the DocumentMatch +// it also resets the state of this SortField for +// processing the next document +func (s *SortField) Value(i *DocumentMatch) string { + iTerms := s.filterTermsByType(s.values) + iTerm := s.filterTermsByMode(iTerms) + s.values = s.values[:0] + return iTerm +} + +func (s *SortField) DecodeValue(value string) string { + switch s.Type { + case SortFieldAsNumber: + i64, err := numeric.PrefixCoded(value).Int64() + if err != nil { + return value + } + return strconv.FormatFloat(numeric.Int64ToFloat64(i64), 'f', -1, 64) + case SortFieldAsDate: + i64, err := numeric.PrefixCoded(value).Int64() + if err != nil { + return value + } + return time.Unix(0, i64).UTC().Format(time.RFC3339Nano) + default: + return value + } +} + +// Descending determines the order of the sort +func (s *SortField) Descending() bool { + return s.Desc +} + +func (s *SortField) filterTermsByMode(terms [][]byte) string { + if len(terms) == 1 || (len(terms) > 1 && s.Mode == SortFieldDefault) { + return string(terms[0]) + } else if len(terms) > 1 { + switch s.Mode { + case SortFieldMin: + sort.Sort(BytesSlice(terms)) + return string(terms[0]) + case SortFieldMax: + sort.Sort(BytesSlice(terms)) + return string(terms[len(terms)-1]) + } + } + + // handle missing terms + if s.Missing == SortFieldMissingLast { + if s.Desc { + return LowTerm + } + return HighTerm + } + if s.Desc { + return HighTerm + } + return LowTerm +} + +// filterTermsByType attempts to make one pass on the terms +// if we are in auto-mode AND all the terms look like prefix-coded numbers +// return only the terms which had shift of 0 +// if we are in explicit number or date mode, return only valid +// prefix coded numbers with shift of 0 +func (s *SortField) filterTermsByType(terms [][]byte) [][]byte { + stype := s.Type + + switch stype { + case SortFieldAuto: + allTermsPrefixCoded := true + termsWithShiftZero := s.tmp[:0] + for _, term := range terms { + valid, shift := numeric.ValidPrefixCodedTermBytes(term) + if valid && shift == 0 { + termsWithShiftZero = append(termsWithShiftZero, term) + } else if !valid { + allTermsPrefixCoded = false + } + } + // reset the terms only when valid zero shift terms are found. + if allTermsPrefixCoded && len(termsWithShiftZero) > 0 { + terms = termsWithShiftZero + s.tmp = termsWithShiftZero[:0] + } + case SortFieldAsNumber, SortFieldAsDate: + termsWithShiftZero := s.tmp[:0] + for _, term := range terms { + valid, shift := numeric.ValidPrefixCodedTermBytes(term) + if valid && shift == 0 { + termsWithShiftZero = append(termsWithShiftZero, term) + } + } + terms = termsWithShiftZero + s.tmp = termsWithShiftZero[:0] + } + + return terms +} + +// RequiresDocID says this SearchSort does not require the DocID be loaded +func (s *SortField) RequiresDocID() bool { return false } + +// RequiresScoring says this SearchStore does not require scoring +func (s *SortField) RequiresScoring() bool { return false } + +// RequiresFields says this SearchStore requires the specified stored field +func (s *SortField) RequiresFields() []string { return []string{s.Field} } + +func (s *SortField) MarshalJSON() ([]byte, error) { + // see if simple format can be used + if s.Missing == SortFieldMissingLast && + s.Mode == SortFieldDefault && + s.Type == SortFieldAuto { + if s.Desc { + return json.Marshal("-" + s.Field) + } + return json.Marshal(s.Field) + } + sfm := map[string]interface{}{ + "by": "field", + "field": s.Field, + } + if s.Desc { + sfm["desc"] = true + } + if s.Missing > SortFieldMissingLast { + switch s.Missing { + case SortFieldMissingFirst: + sfm["missing"] = "first" + } + } + if s.Mode > SortFieldDefault { + switch s.Mode { + case SortFieldMin: + sfm["mode"] = "min" + case SortFieldMax: + sfm["mode"] = "max" + } + } + if s.Type > SortFieldAuto { + switch s.Type { + case SortFieldAsString: + sfm["type"] = "string" + case SortFieldAsNumber: + sfm["type"] = "number" + case SortFieldAsDate: + sfm["type"] = "date" + } + } + + return json.Marshal(sfm) +} + +func (s *SortField) Copy() SearchSort { + rv := *s + return &rv +} + +func (s *SortField) Reverse() { + s.Desc = !s.Desc + if s.Missing == SortFieldMissingFirst { + s.Missing = SortFieldMissingLast + } else { + s.Missing = SortFieldMissingFirst + } +} + +// SortDocID will sort results by the document identifier +type SortDocID struct { + Desc bool +} + +// UpdateVisitor is a no-op for SortDocID as it's value +// is not dependent on any field terms +func (s *SortDocID) UpdateVisitor(field string, term []byte) { +} + +// Value returns the sort value of the DocumentMatch +func (s *SortDocID) Value(i *DocumentMatch) string { + return i.ID +} + +func (s *SortDocID) DecodeValue(value string) string { + return value +} + +// Descending determines the order of the sort +func (s *SortDocID) Descending() bool { + return s.Desc +} + +// RequiresDocID says this SearchSort does require the DocID be loaded +func (s *SortDocID) RequiresDocID() bool { return true } + +// RequiresScoring says this SearchStore does not require scoring +func (s *SortDocID) RequiresScoring() bool { return false } + +// RequiresFields says this SearchStore does not require any stored fields +func (s *SortDocID) RequiresFields() []string { return nil } + +func (s *SortDocID) MarshalJSON() ([]byte, error) { + if s.Desc { + return json.Marshal("-_id") + } + return json.Marshal("_id") +} + +func (s *SortDocID) Copy() SearchSort { + rv := *s + return &rv +} + +func (s *SortDocID) Reverse() { + s.Desc = !s.Desc +} + +// SortScore will sort results by the document match score +type SortScore struct { + Desc bool +} + +// UpdateVisitor is a no-op for SortScore as it's value +// is not dependent on any field terms +func (s *SortScore) UpdateVisitor(field string, term []byte) { +} + +// Value returns the sort value of the DocumentMatch +func (s *SortScore) Value(i *DocumentMatch) string { + return "_score" +} + +func (s *SortScore) DecodeValue(value string) string { + return value +} + +// Descending determines the order of the sort +func (s *SortScore) Descending() bool { + return s.Desc +} + +// RequiresDocID says this SearchSort does not require the DocID be loaded +func (s *SortScore) RequiresDocID() bool { return false } + +// RequiresScoring says this SearchStore does require scoring +func (s *SortScore) RequiresScoring() bool { return true } + +// RequiresFields says this SearchStore does not require any store fields +func (s *SortScore) RequiresFields() []string { return nil } + +func (s *SortScore) MarshalJSON() ([]byte, error) { + if s.Desc { + return json.Marshal("-_score") + } + return json.Marshal("_score") +} + +func (s *SortScore) Copy() SearchSort { + rv := *s + return &rv +} + +func (s *SortScore) Reverse() { + s.Desc = !s.Desc +} + +var maxDistance = string(numeric.MustNewPrefixCodedInt64(math.MaxInt64, 0)) + +// NewSortGeoDistance creates SearchSort instance for sorting documents by +// their distance from the specified point. +func NewSortGeoDistance(field, unit string, lon, lat float64, desc bool) ( + *SortGeoDistance, error, +) { + rv := &SortGeoDistance{ + Field: field, + Desc: desc, + Unit: unit, + Lon: lon, + Lat: lat, + } + var err error + rv.unitMult, err = geo.ParseDistanceUnit(unit) + if err != nil { + return nil, err + } + return rv, nil +} + +// SortGeoDistance will sort results by the distance of an +// indexed geo point, from the provided location. +// +// Field is the name of the field +// Descending reverse the sort order (default false) +type SortGeoDistance struct { + Field string + Desc bool + Unit string + values [][]byte + Lon float64 + Lat float64 + unitMult float64 + tmp []byte +} + +// UpdateVisitor notifies this sort field that in this document +// this field has the specified term +func (s *SortGeoDistance) UpdateVisitor(field string, term []byte) { + if field == s.Field { + s.values = append(s.values, term) + } +} + +// Value returns the sort value of the DocumentMatch +// it also resets the state of this SortGeoDistance for +// processing the next document +func (s *SortGeoDistance) Value(i *DocumentMatch) string { + iTerm := s.findPrefixCodedNumericTerm(s.values) + s.values = s.values[:0] + + if iTerm == nil { + return maxDistance + } + + i64, err := numeric.PrefixCoded(iTerm).Int64() + if err != nil { + return maxDistance + } + docLon := geo.MortonUnhashLon(uint64(i64)) + docLat := geo.MortonUnhashLat(uint64(i64)) + + dist := geo.Haversin(s.Lon, s.Lat, docLon, docLat) + // dist is returned in km, so convert to m + dist *= 1000 + if s.unitMult != 0 { + dist /= s.unitMult + } + distInt64 := numeric.Float64ToInt64(dist) + s.tmp = numeric.MustNewPrefixCodedInt64Prealloc(distInt64, 0, s.tmp) + return string(s.tmp) +} + +func (s *SortGeoDistance) DecodeValue(value string) string { + distInt, err := numeric.PrefixCoded(value).Int64() + if err != nil { + return "" + } + return strconv.FormatFloat(numeric.Int64ToFloat64(distInt), 'f', -1, 64) +} + +// Descending determines the order of the sort +func (s *SortGeoDistance) Descending() bool { + return s.Desc +} + +// findPrefixCodedNumericTerm looks through the provided terms +// and returns the first valid prefix coded numeric term with shift of 0 +func (s *SortGeoDistance) findPrefixCodedNumericTerm(terms [][]byte) []byte { + for _, term := range terms { + valid, shift := numeric.ValidPrefixCodedTermBytes(term) + if valid && shift == 0 { + return term + } + } + return nil +} + +// RequiresDocID says this SearchSort does not require the DocID be loaded +func (s *SortGeoDistance) RequiresDocID() bool { return false } + +// RequiresScoring says this SearchStore does not require scoring +func (s *SortGeoDistance) RequiresScoring() bool { return false } + +// RequiresFields says this SearchStore requires the specified stored field +func (s *SortGeoDistance) RequiresFields() []string { return []string{s.Field} } + +func (s *SortGeoDistance) MarshalJSON() ([]byte, error) { + sfm := map[string]interface{}{ + "by": "geo_distance", + "field": s.Field, + "location": map[string]interface{}{ + "lon": s.Lon, + "lat": s.Lat, + }, + } + if s.Unit != "" { + sfm["unit"] = s.Unit + } + if s.Desc { + sfm["desc"] = true + } + + return json.Marshal(sfm) +} + +func (s *SortGeoDistance) Copy() SearchSort { + rv := *s + return &rv +} + +func (s *SortGeoDistance) Reverse() { + s.Desc = !s.Desc +} + +type BytesSlice [][]byte + +func (p BytesSlice) Len() int { return len(p) } +func (p BytesSlice) Less(i, j int) bool { return bytes.Compare(p[i], p[j]) < 0 } +func (p BytesSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/search/sort_test.go b/search/sort_test.go new file mode 100644 index 0000000..009444b --- /dev/null +++ b/search/sort_test.go @@ -0,0 +1,338 @@ +package search + +import ( + "reflect" + "testing" +) + +func TestParseSearchSortObj(t *testing.T) { + tests := []struct { + name string + input map[string]interface{} + want SearchSort + wantErr bool + }{ + { + name: "sort by id", + input: map[string]interface{}{ + "by": "id", + "desc": false, + }, + want: &SortDocID{ + Desc: false, + }, + wantErr: false, + }, + { + name: "sort by id descending", + input: map[string]interface{}{ + "by": "id", + "desc": true, + }, + want: &SortDocID{ + Desc: true, + }, + wantErr: false, + }, + { + name: "sort by score", + input: map[string]interface{}{ + "by": "score", + "desc": false, + }, + want: &SortScore{ + Desc: false, + }, + wantErr: false, + }, + { + name: "sort by score descending", + input: map[string]interface{}{ + "by": "score", + "desc": true, + }, + want: &SortScore{ + Desc: true, + }, + wantErr: false, + }, + { + name: "sort by geo_distance", + input: map[string]interface{}{ + "by": "geo_distance", + "field": "location", + "location": map[string]interface{}{ + "lon": 1.0, + "lat": 2.0, + }, + "unit": "km", + "desc": false, + }, + want: &SortGeoDistance{ + Field: "location", + Desc: false, + Lon: 1.0, + Lat: 2.0, + Unit: "km", + unitMult: 1000.0, + }, + wantErr: false, + }, + { + name: "sort by field", + input: map[string]interface{}{ + "by": "field", + "field": "name", + "desc": false, + "type": "auto", + "mode": "default", + "missing": "last", + }, + want: &SortField{ + Field: "name", + Desc: false, + Type: SortFieldAuto, + Mode: SortFieldDefault, + Missing: SortFieldMissingLast, + }, + wantErr: false, + }, + { + name: "sort by field with missing", + input: map[string]interface{}{ + "by": "field", + "field": "name", + "desc": false, + "type": "auto", + "mode": "default", + "missing": "first", + }, + want: &SortField{ + Field: "name", + Desc: false, + Type: SortFieldAuto, + Mode: SortFieldDefault, + Missing: SortFieldMissingFirst, + }, + wantErr: false, + }, + { + name: "sort by field descending", + input: map[string]interface{}{ + "by": "field", + "field": "name", + "desc": true, + "type": "string", + "mode": "min", + "missing": "first", + }, + want: &SortField{ + Field: "name", + Desc: true, + Type: SortFieldAsString, + Mode: SortFieldMin, + Missing: SortFieldMissingFirst, + }, + wantErr: false, + }, + { + name: "missing by", + input: map[string]interface{}{ + "desc": true, + }, + want: nil, + wantErr: true, + }, + { + name: "unknown by", + input: map[string]interface{}{ + "by": "unknown", + }, + want: nil, + wantErr: true, + }, + { + name: "missing field for geo_distance", + input: map[string]interface{}{ + "by": "geo_distance", + "location": map[string]interface{}{ + "lon": 1.0, + "lat": 2.0, + }, + }, + want: nil, + wantErr: true, + }, + { + name: "missing location for geo_distance", + input: map[string]interface{}{ + "by": "geo_distance", + "field": "location", + }, + want: nil, + wantErr: true, + }, + { + name: "invalid unit for geo_distance", + input: map[string]interface{}{ + "by": "geo_distance", + "field": "location", + "location": map[string]interface{}{ + "lon": 1.0, + "lat": 2.0, + }, + "unit": "invalid", + }, + want: nil, + wantErr: true, + }, + { + name: "missing field for field sort", + input: map[string]interface{}{ + "by": "field", + }, + want: nil, + wantErr: true, + }, + { + name: "unknown type for field sort", + input: map[string]interface{}{ + "by": "field", + "field": "name", + "type": "unknown", + }, + want: nil, + wantErr: true, + }, + { + name: "number type for field sort with desc", + input: map[string]interface{}{ + "by": "field", + "field": "name", + "type": "number", + "mode": "default", + "desc": true, + "missing": "last", + }, + want: &SortField{ + Field: "name", + Desc: true, + Type: SortFieldAsNumber, + Mode: SortFieldDefault, + Missing: SortFieldMissingLast, + }, + wantErr: false, + }, + { + name: "date type for field sort with desc", + input: map[string]interface{}{ + "by": "field", + "field": "name", + "type": "date", + "mode": "default", + "desc": true, + "missing": "last", + }, + want: &SortField{ + Field: "name", + Desc: true, + Type: SortFieldAsDate, + Mode: SortFieldDefault, + Missing: SortFieldMissingLast, + }, + wantErr: false, + }, + { + name: "unknown type for field sort with missing", + input: map[string]interface{}{ + "by": "field", + "field": "name", + "type": "unknown", + "mode": "default", + "missing": "last", + }, + want: nil, + wantErr: true, + }, + { + name: "unknown mode for field sort", + input: map[string]interface{}{ + "by": "field", + "field": "name", + "mode": "unknown", + }, + want: nil, + wantErr: true, + }, + { + name: "default mode for field sort", + input: map[string]interface{}{ + "by": "field", + "field": "name", + "mode": "default", + }, + want: &SortField{ + Field: "name", + Desc: false, + Type: SortFieldAuto, + Mode: SortFieldDefault, + Missing: SortFieldMissingLast, + }, + wantErr: false, + }, + { + name: "max mode for field sort", + input: map[string]interface{}{ + "by": "field", + "field": "name", + "mode": "max", + }, + want: &SortField{ + Field: "name", + Desc: false, + Type: SortFieldAuto, + Mode: SortFieldMax, + Missing: SortFieldMissingLast, + }, + wantErr: false, + }, + { + name: "min mode for field sort", + input: map[string]interface{}{ + "by": "field", + "field": "name", + "mode": "min", + }, + want: &SortField{ + Field: "name", + Desc: false, + Type: SortFieldAuto, + Mode: SortFieldMin, + Missing: SortFieldMissingLast, + }, + wantErr: false, + }, + { + name: "unknown missing for field sort", + input: map[string]interface{}{ + "by": "field", + "field": "name", + "missing": "unknown", + }, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseSearchSortObj(tt.input) + if (err != nil) != tt.wantErr { + t.Errorf("ParseSearchSortObj() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseSearchSortObj() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/search/term_limit.go b/search/term_limit.go new file mode 100644 index 0000000..331b787 --- /dev/null +++ b/search/term_limit.go @@ -0,0 +1,63 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import ( + "context" + "fmt" + "sync/atomic" +) + +// termSearchersCounter tracks the running total of leaf term searchers created +// while searching a single index and the limit beyond which construction fails. +type termSearchersCounter struct { + limit int + count atomic.Uint64 +} + +// ContextWithTermSearchersCounter installs a fresh counter for the limit at +// MaxTermSearchersKey, when that limit is positive. It is called once per index +// search (indexImpl.SearchInContext), so the counter is scoped to that index: +// within a single index it spans the whole query tree (all fan-out nodes share +// one counter), but across an IndexAlias each index is counted independently. +// context is returned unchanged and RecordTermSearcher stays a no-op. +func ContextWithTermSearchersCounter(ctx context.Context) context.Context { + if ctx == nil { + return ctx + } + limit, ok := ctx.Value(MaxTermSearchersKey).(int) + if !ok || limit <= 0 { + return ctx + } + return context.WithValue(ctx, termSearchersCounterKey, + &termSearchersCounter{limit: limit}) +} + +// RecordTermSearcher counts one leaf term searcher, returning an error once the +// running total exceeds the limit. It is a no-op when no counter was installed. +func RecordTermSearcher(ctx context.Context) error { + if ctx == nil { + return nil + } + c, ok := ctx.Value(termSearchersCounterKey).(*termSearchersCounter) + if !ok || c == nil { + return nil + } + if n := c.count.Add(1); n > uint64(c.limit) { + return fmt.Errorf("TooManyTermSearchers [%d > bleveMaxTerms,"+ + " which is set to %d]", n, c.limit) + } + return nil +} diff --git a/search/term_limit_test.go b/search/term_limit_test.go new file mode 100644 index 0000000..5d0b92d --- /dev/null +++ b/search/term_limit_test.go @@ -0,0 +1,74 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import ( + "context" + "testing" +) + +func TestRecordTermSearcher(t *testing.T) { + // No limit set: RecordTermSearcher must always be a no-op. + t.Run("no limit set", func(t *testing.T) { + ctx := ContextWithTermSearchersCounter(context.Background()) + for i := 0; i < 1000; i++ { + if err := RecordTermSearcher(ctx); err != nil { + t.Fatalf("unexpected error with no limit at i=%d: %v", i, err) + } + } + }) + + // Non-positive limit disables the check. + for _, limit := range []int{0, -5} { + t.Run("disabled limit", func(t *testing.T) { + ctx := context.WithValue(context.Background(), MaxTermSearchersKey, limit) + ctx = ContextWithTermSearchersCounter(ctx) + for i := 0; i < 100; i++ { + if err := RecordTermSearcher(ctx); err != nil { + t.Fatalf("limit %d should disable check, got err at i=%d: %v", + limit, i, err) + } + } + }) + } + + // Positive limit: exactly `limit` leaf term searchers are allowed; the + // one that pushes the total over the limit fails. + t.Run("limit enforced", func(t *testing.T) { + const limit = 3 + ctx := context.WithValue(context.Background(), MaxTermSearchersKey, limit) + ctx = ContextWithTermSearchersCounter(ctx) + for i := 1; i <= limit; i++ { + if err := RecordTermSearcher(ctx); err != nil { + t.Fatalf("term searcher %d/%d should be allowed, got: %v", i, limit, err) + } + } + if err := RecordTermSearcher(ctx); err == nil { + t.Fatalf("term searcher %d should exceed limit %d, got nil error", + limit+1, limit) + } + }) + + // A nil context is tolerated. + t.Run("nil context", func(t *testing.T) { + if err := RecordTermSearcher(nil); err != nil { + t.Fatalf("nil ctx should be a no-op, got: %v", err) + } + //nolint:staticcheck // deliberately exercising nil-ctx tolerance + if got := ContextWithTermSearchersCounter(nil); got != nil { + t.Fatalf("nil ctx should be returned unchanged") + } + }) +} diff --git a/search/util.go b/search/util.go new file mode 100644 index 0000000..d49f72a --- /dev/null +++ b/search/util.go @@ -0,0 +1,317 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import ( + "context" + + "github.com/blevesearch/geo/s2" +) + +func MergeLocations(locations []FieldTermLocationMap) FieldTermLocationMap { + rv := locations[0] + + for i := 1; i < len(locations); i++ { + nextLocations := locations[i] + for field, termLocationMap := range nextLocations { + rvTermLocationMap, rvHasField := rv[field] + if rvHasField { + rv[field] = MergeTermLocationMaps(rvTermLocationMap, termLocationMap) + } else { + rv[field] = termLocationMap + } + } + } + + return rv +} + +func MergeTermLocationMaps(rv, other TermLocationMap) TermLocationMap { + for term, locationMap := range other { + // for a given term/document there cannot be different locations + // if they came back from different clauses, overwrite is ok + rv[term] = locationMap + } + return rv +} + +func MergeFieldTermLocations(dest []FieldTermLocation, matches []*DocumentMatch) []FieldTermLocation { + n := len(dest) + for _, dm := range matches { + if dm != nil { + n += len(dm.FieldTermLocations) + } + } + if n == len(dest) { + return dest + } + if cap(dest) < n { + dest = append(make([]FieldTermLocation, 0, n), dest...) + } + + for _, dm := range matches { + if dm != nil { + dest = mergeFieldTermLocationFromMatch(dest, dm) + } + } + + return dest +} + +// MergeFieldTermLocationsFromMatch merges field term locations from a single DocumentMatch +// into dest, returning the updated slice. +func MergeFieldTermLocationsFromMatch(dest []FieldTermLocation, match *DocumentMatch) []FieldTermLocation { + if match == nil { + return dest + } + n := len(dest) + len(match.FieldTermLocations) + if cap(dest) < n { + dest = append(make([]FieldTermLocation, 0, n), dest...) + } + return mergeFieldTermLocationFromMatch(dest, match) +} + +// mergeFieldTermLocationFromMatch appends field term locations from a DocumentMatch into dest. +// Assumes dest has sufficient capacity. +func mergeFieldTermLocationFromMatch(dest []FieldTermLocation, dm *DocumentMatch) []FieldTermLocation { + for _, ftl := range dm.FieldTermLocations { + dest = append(dest, FieldTermLocation{ + Field: ftl.Field, + Term: ftl.Term, + Location: Location{ + Pos: ftl.Location.Pos, + Start: ftl.Location.Start, + End: ftl.Location.End, + ArrayPositions: append(ArrayPositions(nil), ftl.Location.ArrayPositions...), + }, + }) + } + + return dest +} + +type ( + SearchIncrementalCostCallbackMsg uint + SearchQueryType uint +) + +const ( + Term = SearchQueryType(1 << iota) + Geo + Numeric + GenericCost +) + +const ( + AddM = SearchIncrementalCostCallbackMsg(1 << iota) + AbortM + DoneM +) + +// ContextKey is used to identify the context key in the context.Context +type ContextKey string + +func (c ContextKey) String() string { + return string(c) +} + +const ( + SearchIncrementalCostKey ContextKey = "_search_incremental_cost_key" + QueryTypeKey ContextKey = "_query_type_key" + FuzzyMatchPhraseKey ContextKey = "_fuzzy_match_phrase_key" + IncludeScoreBreakdownKey ContextKey = "_include_score_breakdown_key" + + // PreSearchKey indicates whether to perform a preliminary search to gather necessary + // information which would be used in the actual search down the line. + PreSearchKey ContextKey = "_presearch_key" + + // GetScoringModelCallbackKey is used to help the underlying searcher identify + // which scoring mechanism to use based on index mapping. + GetScoringModelCallbackKey ContextKey = "_get_scoring_model" + + // SearchIOStatsCallbackKey is used to help the underlying searcher identify + SearchIOStatsCallbackKey ContextKey = "_search_io_stats_callback_key" + + // GeoBufferPoolCallbackKey ContextKey is used to help the underlying searcher + GeoBufferPoolCallbackKey ContextKey = "_geo_buffer_pool_callback_key" + + // SearchTypeKey is used to identify type of the search being performed. + // + // for consistent scoring in cases an index is partitioned/sharded (using an + // index alias), GlobalScoring helps in aggregating the necessary stats across + // all the child bleve indexes (shards/partitions) first before the actual search + // is performed, such that the scoring involved using these stats would be at a + // global level. + SearchTypeKey ContextKey = "_search_type_key" + + // The following keys are used to invoke the callbacks at the start and end stages + // of optimizing the disjunction/conjunction searcher creation. + SearcherStartCallbackKey ContextKey = "_searcher_start_callback_key" + SearcherEndCallbackKey ContextKey = "_searcher_end_callback_key" + + // FieldTermSynonymMapKey is used to store and transport the synonym definitions data + // to the actual search phase which would use the synonyms to perform the search. + FieldTermSynonymMapKey ContextKey = "_field_term_synonym_map_key" + + // BM25StatsKey is used to store and transport the BM25 Data + // to the actual search phase which would use it to perform the search. + BM25StatsKey ContextKey = "_bm25_stats_key" + + // ScoreFusionKey is used to communicate whether KNN hits need to be preserved for + // hybrid search algorithms (like RRF) + ScoreFusionKey ContextKey = "_fusion_rescoring_key" + + // NestedSearchKey is used to communicate whether the search is performed + // in an index with nested documents + NestedSearchKey ContextKey = "_nested_search_key" + + // MaxTermSearchersKey carries an application-set int limit on the total + // number of leaf term searchers created for a single search; <= 0 disables + // the check. Unlike DisjunctionMaxClauseCount, which is per fan-out node, + // this bounds the count across the whole query tree. + MaxTermSearchersKey ContextKey = "_max_term_searchers_key" + + // termSearchersCounterKey carries the per-search counter installed by + // ContextWithTermSearchersCounter. + termSearchersCounterKey ContextKey = "_term_searchers_counter_key" +) + +func RecordSearchCost(ctx context.Context, + msg SearchIncrementalCostCallbackMsg, bytes uint64, +) { + if ctx != nil { + queryType, ok := ctx.Value(QueryTypeKey).(SearchQueryType) + if !ok { + // for the cost of the non query type specific factors such as + // doc values and stored fields section. + queryType = GenericCost + } + + aggCallbackFn := ctx.Value(SearchIncrementalCostKey) + if aggCallbackFn != nil { + aggCallbackFn.(SearchIncrementalCostCallbackFn)(msg, queryType, bytes) + } + } +} + +// Assigning the size of the largest buffer in the pool to 24KB and +// the smallest buffer to 24 bytes. The pools are used to read a +// sequence of vertices which are always 24 bytes each. +const ( + MaxGeoBufPoolSize = 24 * 1024 + MinGeoBufPoolSize = 24 +) + +// PreSearchDataKey are used to store the data gathered during the presearch phase +// which would be use in the actual search phase. +const ( + KnnPreSearchDataKey = "_knn_pre_search_data_key" + SynonymPreSearchDataKey = "_synonym_pre_search_data_key" + BM25PreSearchDataKey = "_bm25_pre_search_data_key" +) + +const GlobalScoring = "_global_scoring" + +type ( + // SearcherStartCallbackFn is a callback function type used to signal the start of + // searcher creation phase. + SearcherStartCallbackFn func(size uint64) error + // SearcherEndCallbackFn is a callback function type used to signal the end of + // a searcher creation phase. + SearcherEndCallbackFn func(size uint64) error + // GetScoringModelCallbackFn is a callback function type used to get the scoring model + // to be used for scoring documents during search. + GetScoringModelCallbackFn func() string + // HybridMergeCallbackFn is a callback function type used to merge a KNN document match + // into a full text search document match, of the same docID as part of hybrid search. + HybridMergeCallbackFn func(ftsMatch *DocumentMatch, knnMatch *DocumentMatch) + // DescendantAdderCallback is a callback function type used to customize how a descendant + // DocumentMatch is merged into its parent. This allows different descendant addition strategies for + // different use cases (e.g., TopN vs KNN collection). + DescendantAdderCallbackFn func(parent *DocumentMatch, descendant *DocumentMatch) error + // GeoBufferPoolCallbackFunc is a callback function type used to get the geo buffer pool + // to be used during geo searches. + GeoBufferPoolCallbackFunc func() *s2.GeoBufferPool + // SearchIOStatsCallbackFunc is a callback function type used to report search IO stats + // during search. + SearchIOStatsCallbackFunc func(uint64) + // Implementation of SearchIncrementalCostCallbackFn should handle the following messages + // - add: increment the cost of a search operation + // (which can be specific to a query type as well) + // - abort: query was aborted due to a cancel of search's context (for eg), + // which can be handled differently as well + // - done: indicates that a search was complete and the tracked cost can be + // handled safely by the implementation. + SearchIncrementalCostCallbackFn func(SearchIncrementalCostCallbackMsg, + SearchQueryType, uint64) +) + +// field -> term -> synonyms +type FieldTermSynonymMap map[string]map[string][]string + +func (f FieldTermSynonymMap) MergeWith(fts FieldTermSynonymMap) { + for field, termSynonymMap := range fts { + // Ensure the field exists in the receiver + if _, exists := f[field]; !exists { + f[field] = make(map[string][]string) + } + for term, synonyms := range termSynonymMap { + // Append synonyms + f[field][term] = append(f[field][term], synonyms...) + } + } +} + +// BM25 specific multipliers which control the scoring of a document. +// +// BM25_b - controls the extent to which doc's field length normalize term frequency part of score +// BM25_k1 - controls the saturation of the score due to term frequency +// the default values are as per elastic search's implementation +// - https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules-similarity.html#bm25 +// - https://www.elastic.co/blog/practical-bm25-part-3-considerations-for-picking-b-and-k1-in-elasticsearch +var ( + BM25_k1 float64 = 1.2 + BM25_b float64 = 0.75 +) + +type BM25Stats struct { + DocCount float64 `json:"doc_count"` + FieldCardinality map[string]int `json:"field_cardinality"` +} + +// FieldSet represents a set of queried fields. +type FieldSet map[string]struct{} + +// NewFieldSet creates a new FieldSet. +func NewFieldSet() FieldSet { + return make(map[string]struct{}) +} + +// Add adds a field to the set. +func (fs FieldSet) AddField(field string) { + fs[field] = struct{}{} +} + +// HasID returns true if the field set contains the "_id" field. +func (fs FieldSet) HasID() bool { + _, ok := fs["_id"] + return ok +} + +// HasAll returns true if the field set contains the "_all" field. +func (fs FieldSet) HasAll() bool { + _, ok := fs["_all"] + return ok +} diff --git a/search/util_test.go b/search/util_test.go new file mode 100644 index 0000000..f65e3f9 --- /dev/null +++ b/search/util_test.go @@ -0,0 +1,91 @@ +// Copyright (c) 2013 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import ( + "reflect" + "testing" +) + +func TestMergeLocations(t *testing.T) { + flm1 := FieldTermLocationMap{ + "marty": TermLocationMap{ + "name": { + &Location{ + Pos: 1, + Start: 0, + End: 5, + }, + }, + }, + } + + flm2 := FieldTermLocationMap{ + "marty": TermLocationMap{ + "description": { + &Location{ + Pos: 5, + Start: 20, + End: 25, + }, + }, + }, + } + + flm3 := FieldTermLocationMap{ + "josh": TermLocationMap{ + "description": { + &Location{ + Pos: 5, + Start: 20, + End: 25, + }, + }, + }, + } + + expectedMerge := FieldTermLocationMap{ + "marty": TermLocationMap{ + "description": { + &Location{ + Pos: 5, + Start: 20, + End: 25, + }, + }, + "name": { + &Location{ + Pos: 1, + Start: 0, + End: 5, + }, + }, + }, + "josh": TermLocationMap{ + "description": { + &Location{ + Pos: 5, + Start: 20, + End: 25, + }, + }, + }, + } + + mergedLocations := MergeLocations([]FieldTermLocationMap{flm1, flm2, flm3}) + if !reflect.DeepEqual(expectedMerge, mergedLocations) { + t.Errorf("expected %v, got %v", expectedMerge, mergedLocations) + } +} diff --git a/search_knn.go b/search_knn.go new file mode 100644 index 0000000..808cf1c --- /dev/null +++ b/search_knn.go @@ -0,0 +1,683 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package bleve + +import ( + "context" + "encoding/json" + "fmt" + "sort" + + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/collector" + "github.com/blevesearch/bleve/v2/search/query" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +const supportForVectorSearch = true + +type knnOperator string + +// Must be updated only at init +var BleveMaxK = int64(10000) + +type SearchRequest struct { + ClientContextID string `json:"client_context_id,omitempty"` + Query query.Query `json:"query"` + Size int `json:"size"` + From int `json:"from"` + Highlight *HighlightRequest `json:"highlight,omitempty"` + Fields []string `json:"fields,omitempty"` + Facets FacetsRequest `json:"facets,omitempty"` + Explain bool `json:"explain"` + Sort search.SortOrder `json:"sort"` + IncludeLocations bool `json:"includeLocations"` + Score string `json:"score,omitempty"` + SearchAfter []string `json:"search_after,omitempty"` + SearchBefore []string `json:"search_before,omitempty"` + + KNN []*KNNRequest `json:"knn,omitempty"` + KNNOperator knnOperator `json:"knn_operator,omitempty"` + + // PreSearchData will be a map that will be used + // in the second phase of any 2-phase search, to provide additional + // context to the second phase. This is useful in the case of index + // aliases where the first phase will gather the PreSearchData from all + // the indexes in the alias, and the second phase will use that + // PreSearchData to perform the actual search. + // The currently accepted map configuration is: + // + // "_knn_pre_search_data_key": []*search.DocumentMatch + + PreSearchData map[string]interface{} `json:"pre_search_data,omitempty"` + + Params *RequestParams `json:"params,omitempty"` + + sortFunc func(sort.Interface) +} + +// Vector takes precedence over vectorBase64 in case both fields are given +type KNNRequest struct { + Field string `json:"field"` + Vector []float32 `json:"vector"` + VectorBase64 string `json:"vector_base64"` + K int64 `json:"k"` + Boost *query.Boost `json:"boost,omitempty"` + + // Search parameters for the field's vector index part of the segment. + // Value of it depends on the field's backing vector index implementation. + // + // For Faiss IVF index, supported search params are: + // - ivf_nprobe_pct : int // percentage of total clusters to search + // - ivf_max_codes_pct : float // percentage of total vectors to visit to do a query (across all clusters) + // + // Consult go-faiss to know all supported search params + Params json.RawMessage `json:"params"` + + // Filter query to use with kNN pre-filtering. + // Supports pre-filtering with all existing types of query clauses. + FilterQuery query.Query `json:"filter,omitempty"` +} + +func (r *SearchRequest) AddKNN(field string, vector []float32, k int64, boost float64) { + b := query.Boost(boost) + r.KNN = append(r.KNN, &KNNRequest{ + Field: field, + Vector: vector, + K: k, + Boost: &b, + }) +} + +func (r *SearchRequest) AddKNNWithFilter(field string, vector []float32, k int64, + boost float64, filterQuery query.Query) { + b := query.Boost(boost) + r.KNN = append(r.KNN, &KNNRequest{ + Field: field, + Vector: vector, + K: k, + Boost: &b, + FilterQuery: filterQuery, + }) +} + +func (r *SearchRequest) AddKNNOperator(operator knnOperator) { + r.KNNOperator = operator +} + +// UnmarshalJSON deserializes a JSON representation of +// a SearchRequest +func (r *SearchRequest) UnmarshalJSON(input []byte) error { + type tempKNNReq struct { + Field string `json:"field"` + Vector []float32 `json:"vector"` + VectorBase64 string `json:"vector_base64"` + K int64 `json:"k"` + Boost *query.Boost `json:"boost,omitempty"` + Params OptionalRawMessage `json:"params"` + FilterQuery OptionalRawMessage `json:"filter,omitempty"` + } + + var temp struct { + Q json.RawMessage `json:"query"` + Size *int `json:"size"` + From int `json:"from"` + Highlight *HighlightRequest `json:"highlight"` + Fields []string `json:"fields"` + Facets FacetsRequest `json:"facets"` + Explain bool `json:"explain"` + Sort []json.RawMessage `json:"sort"` + IncludeLocations bool `json:"includeLocations"` + Score string `json:"score"` + SearchAfter []string `json:"search_after"` + SearchBefore []string `json:"search_before"` + KNN []*tempKNNReq `json:"knn"` + KNNOperator knnOperator `json:"knn_operator"` + PreSearchData OptionalRawMessage `json:"pre_search_data"` + Params OptionalRawMessage `json:"params"` + } + + err := util.UnmarshalJSON(input, &temp) + if err != nil { + return err + } + + if temp.Size == nil { + r.Size = 10 + } else { + r.Size = *temp.Size + } + if temp.Sort == nil { + r.Sort = search.SortOrder{&search.SortScore{Desc: true}} + } else { + r.Sort, err = search.ParseSortOrderJSON(temp.Sort) + if err != nil { + return err + } + } + r.From = temp.From + r.Explain = temp.Explain + r.Highlight = temp.Highlight + r.Fields = temp.Fields + r.Facets = temp.Facets + r.IncludeLocations = temp.IncludeLocations + r.Score = temp.Score + r.SearchAfter = temp.SearchAfter + r.SearchBefore = temp.SearchBefore + r.Query, err = query.ParseQuery(temp.Q) + if err != nil { + return err + } + + if r.Size < 0 { + r.Size = 10 + } + if r.From < 0 { + r.From = 0 + } + + if IsScoreFusionRequested(r) { + if temp.Params == nil { + // If params is not present and it is requires rescoring, assign + // default values + r.Params = NewDefaultParams(r.From, r.Size) + } else { + // if it is a request that requires rescoring, parse the rescoring + // parameters. + params, err := ParseParams(r, temp.Params) + if err != nil { + return err + } + r.Params = params + } + } + + r.KNN = make([]*KNNRequest, len(temp.KNN)) + for i, knnReq := range temp.KNN { + r.KNN[i] = &KNNRequest{} + r.KNN[i].Field = temp.KNN[i].Field + r.KNN[i].Vector = temp.KNN[i].Vector + r.KNN[i].VectorBase64 = temp.KNN[i].VectorBase64 + r.KNN[i].K = temp.KNN[i].K + r.KNN[i].Boost = temp.KNN[i].Boost + if len(temp.KNN[i].Params) > 0 { + r.KNN[i].Params = json.RawMessage(temp.KNN[i].Params) + } + if len(temp.KNN[i].FilterQuery) > 0 { + r.KNN[i].FilterQuery, err = query.ParseQuery(knnReq.FilterQuery) + if err != nil { + return err + } + } + } + r.KNNOperator = temp.KNNOperator + if r.KNNOperator == "" { + r.KNNOperator = knnOperatorOr + } + + if temp.PreSearchData != nil { + r.PreSearchData, err = query.ParsePreSearchData(temp.PreSearchData) + if err != nil { + return err + } + } + + return nil + +} + +// ----------------------------------------------------------------------------- + +func copySearchRequest(req *SearchRequest, preSearchData map[string]interface{}) *SearchRequest { + rv := SearchRequest{ + Query: req.Query, + Size: req.Size + req.From, + From: 0, + Highlight: req.Highlight, + Fields: req.Fields, + Facets: req.Facets, + Explain: req.Explain, + Sort: req.Sort.Copy(), + IncludeLocations: req.IncludeLocations, + Score: req.Score, + SearchAfter: req.SearchAfter, + SearchBefore: req.SearchBefore, + KNN: req.KNN, + KNNOperator: req.KNNOperator, + PreSearchData: preSearchData, + Params: req.Params, + } + return &rv + +} + +var ( + knnOperatorAnd = knnOperator("and") + knnOperatorOr = knnOperator("or") +) + +func createKNNQuery(req *SearchRequest, knnFilterResults map[int]index.EligibleDocumentSelector) ( + query.Query, []int64, int64, error) { + if requestHasKNN(req) { + // first perform validation + err := validateKNN(req) + if err != nil { + return nil, nil, 0, err + } + var subQueries []query.Query + kArray := make([]int64, 0, len(req.KNN)) + sumOfK := int64(0) + for i, knn := range req.KNN { + // If it's a filtered kNN but has no eligible filter hits, then + // do not run the kNN query. + if selector, exists := knnFilterResults[i]; exists && selector == nil { + // if the kNN query is filtered and has no eligible filter hits, then + // do not run the kNN query, so we add a match_none query to the subQueries. + // this will ensure that the score breakdown is set to 0 for this kNN query. + subQueries = append(subQueries, NewMatchNoneQuery()) + kArray = append(kArray, 0) + continue + } + knnQuery := query.NewKNNQuery(knn.Vector) + knnQuery.SetField(knn.Field) + knnQuery.SetK(knn.K) + knnQuery.SetBoost(knn.Boost.Value()) + knnQuery.SetParams(knn.Params) + if selector, exists := knnFilterResults[i]; exists { + knnQuery.SetEligibleSelector(selector) + } + subQueries = append(subQueries, knnQuery) + kArray = append(kArray, knn.K) + sumOfK += knn.K + } + rv := query.NewDisjunctionQuery(subQueries) + rv.RetrieveScoreBreakdown(true) + return rv, kArray, sumOfK, nil + } + return nil, nil, 0, nil +} + +func validateKNN(req *SearchRequest) error { + for _, q := range req.KNN { + if q == nil { + return fmt.Errorf("knn query cannot be nil") + } + if len(q.Vector) == 0 && q.VectorBase64 != "" { + // consider vector_base64 only if vector is not provided + decodedVector, err := document.DecodeVector(q.VectorBase64) + if err != nil { + return err + } + + q.Vector = decodedVector + } + if q.K <= 0 || len(q.Vector) == 0 { + return fmt.Errorf("k must be greater than 0 and vector must be non-empty") + } + if q.K > BleveMaxK { + return fmt.Errorf("k must be less than %d", BleveMaxK) + } + // since the DefaultField is not applicable for knn, + // the field must be specified. + if q.Field == "" { + return fmt.Errorf("knn query field must be non-empty") + } + if vfq, ok := q.FilterQuery.(query.ValidatableQuery); ok { + err := vfq.Validate() + if err != nil { + return fmt.Errorf("knn filter query is invalid: %v", err) + } + } + } + switch req.KNNOperator { + case knnOperatorAnd, knnOperatorOr, "": + // Valid cases, do nothing + default: + return fmt.Errorf("knn_operator must be either 'and' / 'or'") + } + + return nil +} + +func addSortAndFieldsToKNNHits(req *SearchRequest, knnHits []*search.DocumentMatch, reader index.IndexReader, name string) (err error) { + requiredSortFields := req.Sort.RequiredFields() + var dvReader index.DocValueReader + var updateFieldVisitor index.DocValueVisitor + if len(requiredSortFields) > 0 { + dvReader, err = reader.DocValueReader(requiredSortFields) + if err != nil { + return err + } + updateFieldVisitor = func(field string, term []byte) { + req.Sort.UpdateVisitor(field, term) + } + } + for _, hit := range knnHits { + if len(requiredSortFields) > 0 { + err = dvReader.VisitDocValues(hit.IndexInternalID, updateFieldVisitor) + if err != nil { + return err + } + } + req.Sort.Value(hit) + err, _ = LoadAndHighlightAllFields(hit, req, "", reader, nil) + if err != nil { + return err + } + hit.Index = name + } + return nil +} + +func (i *indexImpl) runKnnCollector(ctx context.Context, req *SearchRequest, reader index.IndexReader, preSearch bool) (knnHits []*search.DocumentMatch, err error) { + // Maps the index of a KNN query in the request to its pre-filter result: + // - If the KNN query is **not filtered**, the value will be `nil`. + // - If the KNN query **is filtered**, the value will be an eligible document selector + // that can be used to retrieve eligible documents. + // - If there is an **empty entry** for a KNN query, it means no documents match + // the filter query, and the KNN query can be skipped. + knnFilterResults := make(map[int]index.EligibleDocumentSelector) + for idx, knnReq := range req.KNN { + filterQ := knnReq.FilterQuery + if filterQ == nil || isMatchAllQuery(filterQ) { + // When there is no filter query or the filter query is match_all, + // all documents are eligible, and can be treated as unfiltered query. + continue + } else if isMatchNoneQuery(filterQ) { + // If the filter query is match_none, then no documents match the filter query. + knnFilterResults[idx] = nil + continue + } + // Applies to all supported types of queries. + filterSearcher, err := filterQ.Searcher(ctx, reader, i.m, search.SearcherOptions{ + Score: "none", // just want eligible hits --> don't compute scores if not needed + }) + if err != nil { + return nil, err + } + // Using the index doc count to determine collector size since we do not + // have an estimate of the number of eligible docs in the index yet. + indexDocCount, err := i.DocCount() + if err != nil { + // close the searcher before returning + filterSearcher.Close() + return nil, err + } + filterColl := collector.NewEligibleCollector(int(indexDocCount)) + err = filterColl.Collect(ctx, filterSearcher, reader) + if err != nil { + // close the searcher before returning + filterSearcher.Close() + return nil, err + } + knnFilterResults[idx] = filterColl.EligibleSelector() + // Close the filter searcher, as we are done with it. + err = filterSearcher.Close() + if err != nil { + return nil, err + } + } + + // Add the filter hits when creating the kNN query + KNNQuery, kArray, sumOfK, err := createKNNQuery(req, knnFilterResults) + if err != nil { + return nil, err + } + knnSearcher, err := KNNQuery.Searcher(ctx, reader, i.m, search.SearcherOptions{ + Explain: req.Explain, + }) + if err != nil { + return nil, err + } + defer func() { + if serr := knnSearcher.Close(); err == nil && serr != nil { + err = serr + } + }() + knnCollector := collector.NewKNNCollector(kArray, sumOfK) + err = knnCollector.Collect(ctx, knnSearcher, reader) + if err != nil { + return nil, err + } + knnHits = knnCollector.Results() + if !preSearch { + knnHits = finalizeKNNResults(req, knnHits) + } + // at this point, irrespective of whether it is a preSearch or not, + // the knn hits are populated with Sort and Fields. + // it must be ensured downstream that the Sort and Fields are not + // re-evaluated, for these hits. + // also add the index names to the hits, so that when early + // exit takes place after the first phase, the hits will have + // a valid value for Index. + err = addSortAndFieldsToKNNHits(req, knnHits, reader, i.name) + if err != nil { + return nil, err + } + return knnHits, nil +} + +func setKnnHitsInCollector(knnHits []*search.DocumentMatch, coll *collector.TopNCollector) { + if len(knnHits) > 0 { + mergeFn := func(ftsMatch *search.DocumentMatch, knnMatch *search.DocumentMatch) { + // Boost the FTS score using the KNN score + ftsMatch.Score += knnMatch.Score + // Combine the FTS explanation with the KNN explanation, if present + ftsMatch.Expl = ftsMatch.Expl.MergeWith(knnMatch.Expl) + } + coll.SetKNNHits(knnHits, search.HybridMergeCallbackFn(mergeFn)) + } +} + +func finalizeKNNResults(req *SearchRequest, knnHits []*search.DocumentMatch) []*search.DocumentMatch { + // if the KNN operator is AND, then we need to filter out the hits that + // do not have match the KNN queries. + if req.KNNOperator == knnOperatorAnd { + idx := 0 + for _, hit := range knnHits { + if len(hit.ScoreBreakdown) == len(req.KNN) { + knnHits[idx] = hit + idx++ + } + } + knnHits = knnHits[:idx] + } + + // if score fusion required, return early because + // score breakdown is retained + if IsScoreFusionRequested(req) { + return knnHits + } + // fix the score using score breakdown now + // if the score is none, then we need to set the score to 0.0 + // if req.Explain is true, then we need to use the expl breakdown to + // finalize the correct explanation. + for _, hit := range knnHits { + hit.Score = 0.0 + if req.Score != "none" { + for _, score := range hit.ScoreBreakdown { + hit.Score += score + } + } + if req.Explain { + childrenExpl := make([]*search.Explanation, 0, len(hit.ScoreBreakdown)) + for i := range hit.ScoreBreakdown { + childrenExpl = append(childrenExpl, hit.Expl.Children[i]) + } + hit.Expl = &search.Explanation{Value: hit.Score, Message: "sum of:", Children: childrenExpl} + } + // we don't need the score breakdown anymore + // so we can set it to nil + hit.ScoreBreakdown = nil + } + return knnHits +} + +// when we are setting KNN hits in the preSearchData, we need to make sure that +// the KNN hit goes to the right index. This is because the KNN hits are +// collected from all the indexes in the alias, but the preSearchData is +// specific to each index. If alias A1 contains indexes I1 and I2 and +// the KNN hits collected from both I1 and I2, and merged to get top K +// hits, then the top K hits need to be distributed to I1 and I2, +// so that the preSearchData for I1 contains the top K hits from I1 and +// the preSearchData for I2 contains the top K hits from I2. +func validateAndDistributeKNNHits(knnHits []*search.DocumentMatch, indexes []Index) (map[string][]*search.DocumentMatch, error) { + // create a set of all the index names of this alias + indexNames := make(map[string]struct{}, len(indexes)) + for _, index := range indexes { + indexNames[index.Name()] = struct{}{} + } + segregatedKnnHits := make(map[string][]*search.DocumentMatch) + for _, hit := range knnHits { + // for each hit, we need to perform a validation check to ensure that the stack + // is still valid. + // + // if the stack is empty, then we have an inconsistency/abnormality + // since any hit with an empty stack is supposed to land on a leaf index, + // and not an alias. This cannot happen in normal circumstances. But + // performing this check to be safe. Since we extract the stack top + // in the following steps. + if len(hit.IndexNames) == 0 { + return nil, ErrorTwoPhaseSearchInconsistency + } + // since the stack is not empty, we need to check if the top of the stack + // is a valid index name, of an index that is part of this alias. If not, + // then we have an inconsistency that could be caused due to a topology + // change. + stackTopIdx := len(hit.IndexNames) - 1 + top := hit.IndexNames[stackTopIdx] + if _, exists := indexNames[top]; !exists { + return nil, ErrorTwoPhaseSearchInconsistency + } + if stackTopIdx == 0 { + // if the stack consists of only one index, then popping the top + // would result in an empty slice, and handle this case by setting + // indexNames to nil. So that the final search results will not + // contain the indexNames field. + hit.IndexNames = nil + } else { + hit.IndexNames = hit.IndexNames[:stackTopIdx] + } + segregatedKnnHits[top] = append(segregatedKnnHits[top], hit) + } + return segregatedKnnHits, nil +} + +func requestHasKNN(req *SearchRequest) bool { + return len(req.KNN) > 0 +} + +func numKNNQueries(req *SearchRequest) int { + return len(req.KNN) +} + +// returns true if the search request contains a KNN request that can be +// satisfied by just performing a preSearch, completely bypassing the +// actual search. +func isKNNrequestSatisfiedByPreSearch(req *SearchRequest) bool { + // if req.Query is not match_none => then we need to go to phase 2 + // to perform the actual query. + if !isMatchNoneQuery(req.Query) { + return false + } + // req.Query is a match_none query + // + // if request contains facets, we need to perform phase 2 to calculate + // the facet result. Since documents were removed as part of the + // merging process after phase 1, if the facet results were to be calculated + // during phase 1, then they will be now be incorrect, since merging would + // remove some documents. + if req.Facets != nil { + return false + } + // the request is a match_none query and does not contain any facets + // so we can satisfy the request using just the preSearch result. + return true +} + +func constructKnnPreSearchData(mergedOut map[string]map[string]interface{}, preSearchResult *SearchResult, + indexes []Index) (map[string]map[string]interface{}, error) { + + distributedHits, err := validateAndDistributeKNNHits([]*search.DocumentMatch(preSearchResult.Hits), indexes) + if err != nil { + return nil, err + } + for _, index := range indexes { + mergedOut[index.Name()][search.KnnPreSearchDataKey] = distributedHits[index.Name()] + } + return mergedOut, nil +} + +func addKnnToDummyRequest(dummyReq *SearchRequest, realReq *SearchRequest) { + dummyReq.KNN = realReq.KNN + dummyReq.KNNOperator = knnOperatorOr + dummyReq.Explain = realReq.Explain + dummyReq.Fields = realReq.Fields + dummyReq.Sort = realReq.Sort +} + +func newKnnPreSearchResultProcessor(req *SearchRequest) *knnPreSearchResultProcessor { + kArray := make([]int64, len(req.KNN)) + for i, knnReq := range req.KNN { + kArray[i] = knnReq.K + } + knnStore := collector.GetNewKNNCollectorStore(kArray) + return &knnPreSearchResultProcessor{ + addFn: func(sr *SearchResult, indexName string) { + for _, hit := range sr.Hits { + // tag the hit with the index name, so that when the + // final search result is constructed, the hit will have + // a valid path to follow along the alias tree to reach + // the index. + hit.IndexNames = append(hit.IndexNames, indexName) + knnStore.AddDocument(hit) + } + }, + finalizeFn: func(sr *SearchResult) { + // passing nil as the document fixup function, because we don't need to + // fixup the document, since this was already done in the first phase, + // hence error is always nil. + // the merged knn hits are finalized and set in the search result. + sr.Hits, _ = knnStore.Final(nil) + }, + } +} + +// Replace knn boost values for fusion rescoring queries +func (r *rescorer) prepareKnnRequest() { + for i := range r.req.KNN { + b := r.req.KNN[i].Boost + if b != nil { + r.origBoosts[i+1] = b.Value() + newB := query.Boost(1.0) + r.req.KNN[i].Boost = &newB + } else { + r.origBoosts[i+1] = 1.0 + } + } +} + +// Restore knn boost values for fusion rescoring queries +func (r *rescorer) restoreKnnRequest() { + for i := range r.req.KNN { + b := query.Boost(r.origBoosts[i+1]) + r.req.KNN[i].Boost = &b + } +} diff --git a/search_knn_test.go b/search_knn_test.go new file mode 100644 index 0000000..e3b0ff4 --- /dev/null +++ b/search_knn_test.go @@ -0,0 +1,3142 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors +// +build vectors + +package bleve + +import ( + "archive/zip" + "bytes" + "encoding/base64" + "encoding/binary" + "encoding/json" + "fmt" + "math" + "math/rand" + "reflect" + "sort" + "strconv" + "sync" + "testing" + "time" + + "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + "github.com/blevesearch/bleve/v2/analysis/lang/en" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/query" + index "github.com/blevesearch/bleve_index_api" +) + +const testInputCompressedFile = "test/knn/knn_dataset_queries.zip" +const testDatasetFileName = "knn_dataset.json" +const testQueryFileName = "knn_queries.json" + +const testDatasetDims = 384 + +var knnOperators []knnOperator = []knnOperator{knnOperatorAnd, knnOperatorOr} + +func TestSimilaritySearchPartitionedIndex(t *testing.T) { + dataset, searchRequests, err := readDatasetAndQueries(testInputCompressedFile) + if err != nil { + t.Fatal(err) + } + documents := makeDatasetIntoDocuments(dataset) + contentFieldMapping := NewTextFieldMapping() + contentFieldMapping.Analyzer = en.AnalyzerName + + vecFieldMappingL2 := mapping.NewVectorFieldMapping() + vecFieldMappingL2.Dims = testDatasetDims + vecFieldMappingL2.Similarity = index.EuclideanDistance + + indexMappingL2Norm := NewIndexMapping() + indexMappingL2Norm.DefaultMapping.AddFieldMappingsAt("content", contentFieldMapping) + indexMappingL2Norm.DefaultMapping.AddFieldMappingsAt("vector", vecFieldMappingL2) + + vecFieldMappingDot := mapping.NewVectorFieldMapping() + vecFieldMappingDot.Dims = testDatasetDims + vecFieldMappingDot.Similarity = index.InnerProduct + + indexMappingDotProduct := NewIndexMapping() + indexMappingDotProduct.DefaultMapping.AddFieldMappingsAt("content", contentFieldMapping) + indexMappingDotProduct.DefaultMapping.AddFieldMappingsAt("vector", vecFieldMappingDot) + + vecFieldMappingCosine := mapping.NewVectorFieldMapping() + vecFieldMappingCosine.Dims = testDatasetDims + vecFieldMappingCosine.Similarity = index.CosineSimilarity + + indexMappingCosine := NewIndexMapping() + indexMappingCosine.DefaultMapping.AddFieldMappingsAt("content", contentFieldMapping) + indexMappingCosine.DefaultMapping.AddFieldMappingsAt("vector", vecFieldMappingCosine) + + type testCase struct { + testType string + queryIndex int + numIndexPartitions int + mapping mapping.IndexMapping + } + + testCases := []testCase{ + // l2 norm similarity + { + testType: "multi_partition:match_none:oneKNNreq:k=3", + queryIndex: 0, + numIndexPartitions: 4, + mapping: indexMappingL2Norm, + }, + { + testType: "multi_partition:match_none:oneKNNreq:k=2", + queryIndex: 0, + numIndexPartitions: 10, + mapping: indexMappingL2Norm, + }, + { + testType: "multi_partition:match:oneKNNreq:k=2", + queryIndex: 1, + numIndexPartitions: 5, + mapping: indexMappingL2Norm, + }, + { + testType: "multi_partition:disjunction:twoKNNreq:k=2,2", + queryIndex: 2, + numIndexPartitions: 4, + mapping: indexMappingL2Norm, + }, + // dot product similarity + { + testType: "multi_partition:match_none:oneKNNreq:k=3", + queryIndex: 0, + numIndexPartitions: 4, + mapping: indexMappingDotProduct, + }, + { + testType: "multi_partition:match_none:oneKNNreq:k=2", + queryIndex: 0, + numIndexPartitions: 10, + mapping: indexMappingDotProduct, + }, + { + testType: "multi_partition:match:oneKNNreq:k=2", + queryIndex: 1, + numIndexPartitions: 5, + mapping: indexMappingDotProduct, + }, + { + testType: "multi_partition:disjunction:twoKNNreq:k=2,2", + queryIndex: 2, + numIndexPartitions: 4, + mapping: indexMappingDotProduct, + }, + // cosine similarity + { + testType: "multi_partition:match_none:oneKNNreq:k=3", + queryIndex: 0, + numIndexPartitions: 7, + mapping: indexMappingCosine, + }, + { + testType: "multi_partition:match_none:oneKNNreq:k=2", + queryIndex: 0, + numIndexPartitions: 5, + mapping: indexMappingCosine, + }, + { + testType: "multi_partition:match:oneKNNreq:k=2", + queryIndex: 1, + numIndexPartitions: 3, + mapping: indexMappingCosine, + }, + { + testType: "multi_partition:disjunction:twoKNNreq:k=2,2", + queryIndex: 2, + numIndexPartitions: 9, + mapping: indexMappingCosine, + }, + } + + index := NewIndexAlias() + var reqSort = search.SortOrder{&search.SortScore{Desc: true}, &search.SortDocID{Desc: true}, &search.SortField{Desc: false, Field: "content"}} + for testCaseNum, testCase := range testCases { + originalRequest := searchRequests[testCase.queryIndex] + for _, operator := range knnOperators { + + index.indexes = make([]Index, 0) + query := copySearchRequest(originalRequest, nil) + query.AddKNNOperator(operator) + query.Sort = reqSort.Copy() + query.Explain = true + + nameToIndex := createPartitionedIndex(documents, index, 1, testCase.mapping, t, false) + controlResult, err := index.Search(query) + if err != nil { + cleanUp(t, nameToIndex) + t.Fatal(err) + } + if !finalHitsHaveValidIndex(controlResult.Hits, nameToIndex) { + cleanUp(t, nameToIndex) + t.Fatalf("test case #%d failed: expected control result hits to have valid `Index`", testCaseNum) + } + cleanUp(t, nameToIndex) + + index.indexes = make([]Index, 0) + query = copySearchRequest(originalRequest, nil) + query.AddKNNOperator(operator) + query.Sort = reqSort.Copy() + query.Explain = true + + nameToIndex = createPartitionedIndex(documents, index, testCase.numIndexPartitions, testCase.mapping, t, false) + experimentalResult, err := index.Search(query) + if err != nil { + cleanUp(t, nameToIndex) + t.Fatal(err) + } + if !finalHitsHaveValidIndex(experimentalResult.Hits, nameToIndex) { + cleanUp(t, nameToIndex) + t.Fatalf("test case #%d failed: expected experimental Result hits to have valid `Index`", testCaseNum) + } + verifyResult(t, controlResult, experimentalResult, testCaseNum, true) + cleanUp(t, nameToIndex) + + index.indexes = make([]Index, 0) + query = copySearchRequest(originalRequest, nil) + query.AddKNNOperator(operator) + query.Sort = reqSort.Copy() + query.Explain = true + + nameToIndex = createPartitionedIndex(documents, index, testCase.numIndexPartitions, testCase.mapping, t, true) + multiLevelIndexResult, err := index.Search(query) + if err != nil { + cleanUp(t, nameToIndex) + t.Fatal(err) + } + if !finalHitsHaveValidIndex(multiLevelIndexResult.Hits, nameToIndex) { + cleanUp(t, nameToIndex) + t.Fatalf("test case #%d failed: expected experimental Result hits to have valid `Index`", testCaseNum) + } + verifyResult(t, multiLevelIndexResult, experimentalResult, testCaseNum, false) + cleanUp(t, nameToIndex) + + } + } + + var facets = map[string]*FacetRequest{ + "content": { + Field: "content", + Size: 10, + }, + } + + index = NewIndexAlias() + for testCaseNum, testCase := range testCases { + index.indexes = make([]Index, 0) + nameToIndex := createPartitionedIndex(documents, index, testCase.numIndexPartitions, testCase.mapping, t, false) + originalRequest := searchRequests[testCase.queryIndex] + for _, operator := range knnOperators { + from, size := originalRequest.From, originalRequest.Size + query := copySearchRequest(originalRequest, nil) + query.AddKNNOperator(operator) + query.Explain = true + query.From = from + query.Size = size + + // Three types of queries to run wrt sort and facet fields that require fields. + // 1. Sort And Facet are there + // 2. Sort is there, Facet is not there + // 3. Sort is not there, Facet is there + // The case where both sort and facet are not there is already covered in the previous tests. + + // 1. Sort And Facet are there + query.Facets = facets + query.Sort = reqSort.Copy() + + res1, err := index.Search(query) + if err != nil { + cleanUp(t, nameToIndex) + t.Fatal(err) + } + if !finalHitsHaveValidIndex(res1.Hits, nameToIndex) { + cleanUp(t, nameToIndex) + t.Fatalf("test case #%d failed: expected experimental Result hits to have valid `Index`", testCaseNum) + } + + facetRes1 := res1.Facets + facetRes1Str, err := json.Marshal(facetRes1) + if err != nil { + cleanUp(t, nameToIndex) + t.Fatal(err) + } + + // 2. Sort is there, Facet is not there + query.Facets = nil + query.Sort = reqSort.Copy() + + res2, err := index.Search(query) + if err != nil { + cleanUp(t, nameToIndex) + t.Fatal(err) + } + if !finalHitsHaveValidIndex(res2.Hits, nameToIndex) { + cleanUp(t, nameToIndex) + t.Fatalf("test case #%d failed: expected experimental Result hits to have valid `Index`", testCaseNum) + } + + // 3. Sort is not there, Facet is there + query.Facets = facets + query.Sort = nil + res3, err := index.Search(query) + if err != nil { + cleanUp(t, nameToIndex) + t.Fatal(err) + } + if !finalHitsHaveValidIndex(res3.Hits, nameToIndex) { + cleanUp(t, nameToIndex) + t.Fatalf("test case #%d failed: expected experimental Result hits to have valid `Index`", testCaseNum) + } + + facetRes3 := res3.Facets + facetRes3Str, err := json.Marshal(facetRes3) + if err != nil { + cleanUp(t, nameToIndex) + t.Fatal(err) + } + + // Verify the facet results + if string(facetRes1Str) != string(facetRes3Str) { + cleanUp(t, nameToIndex) + t.Fatalf("test case #%d failed: expected facet results to be equal", testCaseNum) + } + + // Verify the results + verifyResult(t, res1, res2, testCaseNum, false) + verifyResult(t, res2, res3, testCaseNum, true) + + // Test early exit fail case -> matchNone + facetRequest + query.Query = NewMatchNoneQuery() + query.Sort = reqSort.Copy() + // control case + query.Facets = nil + res4Ctrl, err := index.Search(query) + if err != nil { + cleanUp(t, nameToIndex) + t.Fatal(err) + } + if !finalHitsHaveValidIndex(res4Ctrl.Hits, nameToIndex) { + cleanUp(t, nameToIndex) + t.Fatalf("test case #%d failed: expected control Result hits to have valid `Index`", testCaseNum) + } + + // experimental case + query.Facets = facets + res4Exp, err := index.Search(query) + if err != nil { + cleanUp(t, nameToIndex) + t.Fatal(err) + } + if !finalHitsHaveValidIndex(res4Exp.Hits, nameToIndex) { + cleanUp(t, nameToIndex) + t.Fatalf("test case #%d failed: expected experimental Result hits to have valid `Index`", testCaseNum) + } + + if !(operator == knnOperatorAnd && res4Ctrl.Total == 0 && res4Exp.Total == 0) { + // catch case where no hits are returned + // due to matchNone query with a KNN request with operator AND + // where no hits are part of the intersection in multi knn request + verifyResult(t, res4Ctrl, res4Exp, testCaseNum, false) + } + } + cleanUp(t, nameToIndex) + } + + // Test Pagination with multi partitioned index + index = NewIndexAlias() + index.indexes = make([]Index, 0) + nameToIndex := createPartitionedIndex(documents, index, 8, indexMappingL2Norm, t, true) + + // Test From + Size pagination for Hybrid Search (2-Phase) + query := copySearchRequest(searchRequests[4], nil) + query.Sort = reqSort.Copy() + query.Facets = facets + query.Explain = true + + testFromSizePagination(t, query, index, nameToIndex) + + // Test From + Size pagination for Early Exit Hybrid Search (1-Phase) + query = copySearchRequest(searchRequests[4], nil) + query.Query = NewMatchNoneQuery() + query.Sort = reqSort.Copy() + query.Facets = nil + query.Explain = true + + testFromSizePagination(t, query, index, nameToIndex) + + cleanUp(t, nameToIndex) +} + +func testFromSizePagination(t *testing.T, query *SearchRequest, index Index, nameToIndex map[string]Index) { + query.From = 0 + query.Size = 30 + + resCtrl, err := index.Search(query) + if err != nil { + cleanUp(t, nameToIndex) + t.Fatal(err) + } + + ctrlHitIds := make([]string, len(resCtrl.Hits)) + for i, doc := range resCtrl.Hits { + ctrlHitIds[i] = doc.ID + } + // experimental case + + fromValues := []int{0, 5, 10, 15, 20, 25} + size := 5 + for fromIdx := 0; fromIdx < len(fromValues); fromIdx++ { + from := fromValues[fromIdx] + query.From = from + query.Size = size + resExp, err := index.Search(query) + if err != nil { + cleanUp(t, nameToIndex) + t.Fatal(err) + } + if from >= len(ctrlHitIds) { + if len(resExp.Hits) != 0 { + cleanUp(t, nameToIndex) + t.Fatalf("expected 0 hits, got %d", len(resExp.Hits)) + } + continue + } + numHitsExp := len(resExp.Hits) + numHitsCtrl := min(len(ctrlHitIds)-from, size) + if numHitsExp != numHitsCtrl { + cleanUp(t, nameToIndex) + t.Fatalf("expected %d hits, got %d", numHitsCtrl, numHitsExp) + } + for i := 0; i < numHitsExp; i++ { + doc := resExp.Hits[i] + startOffset := from + i + if doc.ID != ctrlHitIds[startOffset] { + cleanUp(t, nameToIndex) + t.Fatalf("expected %s at index %d, got %s", ctrlHitIds[startOffset], i, doc.ID) + } + } + } +} + +func TestVectorBase64Index(t *testing.T) { + dataset, searchRequests, err := readDatasetAndQueries(testInputCompressedFile) + if err != nil { + t.Fatal(err) + } + documents := makeDatasetIntoDocuments(dataset) + + _, searchRequestsCopy, err := readDatasetAndQueries(testInputCompressedFile) + if err != nil { + t.Fatal(err) + } + + for _, doc := range documents { + vec, ok := doc["vector"].([]float32) + if !ok { + t.Fatal("Typecasting vector to float array failed") + } + + buf := new(bytes.Buffer) + for _, v := range vec { + err := binary.Write(buf, binary.LittleEndian, v) + if err != nil { + t.Fatal(err) + } + } + + doc["vectorEncoded"] = base64.StdEncoding.EncodeToString(buf.Bytes()) + } + + for _, sr := range searchRequestsCopy { + for _, kr := range sr.KNN { + kr.Field = "vectorEncoded" + } + } + + contentFM := NewTextFieldMapping() + contentFM.Analyzer = en.AnalyzerName + + vecFML2 := mapping.NewVectorFieldMapping() + vecFML2.Dims = testDatasetDims + vecFML2.Similarity = index.EuclideanDistance + + vecBFML2 := mapping.NewVectorBase64FieldMapping() + vecBFML2.Dims = testDatasetDims + vecBFML2.Similarity = index.EuclideanDistance + + vecFMDot := mapping.NewVectorFieldMapping() + vecFMDot.Dims = testDatasetDims + vecFMDot.Similarity = index.InnerProduct + + vecBFMDot := mapping.NewVectorBase64FieldMapping() + vecBFMDot.Dims = testDatasetDims + vecBFMDot.Similarity = index.InnerProduct + + indexMappingL2 := NewIndexMapping() + indexMappingL2.DefaultMapping.AddFieldMappingsAt("content", contentFM) + indexMappingL2.DefaultMapping.AddFieldMappingsAt("vector", vecFML2) + indexMappingL2.DefaultMapping.AddFieldMappingsAt("vectorEncoded", vecBFML2) + + indexMappingDot := NewIndexMapping() + indexMappingDot.DefaultMapping.AddFieldMappingsAt("content", contentFM) + indexMappingDot.DefaultMapping.AddFieldMappingsAt("vector", vecFMDot) + indexMappingDot.DefaultMapping.AddFieldMappingsAt("vectorEncoded", vecBFMDot) + + tmpIndexPathL2 := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPathL2) + + tmpIndexPathDot := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPathDot) + + indexL2, err := New(tmpIndexPathL2, indexMappingL2) + if err != nil { + t.Fatal(err) + } + defer func() { + err := indexL2.Close() + if err != nil { + t.Fatal(err) + } + }() + + indexDot, err := New(tmpIndexPathDot, indexMappingDot) + if err != nil { + t.Fatal(err) + } + defer func() { + err := indexDot.Close() + if err != nil { + t.Fatal(err) + } + }() + + batchL2 := indexL2.NewBatch() + batchDot := indexDot.NewBatch() + + for _, doc := range documents { + err = batchL2.Index(doc["id"].(string), doc) + if err != nil { + t.Fatal(err) + } + err = batchDot.Index(doc["id"].(string), doc) + if err != nil { + t.Fatal(err) + } + } + + err = indexL2.Batch(batchL2) + if err != nil { + t.Fatal(err) + } + + err = indexDot.Batch(batchDot) + if err != nil { + t.Fatal(err) + } + + for i := range searchRequests { + for _, operator := range knnOperators { + controlQuery := searchRequests[i] + testQuery := searchRequestsCopy[i] + + controlQuery.AddKNNOperator(operator) + testQuery.AddKNNOperator(operator) + + controlResultL2, err := indexL2.Search(controlQuery) + if err != nil { + t.Fatal(err) + } + testResultL2, err := indexL2.Search(testQuery) + if err != nil { + t.Fatal(err) + } + + if controlResultL2 != nil && testResultL2 != nil { + if len(controlResultL2.Hits) == len(testResultL2.Hits) { + for j := range controlResultL2.Hits { + if controlResultL2.Hits[j].ID != testResultL2.Hits[j].ID { + t.Fatalf("testcase %d failed: expected hit id %s, got hit id %s", i, controlResultL2.Hits[j].ID, testResultL2.Hits[j].ID) + } + } + } + } else if (controlResultL2 == nil && testResultL2 != nil) || + (controlResultL2 != nil && testResultL2 == nil) { + t.Fatalf("testcase %d failed: expected result %s, got result %s", i, controlResultL2, testResultL2) + } + + controlResultDot, err := indexDot.Search(controlQuery) + if err != nil { + t.Fatal(err) + } + testResultDot, err := indexDot.Search(testQuery) + if err != nil { + t.Fatal(err) + } + + if controlResultDot != nil && testResultDot != nil { + if len(controlResultDot.Hits) == len(testResultDot.Hits) { + for j := range controlResultDot.Hits { + if controlResultDot.Hits[j].ID != testResultDot.Hits[j].ID { + t.Fatalf("testcase %d failed: expected hit id %s, got hit id %s", i, controlResultDot.Hits[j].ID, testResultDot.Hits[j].ID) + } + } + } + } else if (controlResultDot == nil && testResultDot != nil) || + (controlResultDot != nil && testResultDot == nil) { + t.Fatalf("testcase %d failed: expected result %s, got result %s", i, controlResultDot, testResultDot) + } + } + } +} + +// Test to verify that the bivf-flat indexes with vector base64 field mapping returns the +// same results as the non-optimized vector field mapping for L2, Dot Product and Cosine similarities. +// Also test to see no differences in results for any distance metric +func TestVectorBivfIndexes(t *testing.T) { + optimizations := []string{index.IndexBIVFWithBackingSQ8, index.IndexBIVFWithBackingFlat} + for _, optimization := range optimizations { + testVectorBivfIndex(t, optimization) + } +} + +func testVectorBivfIndex(t *testing.T, optimization string) { + + dataset, searchRequests, err := readDatasetAndQueries(testInputCompressedFile) + if err != nil { + t.Fatal(err) + } + documents := makeDatasetIntoDocuments(dataset) + + _, searchRequestsCopy, err := readDatasetAndQueries(testInputCompressedFile) + if err != nil { + t.Fatal(err) + } + + for _, doc := range documents { + vec, ok := doc["vector"].([]float32) + if !ok { + t.Fatal("Typecasting vector to float array failed") + } + + buf := new(bytes.Buffer) + for _, v := range vec { + err := binary.Write(buf, binary.LittleEndian, v) + if err != nil { + t.Fatal(err) + } + } + + doc["vectorEncoded"] = base64.StdEncoding.EncodeToString(buf.Bytes()) + } + + for _, sr := range searchRequestsCopy { + for _, kr := range sr.KNN { + kr.Field = "vectorEncoded" + } + } + + contentFM := NewTextFieldMapping() + contentFM.Analyzer = en.AnalyzerName + + vecFML2 := mapping.NewVectorFieldMapping() + vecFML2.Dims = testDatasetDims + vecFML2.Similarity = index.EuclideanDistance + vecFML2.VectorIndexOptimizedFor = optimization + + vecBFML2 := mapping.NewVectorBase64FieldMapping() + vecBFML2.Dims = testDatasetDims + vecBFML2.Similarity = index.EuclideanDistance + vecBFML2.VectorIndexOptimizedFor = optimization + + vecFMDot := mapping.NewVectorFieldMapping() + vecFMDot.Dims = testDatasetDims + vecFMDot.Similarity = index.InnerProduct + vecFMDot.VectorIndexOptimizedFor = optimization + + vecBFMDot := mapping.NewVectorBase64FieldMapping() + vecBFMDot.Dims = testDatasetDims + vecBFMDot.Similarity = index.InnerProduct + vecBFMDot.VectorIndexOptimizedFor = optimization + + vecFMCosine := mapping.NewVectorFieldMapping() + vecFMCosine.Dims = testDatasetDims + vecFMCosine.Similarity = index.CosineSimilarity + + vecBFMCosine := mapping.NewVectorBase64FieldMapping() + vecBFMCosine.Dims = testDatasetDims + vecBFMCosine.Similarity = index.CosineSimilarity + vecBFMCosine.VectorIndexOptimizedFor = optimization + + indexMappingL2 := NewIndexMapping() + indexMappingL2.DefaultMapping.AddFieldMappingsAt("content", contentFM) + indexMappingL2.DefaultMapping.AddFieldMappingsAt("vector", vecFML2) + indexMappingL2.DefaultMapping.AddFieldMappingsAt("vectorEncoded", vecBFML2) + + indexMappingDot := NewIndexMapping() + indexMappingDot.DefaultMapping.AddFieldMappingsAt("content", contentFM) + indexMappingDot.DefaultMapping.AddFieldMappingsAt("vector", vecFMDot) + indexMappingDot.DefaultMapping.AddFieldMappingsAt("vectorEncoded", vecBFMDot) + + indexMappingCosine := NewIndexMapping() + indexMappingCosine.DefaultMapping.AddFieldMappingsAt("content", contentFM) + indexMappingCosine.DefaultMapping.AddFieldMappingsAt("vector", vecFMCosine) + indexMappingCosine.DefaultMapping.AddFieldMappingsAt("vectorEncoded", vecBFMCosine) + + tmpIndexPathL2 := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPathL2) + + tmpIndexPathDot := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPathDot) + + tmpIndexPathCosine := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPathCosine) + + indexL2, err := New(tmpIndexPathL2, indexMappingL2) + if err != nil { + t.Fatal(err) + } + defer func() { + err := indexL2.Close() + if err != nil { + t.Fatal(err) + } + }() + + indexDot, err := New(tmpIndexPathDot, indexMappingDot) + if err != nil { + t.Fatal(err) + } + defer func() { + err := indexDot.Close() + if err != nil { + t.Fatal(err) + } + }() + + indexCosine, err := New(tmpIndexPathCosine, indexMappingCosine) + if err != nil { + t.Fatal(err) + } + defer func() { + err := indexCosine.Close() + if err != nil { + t.Fatal(err) + } + }() + + batchL2 := indexL2.NewBatch() + batchDot := indexDot.NewBatch() + batchCosine := indexCosine.NewBatch() + + for _, doc := range documents { + err = batchL2.Index(doc["id"].(string), doc) + if err != nil { + t.Fatal(err) + } + err = batchDot.Index(doc["id"].(string), doc) + if err != nil { + t.Fatal(err) + } + err = batchCosine.Index(doc["id"].(string), doc) + if err != nil { + t.Fatal(err) + } + } + + err = indexL2.Batch(batchL2) + if err != nil { + t.Fatal(err) + } + + err = indexDot.Batch(batchDot) + if err != nil { + t.Fatal(err) + } + + err = indexCosine.Batch(batchCosine) + if err != nil { + t.Fatal(err) + } + + for i := range searchRequests { + for _, operator := range knnOperators { + normQuery := searchRequests[i] + base64Query := searchRequestsCopy[i] + + normQuery.AddKNNOperator(operator) + base64Query.AddKNNOperator(operator) + + normResultL2, err := indexL2.Search(normQuery) + if err != nil { + t.Fatal(err) + } + base64ResultL2, err := indexL2.Search(base64Query) + if err != nil { + t.Fatal(err) + } + + if normResultL2 != nil && base64ResultL2 != nil { + if len(normResultL2.Hits) == len(base64ResultL2.Hits) { + for j := range normResultL2.Hits { + if normResultL2.Hits[j].ID != base64ResultL2.Hits[j].ID { + t.Fatalf("testcase %d failed: expected hit id %s, got hit id %s", i, normResultL2.Hits[j].ID, base64ResultL2.Hits[j].ID) + } + } + } + } else if (normResultL2 == nil && base64ResultL2 != nil) || + (normResultL2 != nil && base64ResultL2 == nil) { + t.Fatalf("testcase %d failed: expected result %s, got result %s", i, normResultL2, base64ResultL2) + } + + normResultDot, err := indexDot.Search(normQuery) + if err != nil { + t.Fatal(err) + } + base64ResultDot, err := indexDot.Search(base64Query) + if err != nil { + t.Fatal(err) + } + + if normResultDot != nil && base64ResultDot != nil { + if len(normResultDot.Hits) == len(base64ResultDot.Hits) { + for j := range normResultDot.Hits { + if normResultDot.Hits[j].ID != base64ResultDot.Hits[j].ID { + t.Fatalf("testcase %d failed: expected hit id %s, got hit id %s", i, normResultDot.Hits[j].ID, base64ResultDot.Hits[j].ID) + } + } + } + } else if (normResultDot == nil && base64ResultDot != nil) || + (normResultDot != nil && base64ResultDot == nil) { + t.Fatalf("testcase %d failed: expected result %s, got result %s", i, normResultDot, base64ResultDot) + } + + normResultCosine, err := indexCosine.Search(normQuery) + if err != nil { + t.Fatal(err) + } + base64ResultCosine, err := indexCosine.Search(base64Query) + if err != nil { + t.Fatal(err) + } + + if normResultCosine != nil && base64ResultCosine != nil { + if len(normResultCosine.Hits) == len(base64ResultCosine.Hits) { + for j := range normResultCosine.Hits { + if normResultCosine.Hits[j].ID != base64ResultCosine.Hits[j].ID { + t.Fatalf("testcase %d failed: expected hit id %s, got hit id %s", i, normResultCosine.Hits[j].ID, base64ResultCosine.Hits[j].ID) + } + } + } + } else if (normResultCosine == nil && base64ResultCosine != nil) || + (normResultCosine != nil && base64ResultCosine == nil) { + t.Fatalf("testcase %d failed: expected result %s, got result %s", i, normResultCosine, base64ResultCosine) + } + + if normResultCosine != nil && normResultL2 != nil { + if len(normResultCosine.Hits) == len(normResultL2.Hits) { + for j := range normResultCosine.Hits { + if normResultCosine.Hits[j].ID != normResultL2.Hits[j].ID { + if normResultCosine.Hits[j].Score != normResultL2.Hits[j].Score { + t.Fatalf("testcase %d failed: expected hit id %s, got hit id %s", i, normResultCosine.Hits[j].ID, normResultL2.Hits[j].ID) + } + } + } + } + } else if (normResultCosine == nil && normResultL2 != nil) || + (normResultCosine != nil && normResultL2 == nil) { + t.Fatalf("testcase %d failed: expected result %s, got result %s", i, normResultCosine, normResultL2) + } + + if normResultCosine != nil && normResultDot != nil { + if len(normResultCosine.Hits) == len(normResultDot.Hits) { + for j := range normResultCosine.Hits { + if normResultCosine.Hits[j].ID != normResultDot.Hits[j].ID { + if normResultCosine.Hits[j].Score != normResultDot.Hits[j].Score { + t.Fatalf("testcase %d failed: expected hit id %s, got hit id %s", i, normResultCosine.Hits[j].ID, normResultDot.Hits[j].ID) + } + } + } + } + } else if (normResultCosine == nil && normResultDot != nil) || + (normResultCosine != nil && normResultDot == nil) { + t.Fatalf("testcase %d failed: expected result %s, got result %s", i, normResultCosine, normResultDot) + } + } + } +} + +type testDocument struct { + ID string `json:"id"` + Content string `json:"content"` + Vector []float32 `json:"vector"` +} + +func readDatasetAndQueries(fileName string) ([]testDocument, []*SearchRequest, error) { + // Open the zip archive for reading + r, err := zip.OpenReader(fileName) + if err != nil { + return nil, nil, err + } + var dataset []testDocument + var queries []*SearchRequest + + defer r.Close() + for _, f := range r.File { + jsonFile, err := f.Open() + if err != nil { + return nil, nil, err + } + defer jsonFile.Close() + if f.Name == testDatasetFileName { + err = json.NewDecoder(jsonFile).Decode(&dataset) + if err != nil { + return nil, nil, err + } + } else if f.Name == testQueryFileName { + err = json.NewDecoder(jsonFile).Decode(&queries) + if err != nil { + return nil, nil, err + } + } + } + return dataset, queries, nil +} + +func makeDatasetIntoDocuments(dataset []testDocument) []map[string]interface{} { + documents := make([]map[string]interface{}, len(dataset)) + for i := 0; i < len(dataset); i++ { + document := make(map[string]interface{}) + document["id"] = dataset[i].ID + document["content"] = dataset[i].Content + document["vector"] = dataset[i].Vector + documents[i] = document + } + return documents +} + +func cleanUp(t *testing.T, nameToIndex map[string]Index) { + for path, childIndex := range nameToIndex { + err := childIndex.Close() + if err != nil { + t.Fatal(err) + } + cleanupTmpIndexPath(t, path) + } +} + +func createChildIndex(docs []map[string]interface{}, mapping mapping.IndexMapping, t *testing.T, nameToIndex map[string]Index) Index { + tmpIndexPath := createTmpIndexPath(t) + index, err := New(tmpIndexPath, mapping) + if err != nil { + t.Fatal(err) + } + nameToIndex[index.Name()] = index + batch := index.NewBatch() + for _, doc := range docs { + err := batch.Index(doc["id"].(string), doc) + if err != nil { + cleanUp(t, nameToIndex) + t.Fatal(err) + } + } + err = index.Batch(batch) + if err != nil { + cleanUp(t, nameToIndex) + t.Fatal(err) + } + return index +} + +func createPartitionedIndex(documents []map[string]interface{}, index *indexAliasImpl, numPartitions int, + mapping mapping.IndexMapping, t *testing.T, multiLevel bool) map[string]Index { + + partitionSize := len(documents) / numPartitions + extraDocs := len(documents) % numPartitions + numDocsPerPartition := make([]int, numPartitions) + for i := 0; i < numPartitions; i++ { + numDocsPerPartition[i] = partitionSize + if extraDocs > 0 { + numDocsPerPartition[i]++ + extraDocs-- + } + } + docsPerPartition := make([][]map[string]interface{}, numPartitions) + prevCutoff := 0 + for i := 0; i < numPartitions; i++ { + docsPerPartition[i] = make([]map[string]interface{}, numDocsPerPartition[i]) + for j := 0; j < numDocsPerPartition[i]; j++ { + docsPerPartition[i][j] = documents[prevCutoff+j] + } + prevCutoff += numDocsPerPartition[i] + } + + rv := make(map[string]Index) + if !multiLevel { + // all indexes are at the same level + for i := 0; i < numPartitions; i++ { + index.Add(createChildIndex(docsPerPartition[i], mapping, t, rv)) + } + } else { + // alias tree + indexes := make([]Index, numPartitions) + for i := 0; i < numPartitions; i++ { + indexes[i] = createChildIndex(docsPerPartition[i], mapping, t, rv) + } + numAlias := int(math.Ceil(float64(numPartitions) / 2.0)) + aliases := make([]IndexAlias, numAlias) + for i := 0; i < numAlias; i++ { + aliases[i] = NewIndexAlias() + aliases[i].SetName(fmt.Sprintf("alias%d", i)) + for j := 0; j < 2; j++ { + if i*2+j < numPartitions { + aliases[i].Add(indexes[i*2+j]) + } + } + } + for i := 0; i < numAlias; i++ { + index.Add(aliases[i]) + } + } + return rv +} + +func createMultipleSegmentsIndex(documents []map[string]interface{}, index Index, numSegments int) error { + // create multiple batches to simulate more than one segment + numBatches := numSegments + + batches := make([]*Batch, numBatches) + numDocsPerBatch := len(documents) / numBatches + extraDocs := len(documents) % numBatches + + docsPerBatch := make([]int, numBatches) + for i := 0; i < numBatches; i++ { + docsPerBatch[i] = numDocsPerBatch + if extraDocs > 0 { + docsPerBatch[i]++ + extraDocs-- + } + } + prevCutoff := 0 + for i := 0; i < numBatches; i++ { + batches[i] = index.NewBatch() + for j := prevCutoff; j < prevCutoff+docsPerBatch[i]; j++ { + doc := documents[j] + err := batches[i].Index(doc["id"].(string), doc) + if err != nil { + return err + } + } + prevCutoff += docsPerBatch[i] + } + errMutex := sync.Mutex{} + var errors []error + wg := sync.WaitGroup{} + wg.Add(len(batches)) + for i, batch := range batches { + go func(ix int, batchx *Batch) { + defer wg.Done() + err := index.Batch(batchx) + if err != nil { + errMutex.Lock() + errors = append(errors, err) + errMutex.Unlock() + } + }(i, batch) + } + wg.Wait() + if len(errors) > 0 { + return errors[0] + } + return nil +} + +func truncateScore(score float64) float64 { + epsilon := 1e-4 + truncated := float64(int(score*1e6)) / 1e6 + if math.Abs(truncated-1.0) <= epsilon { + return 1.0 + } + return truncated +} + +// Function to compare two Explanation structs recursively +func compareExplanation(a, b *search.Explanation) bool { + if a == nil && b == nil { + return true + } + if a == nil || b == nil { + return false + } + + if truncateScore(a.Value) != truncateScore(b.Value) || len(a.Children) != len(b.Children) { + return false + } + + // Sort the children slices before comparison + sortChildren(a.Children) + sortChildren(b.Children) + + for i := range a.Children { + if !compareExplanation(a.Children[i], b.Children[i]) { + return false + } + } + return true +} + +// Function to sort the children slices +func sortChildren(children []*search.Explanation) { + sort.Slice(children, func(i, j int) bool { + return children[i].Value < children[j].Value + }) +} + +// All hits from a hybrid search/knn search should not have +// index names or score breakdown. +func finalHitsOmitKNNMetadata(hits []*search.DocumentMatch) bool { + for _, hit := range hits { + if hit.IndexNames != nil || hit.ScoreBreakdown != nil { + return false + } + } + return true +} + +func finalHitsHaveValidIndex(hits []*search.DocumentMatch, indexes map[string]Index) bool { + for _, hit := range hits { + if hit.Index == "" { + return false + } + var idx Index + var ok bool + if idx, ok = indexes[hit.Index]; !ok { + return false + } + if idx == nil { + return false + } + var doc index.Document + doc, err = idx.Document(hit.ID) + if err != nil { + return false + } + if doc == nil { + return false + } + } + return true +} + +func verifyResult(t *testing.T, controlResult *SearchResult, experimentalResult *SearchResult, testCaseNum int, verifyOnlyDocIDs bool) { + if controlResult.Hits.Len() == 0 || experimentalResult.Hits.Len() == 0 { + t.Fatalf("test case #%d failed: 0 hits returned", testCaseNum) + } + if len(controlResult.Hits) != len(experimentalResult.Hits) { + t.Fatalf("test case #%d failed: expected %d results, got %d", testCaseNum, len(controlResult.Hits), len(experimentalResult.Hits)) + } + if controlResult.Total != experimentalResult.Total { + t.Fatalf("test case #%d failed: expected total hits to be %d, got %d", testCaseNum, controlResult.Total, experimentalResult.Total) + } + // KNN Metadata -> Score Breakdown and IndexNames MUST be omitted from the final hits + if !finalHitsOmitKNNMetadata(controlResult.Hits) || !finalHitsOmitKNNMetadata(experimentalResult.Hits) { + t.Fatalf("test case #%d failed: expected no KNN metadata in hits", testCaseNum) + } + if controlResult.Took == 0 || experimentalResult.Took == 0 { + t.Fatalf("test case #%d failed: expected non-zero took time", testCaseNum) + } + if controlResult.Request == nil || experimentalResult.Request == nil { + t.Fatalf("test case #%d failed: expected non-nil request", testCaseNum) + } + if verifyOnlyDocIDs { + // in multi partitioned index, we cannot be sure of the score or the ordering of the hits as the tf-idf scores are localized to each partition + // so we only check the ids + controlMap := make(map[string]struct{}) + experimentalMap := make(map[string]struct{}) + for _, hit := range controlResult.Hits { + controlMap[hit.ID] = struct{}{} + } + for _, hit := range experimentalResult.Hits { + experimentalMap[hit.ID] = struct{}{} + } + if len(controlMap) != len(experimentalMap) { + t.Fatalf("test case #%d failed: expected %d results, got %d", testCaseNum, len(controlMap), len(experimentalMap)) + } + for id := range controlMap { + if _, ok := experimentalMap[id]; !ok { + t.Fatalf("test case #%d failed: expected id %s to be in experimental result", testCaseNum, id) + } + } + return + } + for i := 0; i < len(controlResult.Hits); i++ { + if controlResult.Hits[i].ID != experimentalResult.Hits[i].ID { + t.Fatalf("test case #%d failed: expected hit %d to have id %s, got %s", testCaseNum, i, controlResult.Hits[i].ID, experimentalResult.Hits[i].ID) + } + // Truncate to 6 decimal places + actualScore := truncateScore(experimentalResult.Hits[i].Score) + expectScore := truncateScore(controlResult.Hits[i].Score) + if expectScore != actualScore { + t.Fatalf("test case #%d failed: expected hit %d to have score %f, got %f", testCaseNum, i, expectScore, actualScore) + } + if !compareExplanation(controlResult.Hits[i].Expl, experimentalResult.Hits[i].Expl) { + t.Fatalf("test case #%d failed: expected hit %d to have explanation %v, got %v", testCaseNum, i, controlResult.Hits[i].Expl, experimentalResult.Hits[i].Expl) + } + } + if truncateScore(controlResult.MaxScore) != truncateScore(experimentalResult.MaxScore) { + t.Fatalf("test case #%d: expected maxScore to be %f, got %f", testCaseNum, controlResult.MaxScore, experimentalResult.MaxScore) + } +} + +func TestSimilaritySearchMultipleSegments(t *testing.T) { + // using scorch options to prevent merges during the course of this test + // so that the knnCollector can be accurately tested + scorch.DefaultMemoryPressurePauseThreshold = 0 + scorch.DefaultMinSegmentsForInMemoryMerge = math.MaxInt + dataset, searchRequests, err := readDatasetAndQueries(testInputCompressedFile) + if err != nil { + t.Fatal(err) + } + documents := makeDatasetIntoDocuments(dataset) + + contentFieldMapping := NewTextFieldMapping() + contentFieldMapping.Analyzer = en.AnalyzerName + + vecFieldMappingL2 := mapping.NewVectorFieldMapping() + vecFieldMappingL2.Dims = testDatasetDims + vecFieldMappingL2.Similarity = index.EuclideanDistance + + vecFieldMappingDot := mapping.NewVectorFieldMapping() + vecFieldMappingDot.Dims = testDatasetDims + vecFieldMappingDot.Similarity = index.InnerProduct + + vecFieldMappingCosine := mapping.NewVectorFieldMapping() + vecFieldMappingCosine.Dims = testDatasetDims + vecFieldMappingCosine.Similarity = index.CosineSimilarity + + indexMappingL2Norm := NewIndexMapping() + indexMappingL2Norm.DefaultMapping.AddFieldMappingsAt("content", contentFieldMapping) + indexMappingL2Norm.DefaultMapping.AddFieldMappingsAt("vector", vecFieldMappingL2) + + indexMappingDotProduct := NewIndexMapping() + indexMappingDotProduct.DefaultMapping.AddFieldMappingsAt("content", contentFieldMapping) + indexMappingDotProduct.DefaultMapping.AddFieldMappingsAt("vector", vecFieldMappingDot) + + indexMappingCosine := NewIndexMapping() + indexMappingCosine.DefaultMapping.AddFieldMappingsAt("content", contentFieldMapping) + indexMappingCosine.DefaultMapping.AddFieldMappingsAt("vector", vecFieldMappingCosine) + + var reqSort = search.SortOrder{&search.SortScore{Desc: true}, &search.SortDocID{Desc: true}, &search.SortField{Desc: false, Field: "content"}} + + testCases := []struct { + numSegments int + queryIndex int + mapping mapping.IndexMapping + scoreValue string + }{ + // L2 norm similarity + { + numSegments: 6, + queryIndex: 0, + mapping: indexMappingL2Norm, + }, + { + numSegments: 7, + queryIndex: 1, + mapping: indexMappingL2Norm, + }, + { + numSegments: 8, + queryIndex: 2, + mapping: indexMappingL2Norm, + }, + { + numSegments: 9, + queryIndex: 3, + mapping: indexMappingL2Norm, + }, + { + numSegments: 10, + queryIndex: 4, + mapping: indexMappingL2Norm, + }, + { + numSegments: 11, + queryIndex: 5, + mapping: indexMappingL2Norm, + }, + // dot_product similarity + { + numSegments: 6, + queryIndex: 0, + mapping: indexMappingDotProduct, + }, + { + numSegments: 7, + queryIndex: 1, + mapping: indexMappingDotProduct, + }, + { + numSegments: 8, + queryIndex: 2, + mapping: indexMappingDotProduct, + }, + { + numSegments: 9, + queryIndex: 3, + mapping: indexMappingDotProduct, + }, + { + numSegments: 10, + queryIndex: 4, + mapping: indexMappingDotProduct, + }, + { + numSegments: 11, + queryIndex: 5, + mapping: indexMappingDotProduct, + }, + // cosine similarity + { + numSegments: 9, + queryIndex: 0, + mapping: indexMappingCosine, + }, + { + numSegments: 5, + queryIndex: 1, + mapping: indexMappingCosine, + }, + { + numSegments: 4, + queryIndex: 2, + mapping: indexMappingCosine, + }, + { + numSegments: 12, + queryIndex: 3, + mapping: indexMappingCosine, + }, + { + numSegments: 7, + queryIndex: 4, + mapping: indexMappingCosine, + }, + { + numSegments: 11, + queryIndex: 5, + mapping: indexMappingCosine, + }, + // score none test + { + numSegments: 3, + queryIndex: 0, + mapping: indexMappingL2Norm, + scoreValue: "none", + }, + { + numSegments: 7, + queryIndex: 1, + mapping: indexMappingL2Norm, + scoreValue: "none", + }, + { + numSegments: 8, + queryIndex: 2, + mapping: indexMappingL2Norm, + scoreValue: "none", + }, + { + numSegments: 3, + queryIndex: 0, + mapping: indexMappingDotProduct, + scoreValue: "none", + }, + { + numSegments: 7, + queryIndex: 1, + mapping: indexMappingDotProduct, + scoreValue: "none", + }, + { + numSegments: 8, + queryIndex: 2, + mapping: indexMappingDotProduct, + scoreValue: "none", + }, + { + numSegments: 3, + queryIndex: 0, + mapping: indexMappingCosine, + scoreValue: "none", + }, + { + numSegments: 7, + queryIndex: 1, + mapping: indexMappingCosine, + scoreValue: "none", + }, + { + numSegments: 8, + queryIndex: 2, + mapping: indexMappingCosine, + scoreValue: "none", + }, + } + for testCaseNum, testCase := range testCases { + originalRequest := searchRequests[testCase.queryIndex] + for _, operator := range knnOperators { + // run single segment test first + tmpIndexPath := createTmpIndexPath(t) + index, err := New(tmpIndexPath, testCase.mapping) + if err != nil { + t.Fatal(err) + } + query := copySearchRequest(originalRequest, nil) + query.Sort = reqSort.Copy() + query.AddKNNOperator(operator) + query.Explain = true + + nameToIndex := make(map[string]Index) + nameToIndex[index.Name()] = index + + err = createMultipleSegmentsIndex(documents, index, 1) + if err != nil { + cleanUp(t, nameToIndex) + t.Fatal(err) + } + controlResult, err := index.Search(query) + if err != nil { + cleanUp(t, nameToIndex) + t.Fatal(err) + } + if !finalHitsHaveValidIndex(controlResult.Hits, nameToIndex) { + cleanUp(t, nameToIndex) + t.Fatalf("test case #%d failed: expected control result hits to have valid `Index`", testCaseNum) + } + if testCase.scoreValue == "none" { + + query := copySearchRequest(originalRequest, nil) + query.Sort = reqSort.Copy() + query.AddKNNOperator(operator) + query.Explain = true + query.Score = testCase.scoreValue + + expectedResultScoreNone, err := index.Search(query) + if err != nil { + cleanUp(t, nameToIndex) + t.Fatal(err) + } + if !finalHitsHaveValidIndex(expectedResultScoreNone.Hits, nameToIndex) { + cleanUp(t, nameToIndex) + t.Fatalf("test case #%d failed: expected score none hits to have valid `Index`", testCaseNum) + } + verifyResult(t, controlResult, expectedResultScoreNone, testCaseNum, true) + } + cleanUp(t, nameToIndex) + + // run multiple segments test + tmpIndexPath = createTmpIndexPath(t) + index, err = New(tmpIndexPath, testCase.mapping) + if err != nil { + t.Fatal(err) + } + nameToIndex = make(map[string]Index) + nameToIndex[index.Name()] = index + err = createMultipleSegmentsIndex(documents, index, testCase.numSegments) + if err != nil { + cleanUp(t, nameToIndex) + t.Fatal(err) + } + + query = copySearchRequest(originalRequest, nil) + query.Sort = reqSort.Copy() + query.AddKNNOperator(operator) + query.Explain = true + + experimentalResult, err := index.Search(query) + if err != nil { + cleanUp(t, nameToIndex) + t.Fatal(err) + } + if !finalHitsHaveValidIndex(experimentalResult.Hits, nameToIndex) { + cleanUp(t, nameToIndex) + t.Fatalf("test case #%d failed: expected experimental result hits to have valid `Index`", testCaseNum) + } + verifyResult(t, controlResult, experimentalResult, testCaseNum, false) + cleanUp(t, nameToIndex) + } + } +} + +// Test to determine the impact of boost on kNN queries. +func TestKNNScoreBoosting(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + const dims = 5 + getRandomVector := func() []float32 { + vec := make([]float32, dims) + for i := 0; i < dims; i++ { + vec[i] = rand.Float32() + } + return vec + } + + dataset := make([]map[string]interface{}, 10) + + // Indexing just a few docs to populate index. + for i := 0; i < 100; i++ { + dataset = append(dataset, map[string]interface{}{ + "type": "vectorStuff", + "content": strconv.Itoa(i), + "vector": getRandomVector(), + }) + } + + indexMapping := NewIndexMapping() + indexMapping.TypeField = "type" + indexMapping.DefaultAnalyzer = "en" + documentMapping := NewDocumentMapping() + indexMapping.AddDocumentMapping("vectorStuff", documentMapping) + + contentFieldMapping := NewTextFieldMapping() + contentFieldMapping.Index = true + contentFieldMapping.Store = true + documentMapping.AddFieldMappingsAt("content", contentFieldMapping) + + vecFieldMapping := mapping.NewVectorFieldMapping() + vecFieldMapping.Index = true + vecFieldMapping.Dims = 5 + vecFieldMapping.Similarity = "dot_product" + documentMapping.AddFieldMappingsAt("vector", vecFieldMapping) + + index, err := New(tmpIndexPath, indexMapping) + if err != nil { + t.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + }() + + batch := index.NewBatch() + for i := 0; i < len(dataset); i++ { + err = batch.Index(strconv.Itoa(i), dataset[i]) + if err != nil { + t.Fatal(err) + } + } + + err = index.Batch(batch) + if err != nil { + t.Fatal(err) + } + + queryVec := getRandomVector() + searchRequest := NewSearchRequest(NewMatchNoneQuery()) + searchRequest.AddKNN("vector", queryVec, 3, 1.0) + searchRequest.Fields = []string{"content", "vector"} + + hits, err := index.Search(searchRequest) + if err != nil { + t.Fatal(err) + } + hitsMap := make(map[string]float64, 0) + for _, hit := range hits.Hits { + hitsMap[hit.ID] = (hit.Score) + } + + searchRequest = NewSearchRequest(NewMatchNoneQuery()) + searchRequest.AddKNN("vector", queryVec, 3, 10.0) + searchRequest.Fields = []string{"content", "vector"} + + hits, err = index.Search(searchRequest) + if err != nil { + t.Fatal(err) + } + hitsMap2 := make(map[string]float64, 0) + for _, hit := range hits.Hits { + hitsMap2[hit.ID] = (hit.Score) + } + + for _, hit := range hits.Hits { + if hitsMap[hit.ID] != hitsMap2[hit.ID]/10 { + t.Errorf("boosting not working: %v %v \n", hitsMap[hit.ID], hitsMap2[hit.ID]) + } + } +} + +// Test to see if KNN Operators get added right to the query. +func TestKNNOperator(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + const dims = 5 + getRandomVector := func() []float32 { + vec := make([]float32, dims) + for i := 0; i < dims; i++ { + vec[i] = rand.Float32() + } + return vec + } + + dataset := make([]map[string]interface{}, 10) + + // Indexing just a few docs to populate index. + for i := 0; i < 10; i++ { + dataset = append(dataset, map[string]interface{}{ + "type": "vectorStuff", + "content": strconv.Itoa(i), + "vector": getRandomVector(), + }) + } + + indexMapping := NewIndexMapping() + indexMapping.TypeField = "type" + indexMapping.DefaultAnalyzer = "en" + documentMapping := NewDocumentMapping() + indexMapping.AddDocumentMapping("vectorStuff", documentMapping) + + contentFieldMapping := NewTextFieldMapping() + contentFieldMapping.Index = true + contentFieldMapping.Store = true + documentMapping.AddFieldMappingsAt("content", contentFieldMapping) + + vecFieldMapping := mapping.NewVectorFieldMapping() + vecFieldMapping.Index = true + vecFieldMapping.Dims = 5 + vecFieldMapping.Similarity = "dot_product" + documentMapping.AddFieldMappingsAt("vector", vecFieldMapping) + + index, err := New(tmpIndexPath, indexMapping) + if err != nil { + t.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + }() + + batch := index.NewBatch() + for i := 0; i < len(dataset); i++ { + err = batch.Index(strconv.Itoa(i), dataset[i]) + if err != nil { + t.Fatal(err) + } + } + + err = index.Batch(batch) + if err != nil { + t.Fatal(err) + } + + termQuery := query.NewTermQuery("2") + + searchRequest := NewSearchRequest(termQuery) + searchRequest.AddKNN("vector", getRandomVector(), 3, 2.0) + searchRequest.AddKNN("vector", getRandomVector(), 2, 1.5) + searchRequest.Fields = []string{"content", "vector"} + + // Conjunction + searchRequest.AddKNNOperator(knnOperatorAnd) + conjunction, _, _, err := createKNNQuery(searchRequest, nil) + if err != nil { + t.Fatalf("unexpected error for AND knn operator") + } + + conj, ok := conjunction.(*query.DisjunctionQuery) + if !ok { + t.Fatalf("expected disjunction query") + } + + if len(conj.Disjuncts) != 2 { + t.Fatalf("expected 2 disjuncts") + } + + // Disjunction + searchRequest.AddKNNOperator(knnOperatorOr) + disjunction, _, _, err := createKNNQuery(searchRequest, nil) + if err != nil { + t.Fatalf("unexpected error for OR knn operator") + } + + disj, ok := disjunction.(*query.DisjunctionQuery) + if !ok { + t.Fatalf("expected disjunction query") + } + + if len(disj.Disjuncts) != 2 { + t.Fatalf("expected 2 disjuncts") + } + + // Incorrect operator. + searchRequest.AddKNNOperator("bs_op") + searchRequest.Query, _, _, err = createKNNQuery(searchRequest, nil) + if err == nil { + t.Fatalf("expected error for incorrect knn operator") + } +} + +func TestKNNFiltering(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + const dims = 5 + getRandomVector := func() []float32 { + vec := make([]float32, dims) + for i := 0; i < dims; i++ { + vec[i] = rand.Float32() + } + return vec + } + + dataset := make([]map[string]interface{}, 0) + + // Indexing just a few docs to populate index. + for i := 0; i < 10; i++ { + dataset = append(dataset, map[string]interface{}{ + "type": "vectorStuff", + "content": strconv.Itoa(i + 1000), + "vector": getRandomVector(), + }) + } + + indexMapping := NewIndexMapping() + indexMapping.TypeField = "type" + indexMapping.DefaultAnalyzer = "en" + documentMapping := NewDocumentMapping() + indexMapping.AddDocumentMapping("vectorStuff", documentMapping) + + contentFieldMapping := NewTextFieldMapping() + contentFieldMapping.Index = true + contentFieldMapping.Store = true + documentMapping.AddFieldMappingsAt("content", contentFieldMapping) + + vecFieldMapping := mapping.NewVectorFieldMapping() + vecFieldMapping.Index = true + vecFieldMapping.Dims = 5 + vecFieldMapping.Similarity = "dot_product" + documentMapping.AddFieldMappingsAt("vector", vecFieldMapping) + + index, err := New(tmpIndexPath, indexMapping) + if err != nil { + t.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + }() + + batch := index.NewBatch() + for i := 0; i < len(dataset); i++ { + // the id of term "i" is (i-1000) + err = batch.Index(strconv.Itoa(i), dataset[i]) + if err != nil { + t.Fatal(err) + } + } + + err = index.Batch(batch) + if err != nil { + t.Fatal(err) + } + + termQuery := query.NewTermQuery("1004") + filterRequest := NewSearchRequest(termQuery) + filteredHits, err := index.Search(filterRequest) + if err != nil { + t.Fatal(err) + } + filteredDocIDs := make(map[string]struct{}) + for _, match := range filteredHits.Hits { + filteredDocIDs[match.ID] = struct{}{} + } + + searchRequest := NewSearchRequest(NewMatchNoneQuery()) + searchRequest.AddKNNWithFilter("vector", getRandomVector(), 3, 2.0, termQuery) + searchRequest.Fields = []string{"content", "vector"} + + res, err := index.Search(searchRequest) + if err != nil { + t.Fatal(err) + } + // check if any of the returned results are not part of the filtered hits. + for _, match := range res.Hits { + if _, exists := filteredDocIDs[match.ID]; !exists { + t.Errorf("returned result not present in filtered hits") + } + } + + // No results should be returned with a match_none filter. + searchRequest = NewSearchRequest(NewMatchNoneQuery()) + searchRequest.AddKNNWithFilter("vector", getRandomVector(), 3, 2.0, + NewMatchNoneQuery()) + res, err = index.Search(searchRequest) + if err != nil { + t.Fatal(err) + } + if len(res.Hits) != 0 { + t.Errorf("match none filter should return no hits") + } + + // Testing with a disjunction query. + + termQuery = query.NewTermQuery("1003") + termQuery2 := query.NewTermQuery("1005") + disjQuery := query.NewDisjunctionQuery([]query.Query{termQuery, termQuery2}) + filterRequest = NewSearchRequest(disjQuery) + filteredHits, err = index.Search(filterRequest) + if err != nil { + t.Fatal(err) + } + filteredDocIDs = make(map[string]struct{}) + for _, match := range filteredHits.Hits { + filteredDocIDs[match.ID] = struct{}{} + } + + searchRequest = NewSearchRequest(NewMatchNoneQuery()) + searchRequest.AddKNNWithFilter("vector", getRandomVector(), 3, 2.0, disjQuery) + searchRequest.Fields = []string{"content", "vector"} + + res, err = index.Search(searchRequest) + if err != nil { + t.Fatal(err) + } + + for _, match := range res.Hits { + if _, exists := filteredDocIDs[match.ID]; !exists { + t.Errorf("returned result not present in filtered hits") + } + } +} + +// ----------------------------------------------------------------------------- +// Test nested vectors + +func TestNestedVectors(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + const dims = 3 + const k = 1 // one nearest neighbor + const vecFieldName = "vecData" + + dataset := map[string]map[string]interface{}{ // docID -> Doc + "doc1": { + vecFieldName: []float32{100, 100, 100}, + }, + "doc2": { + vecFieldName: [][]float32{{0, 0, 0}, {1000, 1000, 1000}}, + }, + } + + // Index mapping + indexMapping := NewIndexMapping() + vm := mapping.NewVectorFieldMapping() + vm.Dims = dims + vm.Similarity = "l2_norm" + indexMapping.DefaultMapping.AddFieldMappingsAt(vecFieldName, vm) + + // Create index and upload documents + index, err := New(tmpIndexPath, indexMapping) + if err != nil { + t.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + }() + + batch := index.NewBatch() + for docID, doc := range dataset { + err = batch.Index(docID, doc) + if err != nil { + t.Fatal(err) + } + } + + err = index.Batch(batch) + if err != nil { + t.Fatal(err) + } + + // Run searches + + tests := []struct { + queryVec []float32 + expectedDocID string + }{ + { + queryVec: []float32{100, 100, 100}, + expectedDocID: "doc1", + }, + { + queryVec: []float32{0, 0, 0}, + expectedDocID: "doc2", + }, + { + queryVec: []float32{1000, 1000, 1000}, + expectedDocID: "doc2", + }, + } + + for _, test := range tests { + searchReq := NewSearchRequest(query.NewMatchNoneQuery()) + searchReq.AddKNNWithFilter(vecFieldName, test.queryVec, k, 1000, + NewMatchAllQuery()) + + res, err := index.Search(searchReq) + if err != nil { + t.Fatal(err) + } + + if len(res.Hits) != 1 { + t.Fatalf("expected 1 hit, got %d", len(res.Hits)) + } + + if res.Hits[0].ID != test.expectedDocID { + t.Fatalf("expected docID %s, got %s", test.expectedDocID, + res.Hits[0].ID) + } + } +} + +// ----------------------------------------------------------------------------- +// TestMultiVector tests the KNN functionality which handles duplicate +// vectors being matched within the same document. When a document has multiple vectors +// (via [[]] array of vectors or [{}] array of objects with vectors), the KNN +// searcher must pick the best scoring vector match for that document. This test covers these scenarios: +// - Single vector field (baseline) +// - [[]] style: array of vectors (same doc appears multiple times) +// - [{}] style: array of objects with vector field (chunks pattern) +func TestMultiVector(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + // JSON documents covering merger scenarios: + // - Single vector (baseline) + // - [[]] style: array of vectors (same doc appears multiple times) + // - [{}] style: array of objects with vector field (chunks pattern) + docs := map[string]string{ + // Single vector - baseline + "doc1": `{ + "vec": [10, 10, 10], + "vecB": [100, 100, 100] + }`, + // [[]] style - array of 2 vectors + "doc2": `{ + "vec": [[0, 0, 0], [500, 500, 500]], + "vecB": [[900, 900, 900], [950, 950, 950], [975, 975, 975], [990, 990, 990]] + }`, + // [[]] style - array of 3 vectors + "doc3": `{ + "vec": [[50, 50, 50], [200, 200, 200], [400, 400, 400]], + "vecB": [[800, 800, 800], [850, 850, 850]] + }`, + // Single vector - baseline + "doc4": `{ + "vec": [1000, 1000, 1000], + "vecB": [1, 1, 1] + }`, + // [{}] style - array of objects with vector field (chunks pattern) + "doc5": `{ + "chunks": [ + {"vec": [10, 10, 10], "text": "chunk1"}, + {"vec": [20, 20, 20], "text": "chunk2"}, + {"vec": [30, 30, 30], "text": "chunk3"}, + {"vec": [40, 40, 40], "text": "chunk4"} + ] + }`, + "doc6": `{ + "chunks": [ + {"vec": [[10, 10, 10],[20, 20, 20]], "text": "chunk1"}, + {"vec": [[30, 30, 30],[40, 40, 40]], "text": "chunk2"} + ] + }`, + } + + // Parse JSON documents + dataset := make(map[string]map[string]interface{}) + for docID, jsonStr := range docs { + var doc map[string]interface{} + if err := json.Unmarshal([]byte(jsonStr), &doc); err != nil { + t.Fatalf("failed to unmarshal %s: %v", docID, err) + } + dataset[docID] = doc + } + + // Index mapping + indexMapping := NewIndexMapping() + + vecMapping := mapping.NewVectorFieldMapping() + vecMapping.Dims = 3 + vecMapping.Similarity = index.InnerProduct + indexMapping.DefaultMapping.AddFieldMappingsAt("vec", vecMapping) + indexMapping.DefaultMapping.AddFieldMappingsAt("vecB", vecMapping) + + // Nested chunks mapping for [{}] style + chunksMapping := mapping.NewDocumentMapping() + chunksMapping.AddFieldMappingsAt("vec", vecMapping) + indexMapping.DefaultMapping.AddSubDocumentMapping("chunks", chunksMapping) + + // Create and populate index + idx, err := New(tmpIndexPath, indexMapping) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := idx.Close(); err != nil { + t.Fatal(err) + } + }() + + batch := idx.NewBatch() + for docID, doc := range dataset { + if err := batch.Index(docID, doc); err != nil { + t.Fatal(err) + } + } + if err := idx.Batch(batch); err != nil { + t.Fatal(err) + } + + // Test: Single KNN query - basic functionality + t.Run("VecFieldSingle", func(t *testing.T) { + searchReq := NewSearchRequest(query.NewMatchNoneQuery()) + searchReq.AddKNN("vec", []float32{1, 1, 1}, 20, 1.0) + res, err := idx.Search(searchReq) + if err != nil { + t.Fatal(err) + } + // Inner product: score = sum(query_i * doc_i) + // doc1 vec=[10,10,10]: 1*10*3 = 30 + // doc2 vec best is [500,500,500]: 1*500*3 = 1500 + // doc3 vec best is [400,400,400]: 1*400*3 = 1200 + // doc4 vec=[1000,1000,1000]: 1*1000*3 = 3000 + expectedResult := []struct { + docID string + expectedScore float64 + }{ + {docID: "doc4", expectedScore: 3000}, + {docID: "doc2", expectedScore: 1500}, + {docID: "doc3", expectedScore: 1200}, + {docID: "doc1", expectedScore: 30}, + } + + if len(res.Hits) != len(expectedResult) { + t.Fatalf("expected %d hits, got %d", len(expectedResult), len(res.Hits)) + } + + for i, expected := range expectedResult { + if res.Hits[i].ID != expected.docID { + t.Fatalf("at rank %d, expected docID %s, got %s", i+1, expected.docID, res.Hits[i].ID) + } + if res.Hits[i].Score != expected.expectedScore { + t.Fatalf("at rank %d, expected score %v, got %v", i+1, expected.expectedScore, res.Hits[i].Score) + } + } + }) + + // Test: Single KNN query on vecB field + t.Run("VecBFieldSingle", func(t *testing.T) { + searchReq := NewSearchRequest(query.NewMatchNoneQuery()) + searchReq.AddKNN("vecB", []float32{1000, 1000, 1000}, 20, 1.0) + res, err := idx.Search(searchReq) + if err != nil { + t.Fatal(err) + } + // Inner product: score = sum(query_i * doc_i) for each dimension + // doc1: vecB=[100,100,100] -> 1000*100*3 = 300,000 + // doc2: vecB best is [990,990,990] -> 1000*990*3 = 2,970,000 + // doc3: vecB best is [850,850,850] -> 1000*850*3 = 2,550,000 + // doc4: vecB=[1,1,1] -> 1000*1*3 = 3,000 + expectedResult := []struct { + docID string + expectedScore float64 + }{ + {docID: "doc2", expectedScore: 2970000}, + {docID: "doc3", expectedScore: 2550000}, + {docID: "doc1", expectedScore: 300000}, + {docID: "doc4", expectedScore: 3000}, + } + + if len(res.Hits) != len(expectedResult) { + t.Fatalf("expected %d hits, got %d", len(expectedResult), len(res.Hits)) + } + + for i, expected := range expectedResult { + if res.Hits[i].ID != expected.docID { + t.Fatalf("at rank %d, expected docID %s, got %s", i+1, expected.docID, res.Hits[i].ID) + } + if res.Hits[i].Score != expected.expectedScore { + t.Fatalf("at rank %d, expected score %v, got %v", i+1, expected.expectedScore, res.Hits[i].Score) + } + } + }) + + // Test: Single KNN query on nested chunks.vec field + t.Run("ChunksVecFieldSingle", func(t *testing.T) { + searchReq := NewSearchRequest(query.NewMatchNoneQuery()) + searchReq.AddKNN("chunks.vec", []float32{1, 1, 1}, 20, 1.0) + searchReq.SortBy([]string{"_score", "docID"}) + res, err := idx.Search(searchReq) + if err != nil { + t.Fatal(err) + } + + // Only doc5 and doc6 have chunks.vec + // doc5 chunks: [10,10,10], [20,20,20], [30,30,30], [40,40,40] + // Best score: 1*40*3 = 120 + // doc6 chunks: [[10,10,10],[20,20,20]], [[30,30,30],[40,40,40]] + // Best score: 1*40*3 = 120 + if len(res.Hits) != 2 { + t.Fatalf("expected 2 hits, got %d", len(res.Hits)) + } + + // Both should have score 120 + for _, hit := range res.Hits { + if hit.ID != "doc5" && hit.ID != "doc6" { + t.Fatalf("unexpected docID %s, expected doc5 or doc6", hit.ID) + } + if hit.Score != 120 { + t.Fatalf("for %s, expected score 120, got %v", hit.ID, hit.Score) + } + } + }) +} + +// TestMultiVectorCosineNormalization verifies that multi-vector fields are +// normalized correctly with cosine similarity. Each sub-vector in a multi-vector +// should be independently normalized, producing correct similarity scores. +func TestMultiVectorCosineNormalization(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + const dims = 3 + + // Create index with cosine similarity + indexMapping := NewIndexMapping() + vecFieldMapping := mapping.NewVectorFieldMapping() + vecFieldMapping.Dims = dims + vecFieldMapping.Similarity = index.CosineSimilarity + + // Single-vector field + indexMapping.DefaultMapping.AddFieldMappingsAt("vec", vecFieldMapping) + // Multi-vector field + indexMapping.DefaultMapping.AddFieldMappingsAt("multi_vec", vecFieldMapping) + + idx, err := New(tmpIndexPath, indexMapping) + if err != nil { + t.Fatal(err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + docsString := []string{ + `{"vec": [3, 0, 0]}`, + `{"vec": [0, 4, 0]}`, + `{"multi_vec": [[3, 0, 0], [0, 4, 0]]}`, + } + + for i, docStr := range docsString { + var doc map[string]interface{} + err = json.Unmarshal([]byte(docStr), &doc) + if err != nil { + t.Fatal(err) + } + err = idx.Index(fmt.Sprintf("doc%d", i+1), doc) + if err != nil { + t.Fatal(err) + } + } + + // Query for X direction [1,0,0] + searchReq := NewSearchRequest(query.NewMatchNoneQuery()) + searchReq.AddKNN("vec", []float32{1, 0, 0}, 3, 1.0) + res, err := idx.Search(searchReq) + if err != nil { + t.Fatal(err) + } + if len(res.Hits) != 2 { + t.Fatalf("expected 2 hits, got %d", len(res.Hits)) + } + // Hit 1 should be doc1 with score 1.0 (perfect match) + if res.Hits[0].ID != "doc1" { + t.Fatalf("expected doc1 as first hit, got %s", res.Hits[0].ID) + } + if math.Abs(float64(res.Hits[0].Score-1.0)) > 1e-6 { + t.Fatalf("expected score 1.0, got %f", res.Hits[0].Score) + } + // Hit 2 should be doc2 with a score of 0.0 (orthogonal) + if res.Hits[1].ID != "doc2" { + t.Fatalf("expected doc2 as second hit, got %s", res.Hits[1].ID) + } + if math.Abs(float64(res.Hits[1].Score-0.0)) > 1e-6 { + t.Fatalf("expected score 0.0, got %f", res.Hits[1].Score) + } + + // Query for Y direction [0,1,0] + searchReq = NewSearchRequest(query.NewMatchNoneQuery()) + searchReq.AddKNN("vec", []float32{0, 1, 0}, 3, 1.0) + res, err = idx.Search(searchReq) + if err != nil { + t.Fatal(err) + } + if len(res.Hits) != 2 { + t.Fatalf("expected 2 hits, got %d", len(res.Hits)) + } + // Hit 1 should be doc2 with score 1.0 (perfect match) + if res.Hits[0].ID != "doc2" { + t.Fatalf("expected doc2 as first hit, got %s", res.Hits[0].ID) + } + if math.Abs(float64(res.Hits[0].Score-1.0)) > 1e-6 { + t.Fatalf("expected score 1.0, got %f", res.Hits[0].Score) + } + // Hit 2 should be doc1 with a score of 0.0 (orthogonal) + if res.Hits[1].ID != "doc1" { + t.Fatalf("expected doc1 as second hit, got %s", res.Hits[1].ID) + } + if math.Abs(float64(res.Hits[1].Score-0.0)) > 1e-6 { + t.Fatalf("expected score 0.0, got %f", res.Hits[1].Score) + } + + // Now test querying the nested multi-vector field + searchReq = NewSearchRequest(query.NewMatchNoneQuery()) + searchReq.AddKNN("multi_vec", []float32{1, 0, 0}, 3, 1.0) + res, err = idx.Search(searchReq) + if err != nil { + t.Fatal(err) + } + if len(res.Hits) != 1 { + t.Fatalf("expected 1 hit, got %d", len(res.Hits)) + } + // Hit should be doc3 with score 1.0 (perfect match on first sub-vector) + if res.Hits[0].ID != "doc3" { + t.Fatalf("expected doc3 as first hit, got %s", res.Hits[0].ID) + } + if math.Abs(float64(res.Hits[0].Score-1.0)) > 1e-6 { + t.Fatalf("expected score 1.0, got %f", res.Hits[0].Score) + } + // Query for Y direction [0,1,0] on nested field + searchReq = NewSearchRequest(query.NewMatchNoneQuery()) + searchReq.AddKNN("multi_vec", []float32{0, 1, 0}, 3, 1.0) + res, err = idx.Search(searchReq) + if err != nil { + t.Fatal(err) + } + if len(res.Hits) != 1 { + t.Fatalf("expected 1 hit, got %d", len(res.Hits)) + } + // Hit should be doc3 with score 1.0 (perfect match on second sub-vector) + if res.Hits[0].ID != "doc3" { + t.Fatalf("expected doc3 as first hit, got %s", res.Hits[0].ID) + } + if math.Abs(float64(res.Hits[0].Score-1.0)) > 1e-6 { + t.Fatalf("expected score 1.0, got %f", res.Hits[0].Score) + } +} + +func TestNumVecsStat(t *testing.T) { + + dataset, _, err := readDatasetAndQueries(testInputCompressedFile) + if err != nil { + t.Fatal(err) + } + documents := makeDatasetIntoDocuments(dataset) + + indexMapping := NewIndexMapping() + + contentFieldMapping := NewTextFieldMapping() + contentFieldMapping.Analyzer = en.AnalyzerName + indexMapping.DefaultMapping.AddFieldMappingsAt("content", contentFieldMapping) + + vecFieldMapping1 := mapping.NewVectorFieldMapping() + vecFieldMapping1.Dims = testDatasetDims + vecFieldMapping1.Similarity = index.EuclideanDistance + indexMapping.DefaultMapping.AddFieldMappingsAt("vector", vecFieldMapping1) + + tmpIndexPath := createTmpIndexPath(t) + index, err := New(tmpIndexPath, indexMapping) + if err != nil { + t.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + }() + + for i := 0; i < 10; i++ { + batch := index.NewBatch() + for j := 0; j < 3; j++ { + for k := 0; k < 10; k++ { + err := batch.Index(fmt.Sprintf("%d", i*30+j*10+k), documents[j*10+k]) + if err != nil { + t.Fatal(err) + } + } + } + err = index.Batch(batch) + if err != nil { + t.Fatal(err) + } + } + + statsMap := index.StatsMap() + + if indexStats, exists := statsMap["index"]; exists { + if indexStatsMap, ok := indexStats.(map[string]interface{}); ok { + v1, ok := indexStatsMap["field:vector:num_vectors"].(uint64) + if !ok || v1 != uint64(300) { + t.Fatalf("mismatch in the number of vectors, expected 300, got %d", indexStatsMap["field:vector:num_vectors"]) + } + } + } +} + +func TestIndexUpdateVector(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + indexMappingBefore := mapping.NewIndexMapping() + indexMappingBefore.TypeMapping = map[string]*mapping.DocumentMapping{} + indexMappingBefore.DefaultMapping = &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "vector", + Index: true, + Dims: 4, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "latency", + }, + }, + }, + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "vector", + Index: true, + Dims: 4, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "latency", + }, + }, + }, + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "vector_base64", + Index: true, + Dims: 4, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "latency", + }, + }, + }, + "d": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "vector_base64", + Index: true, + Dims: 4, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "latency", + }, + }, + }, + }, + Fields: []*mapping.FieldMapping{}, + } + indexMappingBefore.IndexDynamic = false + indexMappingBefore.StoreDynamic = false + indexMappingBefore.DocValuesDynamic = false + + index, err := New(tmpIndexPath, indexMappingBefore) + if err != nil { + t.Fatal(err) + } + doc1 := map[string]interface{}{"a": []float32{0.32894259691238403, 0.6973215341567993, 0.6835201978683472, 0.38296082615852356}, "b": []float32{0.32894259691238403, 0.6973215341567993, 0.6835201978683472, 0.38296082615852356}, "c": "L5MOPw7NID5SQMU9pHUoPw==", "d": "L5MOPw7NID5SQMU9pHUoPw=="} + doc2 := map[string]interface{}{"a": []float32{0.0018692062003538013, 0.41076546907424927, 0.5675257444381714, 0.45832985639572144}, "b": []float32{0.0018692062003538013, 0.41076546907424927, 0.5675257444381714, 0.45832985639572144}, "c": "czloP94ZCD71ldY+GbAOPw==", "d": "czloP94ZCD71ldY+GbAOPw=="} + doc3 := map[string]interface{}{"a": []float32{0.7853356599807739, 0.6904757618904114, 0.5643226504325867, 0.682637631893158}, "b": []float32{0.7853356599807739, 0.6904757618904114, 0.5643226504325867, 0.682637631893158}, "c": "Chh6P2lOqT47mjg/0odlPg==", "d": "Chh6P2lOqT47mjg/0odlPg=="} + batch := index.NewBatch() + err = batch.Index("001", doc1) + if err != nil { + t.Fatal(err) + } + err = batch.Index("002", doc2) + if err != nil { + t.Fatal(err) + } + err = batch.Index("003", doc3) + if err != nil { + t.Fatal(err) + } + err = index.Batch(batch) + if err != nil { + t.Fatal(err) + } + err = index.Close() + if err != nil { + t.Fatal(err) + } + + indexMappingAfter := mapping.NewIndexMapping() + indexMappingAfter.TypeMapping = map[string]*mapping.DocumentMapping{} + indexMappingAfter.DefaultMapping = &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "vector", + Index: true, + Dims: 4, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "latency", + }, + }, + }, + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "vector_base64", + Index: true, + Dims: 4, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "latency", + }, + }, + }, + "d": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "vector_base64", + Index: false, + Dims: 4, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "latency", + }, + }, + }, + }, + Fields: []*mapping.FieldMapping{}, + } + indexMappingAfter.IndexDynamic = false + indexMappingAfter.StoreDynamic = false + indexMappingAfter.DocValuesDynamic = false + + mappingString, err := json.Marshal(indexMappingAfter) + if err != nil { + t.Fatal(err) + } + config := map[string]interface{}{ + "updated_mapping": string(mappingString), + } + + index, err = OpenUsing(tmpIndexPath, config) + if err != nil { + t.Fatal(err) + } + + q1 := NewSearchRequest(NewMatchNoneQuery()) + q1.AddKNN("a", []float32{1, 2, 3, 4}, 3, 1.0) + res1, err := index.Search(q1) + if err != nil { + t.Fatal(err) + } + if len(res1.Hits) != 3 { + t.Fatalf("Expected 3 hits, got %d", len(res1.Hits)) + } + q2 := NewSearchRequest(NewMatchNoneQuery()) + q2.AddKNN("b", []float32{1, 2, 3, 4}, 3, 1.0) + res2, err := index.Search(q2) + if err != nil { + t.Fatal(err) + } + if len(res2.Hits) != 0 { + t.Fatalf("Expected 0 hits, got %d", len(res2.Hits)) + } + q3 := NewSearchRequest(NewMatchNoneQuery()) + q3.AddKNN("c", []float32{1, 2, 3, 4}, 3, 1.0) + res3, err := index.Search(q3) + if err != nil { + t.Fatal(err) + } + if len(res3.Hits) != 3 { + t.Fatalf("Expected 3 hits, got %d", len(res3.Hits)) + } + q4 := NewSearchRequest(NewMatchNoneQuery()) + q4.AddKNN("d", []float32{1, 2, 3, 4}, 3, 1.0) + res4, err := index.Search(q4) + if err != nil { + t.Fatal(err) + } + if len(res4.Hits) != 0 { + t.Fatalf("Expected 0 hits, got %d", len(res4.Hits)) + } +} + +func TestIndexInsightsTermFrequencies(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + mp := mapping.NewIndexMapping() + textMapping := mapping.NewTextFieldMapping() + textMapping.Analyzer = "en" + mp.DefaultMapping.AddFieldMappingsAt("text", textMapping) + + idx, err := New(tmpIndexPath, mp) + if err != nil { + t.Fatal(err) + } + defer func() { + err = idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + data := []map[string]string{ + { + "id": "one", + "text": "She sells sea shells by the sea shore", + }, + { + "id": "two", + "text": "The quick brown fox jumps over the lazy dog", + }, + { + "id": "three", + "text": "She sold sea shells to the person with the dog", + }, + { + "id": "four", + "text": "But there are a lot of dogs on the beach", + }, + { + "id": "five", + "text": "To hell with the foxes", + }, + { + "id": "six", + "text": "What about the dogs", + }, + { + "id": "seven", + "text": "Dogs are OK, foxes are not", + }, + } + + expectTermFreqs := []index.TermFreq{ + {Term: "dog", Frequency: 5}, + {Term: "fox", Frequency: 3}, + {Term: "sea", Frequency: 2}, + {Term: "shell", Frequency: 2}, + {Term: "beach", Frequency: 1}, + } + + for _, d := range data { + err = idx.Index(d["id"], d) + if err != nil { + t.Errorf("Error updating index: %v", err) + } + } + + insightsIdx, ok := idx.(InsightsIndex) + if !ok { + t.Fatal("index does not support insights") + } + + termFreqs, err := insightsIdx.TermFrequencies("text", 5, true) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(termFreqs, expectTermFreqs) { + t.Fatalf("term freqs do not match: got: %v, expected: %v", termFreqs, expectTermFreqs) + } +} + +func TestIndexInsightsCentroidCardinalities(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + vectorDims := 5 + + mp := mapping.NewIndexMapping() + vecFieldMapping := mapping.NewVectorFieldMapping() + vecFieldMapping.Dims = vectorDims + vecFieldMapping.Similarity = index.CosineSimilarity + mp.DefaultMapping.AddFieldMappingsAt("vec", vecFieldMapping) + + idx, err := New(tmpIndexPath, mp) + if err != nil { + t.Fatal(err) + } + defer func() { + err = idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + rand.Seed(time.Now().UnixNano()) + min, max := float32(-10.0), float32(10.0) + genRandomVector := func() []float32 { + vec := make([]float32, vectorDims) + for i := range vec { + vec[i] = min + rand.Float32()*(max-min) + } + return vec + } + + batch := idx.NewBatch() + for i := 1; i <= 50000; i++ { + if err = batch.Index(fmt.Sprintf("doc-%d", i), map[string]interface{}{ + "vec": genRandomVector(), + }); err != nil { + t.Fatalf("error indexing doc: %v", err) + } + + if i%200 == 0 { + err = idx.Batch(batch) + if err != nil { + t.Fatalf("Error adding batch to index: %v", err) + } + batch = idx.NewBatch() + } + } + + if batch.Size() > 0 { + // In case doc count is not a multiple of 200, we need to add the final batch + err = idx.Batch(batch) + if err != nil { + t.Errorf("Error adding final batch to index: %v", err) + } + } + + insightsIdx, ok := idx.(InsightsIndex) + if !ok { + t.Fatal("index does not support insights") + } + + centroids, err := insightsIdx.CentroidCardinalities("vec", 5, true) + if err != nil { + t.Fatal(err) + } + + if len(centroids) != 5 { + t.Fatalf("expected 5 centroids, got %d", len(centroids)) + } + + for _, entry := range centroids { + if len(entry.Index) == 0 { + t.Fatal("expected index name for each centroid") + } + } +} + +func TestHierarchicalNestedVectorSearch(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + dataset := ` + [ + { + "id": "doc1", + "items": [ + { + "description": "I like trains", + "embedding_vector": [ + 1, + 0, + 0 + ], + "type": "transport" + }, + { + "description": "I love pizza", + "embedding_vector": [ + 0, + 1, + 0 + ], + "type": "food" + } + ] + }, + { + "id": "doc2", + "items": [ + { + "description": "I go to school by bus", + "embedding_vector": [ + 0.9, + 0.1, + 0 + ], + "type": "transport" + }, + { + "description": "Sushi is delicious", + "embedding_vector": [ + 0, + 1, + 0 + ], + "type": "food" + } + ] + }, + { + "id": "doc3", + "items": [ + { + "description": "Hamburgers are tasty", + "embedding_vector": [ + 0, + 0.8, + 0.2 + ], + "type": "food" + }, + { + "description": "I enjoy biking", + "embedding_vector": [ + 0.7, + 0, + 0.3 + ], + "type": "transport" + } + ] + } + ]` + var documents []map[string]interface{} + err := json.Unmarshal([]byte(dataset), &documents) + if err != nil { + t.Fatalf("failed to unmarshal dataset: %v", err) + } + indexMapping := NewIndexMapping() + vecFieldMapping := mapping.NewVectorFieldMapping() + vecFieldMapping.Dims = 3 + vecFieldMapping.Similarity = index.CosineSimilarity + + typeMapping := mapping.NewTextFieldMapping() + typeMapping.Analyzer = keyword.Name + + descMapping := mapping.NewTextFieldMapping() + descMapping.Analyzer = en.AnalyzerName + + // items is NOT nested + itemsMapping := mapping.NewDocumentMapping() + itemsMapping.AddFieldMappingsAt("embedding_vector", vecFieldMapping) + itemsMapping.AddFieldMappingsAt("type", typeMapping) + itemsMapping.AddFieldMappingsAt("description", descMapping) + + indexMapping.DefaultMapping.AddSubDocumentMapping("items", itemsMapping) + idx, err := New(tmpIndexPath, indexMapping) + if err != nil { + t.Fatalf("failed to create index: %v", err) + } + defer func() { + if err := idx.Close(); err != nil { + t.Fatalf("failed to close index: %v", err) + } + }() + + batch := idx.NewBatch() + for _, doc := range documents { + err := batch.Index(doc["id"].(string), doc) + if err != nil { + t.Fatalf("failed to index document %s: %v", doc["id"], err) + } + } + err = idx.Batch(batch) + if err != nil { + t.Fatalf("failed to batch index documents: %v", err) + } + + // Plain vector search + searchReq := NewSearchRequest(query.NewMatchNoneQuery()) + searchReq.AddKNN("items.embedding_vector", []float32{0, 1, 0}, 5, 1.0) + searchReq.SortBy([]string{"-_score", "_id"}) + + res, err := idx.Search(searchReq) + if err != nil { + t.Fatalf("failed to execute search: %v", err) + } + + expectedOrder := []string{"doc1", "doc2", "doc3"} + expectedScores := []float64{1.0, 1.0, 0.970} + if len(res.Hits) != len(expectedOrder) { + t.Fatalf("expected %d hits, got %d", len(expectedOrder), len(res.Hits)) + } + for i, expectedID := range expectedOrder { + if res.Hits[i].ID != expectedID { + t.Fatalf("at rank %d, expected docID %s, got %s", i+1, expectedID, res.Hits[i].ID) + } + if math.Abs(res.Hits[i].Score-expectedScores[i]) > 0.01 { + t.Fatalf("at rank %d, expected score %.3f, got %.3f", i+1, expectedScores[i], res.Hits[i].Score) + } + } + + // Filtered vector search - should match output of plain vector search in non-nested case + filterQuery := NewTermQuery("transport") + filterQuery.SetField("items.type") + searchReq = NewSearchRequest(query.NewMatchNoneQuery()) + searchReq.AddKNNWithFilter("items.embedding_vector", []float32{0, 1, 0}, 5, 1.0, filterQuery) + searchReq.SortBy([]string{"-_score", "_id"}) + res, err = idx.Search(searchReq) + if err != nil { + t.Fatalf("failed to execute filtered search: %v", err) + } + if len(res.Hits) != len(expectedOrder) { + t.Fatalf("expected %d hits, got %d", len(expectedOrder), len(res.Hits)) + } + for i, expectedID := range expectedOrder { + if res.Hits[i].ID != expectedID { + t.Fatalf("at rank %d, expected docID %s, got %s", i+1, expectedID, res.Hits[i].ID) + } + if math.Abs(res.Hits[i].Score-expectedScores[i]) > 0.01 { + t.Fatalf("at rank %d, expected score %.3f, got %.3f", i+1, expectedScores[i], res.Hits[i].Score) + } + } + + // items IS nested + nestedItemsMapping := mapping.NewNestedDocumentMapping() + nestedItemsMapping.AddFieldMappingsAt("embedding_vector", vecFieldMapping) + nestedItemsMapping.AddFieldMappingsAt("type", typeMapping) + nestedItemsMapping.AddFieldMappingsAt("description", descMapping) + + indexMappingNested := NewIndexMapping() + indexMappingNested.DefaultMapping.AddSubDocumentMapping("items", nestedItemsMapping) + idxNested, err := New(tmpIndexPath+"_nested", indexMappingNested) + if err != nil { + t.Fatalf("failed to create nested index: %v", err) + } + defer func() { + if err := idxNested.Close(); err != nil { + t.Fatalf("failed to close nested index: %v", err) + } + }() + + batch = idxNested.NewBatch() + for _, doc := range documents { + err := batch.Index(doc["id"].(string), doc) + if err != nil { + t.Fatalf("failed to index document %s in nested index: %v", doc["id"], err) + } + } + err = idxNested.Batch(batch) + if err != nil { + t.Fatalf("failed to batch index documents in nested index: %v", err) + } + // Plain vector search on nested index + searchReq = NewSearchRequest(query.NewMatchNoneQuery()) + searchReq.AddKNN("items.embedding_vector", []float32{0, 1, 0}, 5, 1.0) + searchReq.SortBy([]string{"-_score", "_id"}) + + res, err = idxNested.Search(searchReq) + if err != nil { + t.Fatalf("failed to execute search on nested index: %v", err) + } + // Exact same behavior as non-nested in this case + if len(res.Hits) != len(expectedOrder) { + t.Fatalf("expected %d hits, got %d", len(expectedOrder), len(res.Hits)) + } + for i, expectedID := range expectedOrder { + if res.Hits[i].ID != expectedID { + t.Fatalf("at rank %d, expected docID %s, got %s", i+1, expectedID, res.Hits[i].ID) + } + if math.Abs(res.Hits[i].Score-expectedScores[i]) > 0.01 { + t.Fatalf("at rank %d, expected score %.3f, got %.3f", i+1, expectedScores[i], res.Hits[i].Score) + } + } + + // Filtered vector search on nested index - should NOT match output of plain vector search in nested case + filterQuery = NewTermQuery("transport") + filterQuery.SetField("items.type") + searchReq = NewSearchRequest(query.NewMatchNoneQuery()) + searchReq.AddKNNWithFilter("items.embedding_vector", []float32{0, 1, 0}, 5, 1.0, filterQuery) + searchReq.SortBy([]string{"-_score", "_id"}) + res, err = idxNested.Search(searchReq) + if err != nil { + t.Fatalf("failed to execute filtered search on nested index: %v", err) + } + expectedNestedOrder := []string{"doc2", "doc1", "doc3"} + expectedNestedScores := []float64{0.110, 0, 0} + if len(res.Hits) != len(expectedNestedOrder) { + t.Fatalf("expected %d hits, got %d", len(expectedNestedOrder), len(res.Hits)) + } + for i, expectedID := range expectedNestedOrder { + if res.Hits[i].ID != expectedID { + t.Fatalf("at rank %d, expected docID %s, got %s", i+1, expectedID, res.Hits[i].ID) + } + if math.Abs(res.Hits[i].Score-expectedNestedScores[i]) > 0.01 { + t.Fatalf("at rank %d, expected score %.3f, got %.3f", i+1, expectedNestedScores[i], res.Hits[i].Score) + } + } +} + +func TestKNNNullParams(t *testing.T) { + queries := []struct { + query []byte + }{ + {query: []byte(`{"knn": [{"field": "emb", "vector": [1, 2], "k": 3}]}`)}, + {query: []byte(`{"knn": [{"field": "emb", "params": null, "vector": [1, 2], "k": 3}]}`)}, + {query: []byte(`{"knn": [{"field": "emb","vector": [1, 2], "k": 3, "filter": null}]}`)}, + {query: []byte(`{"pre_search_data": null, "knn": [{"field": "emb", "vector": [1, 2], "k": 3}]}`)}, + } + + for _, q := range queries { + var searchReq SearchRequest + err := json.Unmarshal(q.query, &searchReq) + if err != nil { + t.Fatalf("failed to parse query: %v", err) + } + if len(searchReq.PreSearchData) > 0 { + t.Fatalf("expected no pre_search_data for query: %s, got %v", q.query, searchReq.PreSearchData) + } + for _, req := range searchReq.KNN { + if len(req.Params) > 0 { + t.Fatalf("expected no params for query: %s, got %v", q.query, req.Params) + } + if req.FilterQuery != nil { + t.Fatalf("expected no filter for query: %s, got %v", q.query, req.FilterQuery) + } + } + marshalled, err := json.Marshal(searchReq) + if err != nil { + t.Fatalf("failed to marshal search request: %v", err) + } + var unmarshalled SearchRequest + err = json.Unmarshal(marshalled, &unmarshalled) + if err != nil { + t.Fatalf("failed to unmarshal marshalled search request: %v", err) + } + if len(unmarshalled.PreSearchData) > 0 { + t.Fatalf("expected no pre_search_data after marshal/unmarshal for query: %s, got %v", q.query, unmarshalled.PreSearchData) + } + for _, req := range unmarshalled.KNN { + if len(req.Params) > 0 { + t.Fatalf("expected no params after marshal/unmarshal for query: %s, got %v", q.query, req.Params) + } + if req.FilterQuery != nil { + t.Fatalf("expected no filter after marshal/unmarshal for query: %s, got %v", q.query, req.FilterQuery) + } + } + } +} + +func TestVectorIndexExhaustion(t *testing.T) { + for optimization := range index.SupportedVectorIndexOptimizations { + t.Run(optimization, func(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + const dims = 8 + + vecFieldMapping := mapping.NewVectorFieldMapping() + vecFieldMapping.Dims = dims + vecFieldMapping.VectorIndexOptimizedFor = optimization + + indexMapping := NewIndexMapping() + indexMapping.DefaultMapping.AddFieldMappingsAt("vec", vecFieldMapping) + + idx, err := New(tmpIndexPath, indexMapping) + if err != nil { + t.Fatal(err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + docs := []struct { + id string + vec []float32 + }{ + {"doc1", []float32{1, 0, 0, 0, 0, 0, 0, 0}}, + {"doc2", []float32{0, 1, 0, 0, 0, 0, 0, 0}}, + {"doc3", []float32{0, 0, 1, 0, 0, 0, 0, 0}}, + } + + batch := idx.NewBatch() + for _, doc := range docs { + err = batch.Index(doc.id, map[string]interface{}{ + "vec": doc.vec, + }) + if err != nil { + t.Fatal(err) + } + } + err = idx.Batch(batch) + if err != nil { + t.Fatal(err) + } + + searchReq := NewSearchRequest(query.NewMatchNoneQuery()) + searchReq.AddKNN("vec", []float32{1, 0, 0, 0, 0, 0, 0, 0}, 10, 1.0) + res, err := idx.Search(searchReq) + if err != nil { + t.Fatal(err) + } + if len(res.Hits) != 3 { + t.Fatalf("expected 3 hits, got %d", len(res.Hits)) + } + if res.Hits[0].ID != "doc1" { + t.Fatalf("expected doc1 as top hit, got %s", res.Hits[0].ID) + } + }) + } +} + +// TestHybridExplainNotDropped verifies that a doc matching BOTH the text query +// and the KNN branch keeps a merged explanation (text + vector) whose value +// equals the summed score, i.e. the KNN part isn't dropped from the explain tree. +func TestHybridExplainNotDropped(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + // vdocBoth matches the text query (category=health) AND is the nearest + // vector to the query vector, so it exercises the hybrid merge path. + docs := map[string]map[string]interface{}{ + "vdocBoth": {"category": "health", "vector": []float32{0.1, 0.9, 0.2, 0.5}}, + "vdocVecOnly": {"vector": []float32{0.1, 0.9, 0.2, 0.5}}, + "vdocTextOnly": {"category": "health", "vector": []float32{0.1, 0.2, 0.9, 0.1}}, + "vdocNeither": {"category": "finance", "vector": []float32{0.8, 0.1, 0.1, 0.9}}, + } + + indexMapping := NewIndexMapping() + catFM := NewTextFieldMapping() + catFM.Analyzer = "keyword" + indexMapping.DefaultMapping.AddFieldMappingsAt("category", catFM) + + vecFM := mapping.NewVectorFieldMapping() + vecFM.Index = true + vecFM.Dims = 4 + vecFM.Similarity = "l2_norm" + indexMapping.DefaultMapping.AddFieldMappingsAt("vector", vecFM) + + idx, err := New(tmpIndexPath, indexMapping) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := idx.Close(); err != nil { + t.Fatal(err) + } + }() + + batch := idx.NewBatch() + for id, doc := range docs { + if err := batch.Index(id, doc); err != nil { + t.Fatal(err) + } + } + if err := idx.Batch(batch); err != nil { + t.Fatal(err) + } + + // Root-level text query (matches vdocBoth and vdocTextOnly) + KNN branch + // whose nearest neighbours are vdocBoth and vdocVecOnly. + tq := query.NewTermQuery("health") + tq.SetField("category") + req := NewSearchRequest(tq) + req.AddKNN("vector", []float32{0.1, 0.9, 0.2, 0.4}, 2, 1.0) + req.Explain = true + req.Fields = []string{"category"} + + res, err := idx.Search(req) + if err != nil { + t.Fatal(err) + } + + var checkedBoth bool + for _, hit := range res.Hits { + if hit.Expl == nil { + t.Fatalf("hit %s has no explanation", hit.ID) + } + // The invariant the bug violated: the explanation value must equal the + // score it is attached to. + if diff := hit.Expl.Value - hit.Score; diff > 1e-6 || diff < -1e-6 { + t.Errorf("hit %s: explanation.value=%v != score=%v (vector explanation dropped)", + hit.ID, hit.Expl.Value, hit.Score) + } + + if hit.ID == "vdocBoth" { + checkedBoth = true + // A both-branches hit must be a merged explanation carrying both + // the text and the vector sub-explanations. + if len(hit.Expl.Children) < 2 { + t.Errorf("hit vdocBoth: expected merged explanation with >=2 children "+ + "(text + vector), got message=%q value=%v children=%d", + hit.Expl.Message, hit.Expl.Value, len(hit.Expl.Children)) + } + } + } + if !checkedBoth { + t.Fatal("vdocBoth was not returned by the hybrid query") + } +} diff --git a/search_nested_test.go b/search_nested_test.go new file mode 100644 index 0000000..3c55c83 --- /dev/null +++ b/search_nested_test.go @@ -0,0 +1,1216 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/highlight/highlighter/ansi" + "github.com/blevesearch/bleve/v2/search/query" +) + +func createNestedIndexMapping() mapping.IndexMapping { + /* + company + ├── id + ├── name + ├── departments[] (nested) + │ ├── name + │ ├── budget + │ ├── employees[] (nested) + │ │ ├── name + │ │ ├── role + │ └── projects[] (nested) + │ ├── title + │ ├── status + └── locations[] (nested) + ├── city + ├── country + */ + + // Create the index mapping + imap := mapping.NewIndexMapping() + + // Create company mapping + companyMapping := mapping.NewDocumentMapping() + + // Company ID field + companyIDField := mapping.NewTextFieldMapping() + companyMapping.AddFieldMappingsAt("id", companyIDField) + + // Company name field + companyNameField := mapping.NewTextFieldMapping() + companyMapping.AddFieldMappingsAt("name", companyNameField) + + // Departments mapping + departmentsMapping := mapping.NewNestedDocumentMapping() + + // Department name field + deptNameField := mapping.NewTextFieldMapping() + departmentsMapping.AddFieldMappingsAt("name", deptNameField) + + // Department budget field + deptBudgetField := mapping.NewNumericFieldMapping() + departmentsMapping.AddFieldMappingsAt("budget", deptBudgetField) + + // Employees mapping + employeesMapping := mapping.NewNestedDocumentMapping() + + // Employee name field + empNameField := mapping.NewTextFieldMapping() + employeesMapping.AddFieldMappingsAt("name", empNameField) + + // Employee role field + empRoleField := mapping.NewTextFieldMapping() + employeesMapping.AddFieldMappingsAt("role", empRoleField) + + departmentsMapping.AddSubDocumentMapping("employees", employeesMapping) + + // Projects mapping + projectsMapping := mapping.NewNestedDocumentMapping() + + // Project title field + projTitleField := mapping.NewTextFieldMapping() + projectsMapping.AddFieldMappingsAt("title", projTitleField) + + // Project status field + projStatusField := mapping.NewTextFieldMapping() + projectsMapping.AddFieldMappingsAt("status", projStatusField) + + departmentsMapping.AddSubDocumentMapping("projects", projectsMapping) + + companyMapping.AddSubDocumentMapping("departments", departmentsMapping) + + // Locations mapping + locationsMapping := mapping.NewNestedDocumentMapping() + + // Location city field + cityField := mapping.NewTextFieldMapping() + locationsMapping.AddFieldMappingsAt("city", cityField) + + // Location country field + countryField := mapping.NewTextFieldMapping() + locationsMapping.AddFieldMappingsAt("country", countryField) + + companyMapping.AddSubDocumentMapping("locations", locationsMapping) + + // Add company to type mapping + imap.DefaultMapping.AddSubDocumentMapping("company", companyMapping) + + return imap +} + +func TestNestedPrefixes(t *testing.T) { + imap := createNestedIndexMapping() + + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + idx, err := New(tmpIndexPath, imap) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := idx.Close(); err != nil { + t.Fatal(err) + } + }() + + nmap, ok := imap.(mapping.NestedMapping) + if !ok { + t.Fatal("index mapping is not a NestedMapping") + } + + // ---------------------------------------------------------------------- + // Test 1: Employee Role AND Employee Name + // ---------------------------------------------------------------------- + fs := search.NewFieldSet() + fs.AddField("company.departments.employees.role") + fs.AddField("company.departments.employees.name") + + expectedCommon := 2 + expectedMax := 2 + + common, max := nmap.NestedDepth(fs) + if common != expectedCommon || max != expectedMax { + t.Fatalf("Test1: expected (common=%d, max=%d), got (common=%d, max=%d)", + expectedCommon, expectedMax, common, max) + } + + // ---------------------------------------------------------------------- + // Test 2: Employee Role AND Employee Name AND Department Name + // ---------------------------------------------------------------------- + fs = search.NewFieldSet() + fs.AddField("company.departments.employees.role") + fs.AddField("company.departments.employees.name") + fs.AddField("company.departments.name") + + expectedCommon = 1 + expectedMax = 2 // employees nested deeper + + common, max = nmap.NestedDepth(fs) + if common != expectedCommon || max != expectedMax { + t.Fatalf("Test2: expected (common=%d, max=%d), got (common=%d, max=%d)", + expectedCommon, expectedMax, common, max) + } + + // ---------------------------------------------------------------------- + // Test 3: Employee Role AND Location City + // ---------------------------------------------------------------------- + fs = search.NewFieldSet() + fs.AddField("company.departments.employees.role") + fs.AddField("company.locations.city") + + expectedCommon = 0 + expectedMax = 2 // employees deeper than locations (1) + + common, max = nmap.NestedDepth(fs) + if common != expectedCommon || max != expectedMax { + t.Fatalf("Test3: expected (common=%d, max=%d), got (common=%d, max=%d)", + expectedCommon, expectedMax, common, max) + } + + // ---------------------------------------------------------------------- + // Test 4: Company Name AND Location Country + // ---------------------------------------------------------------------- + fs = search.NewFieldSet() + fs.AddField("company.name") + fs.AddField("company.locations.country") + fs.AddField("company.locations.city") + + expectedCommon = 0 + expectedMax = 1 // locations.country and locations.city share depth 1 + + common, max = nmap.NestedDepth(fs) + if common != expectedCommon || max != expectedMax { + t.Fatalf("Test4: expected (common=%d, max=%d), got (common=%d, max=%d)", + expectedCommon, expectedMax, common, max) + } + + // ---------------------------------------------------------------------- + // Test 5: Department Budget AND Project Status AND Employee Name + // ---------------------------------------------------------------------- + fs = search.NewFieldSet() + fs.AddField("company.departments.budget") + fs.AddField("company.departments.projects.status") + fs.AddField("company.departments.employees.name") + + expectedCommon = 1 + expectedMax = 2 // employees + projects go deeper + + common, max = nmap.NestedDepth(fs) + if common != expectedCommon || max != expectedMax { + t.Fatalf("Test5: expected (common=%d, max=%d), got (common=%d, max=%d)", + expectedCommon, expectedMax, common, max) + } + + // ---------------------------------------------------------------------- + // Test 6: Single Field + // ---------------------------------------------------------------------- + fs = search.NewFieldSet() + fs.AddField("company.id") + + expectedCommon = 0 + expectedMax = 0 + + common, max = nmap.NestedDepth(fs) + if common != expectedCommon || max != expectedMax { + t.Fatalf("Test6: expected (common=%d, max=%d), got (common=%d, max=%d)", + expectedCommon, expectedMax, common, max) + } + + // ---------------------------------------------------------------------- + // Test 7: No Fields + // ---------------------------------------------------------------------- + fs = search.NewFieldSet() + + expectedCommon = 0 + expectedMax = 0 + + common, max = nmap.NestedDepth(fs) + if common != expectedCommon || max != expectedMax { + t.Fatalf("Test7: expected (common=%d, max=%d), got (common=%d, max=%d)", + expectedCommon, expectedMax, common, max) + } + + // ---------------------------------------------------------------------- + // Test 8: All Fields + // ---------------------------------------------------------------------- + fs = search.NewFieldSet() + fs.AddField("company.id") + fs.AddField("company.name") + fs.AddField("company.departments.name") + fs.AddField("company.departments.budget") + fs.AddField("company.departments.employees.name") + fs.AddField("company.departments.employees.role") + fs.AddField("company.departments.projects.title") + fs.AddField("company.departments.projects.status") + fs.AddField("company.locations.city") + fs.AddField("company.locations.country") + + expectedCommon = 0 // spans different contexts + expectedMax = 2 + + common, max = nmap.NestedDepth(fs) + if common != expectedCommon || max != expectedMax { + t.Fatalf("Test8: expected (common=%d, max=%d), got (common=%d, max=%d)", + expectedCommon, expectedMax, common, max) + } + + // ---------------------------------------------------------------------- + // Test 9: Project Title AND Project Status + // ---------------------------------------------------------------------- + fs = search.NewFieldSet() + fs.AddField("company.departments.projects.title") + fs.AddField("company.departments.projects.status") + + expectedCommon = 2 + expectedMax = 2 + + common, max = nmap.NestedDepth(fs) + if common != expectedCommon || max != expectedMax { + t.Fatalf("Test9: expected (common=%d, max=%d), got (common=%d, max=%d)", + expectedCommon, expectedMax, common, max) + } + + // ---------------------------------------------------------------------- + // Test 10: Department Name AND Location Country + // ---------------------------------------------------------------------- + fs = search.NewFieldSet() + fs.AddField("company.departments.name") + fs.AddField("company.locations.country") + fs.AddField("company.locations.city") + + expectedCommon = 0 + expectedMax = 1 // locations share depth 1 + + common, max = nmap.NestedDepth(fs) + if common != expectedCommon || max != expectedMax { + t.Fatalf("Test10: expected (common=%d, max=%d), got (common=%d, max=%d)", + expectedCommon, expectedMax, common, max) + } +} + +func TestNestedConjunctionQuery(t *testing.T) { + imap := createNestedIndexMapping() + err := imap.Validate() + if err != nil { + t.Fatalf("expected valid nested index mapping, got error: %v", err) + } + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + idx, err := New(tmpIndexPath, imap) + if err != nil { + t.Fatal(err) + } + defer func() { + err = idx.Close() + if err != nil { + t.Fatal(err) + } + }() + // Index 3 sample documents + docs := []struct { + id string + data string + }{ + { + id: "doc1", + data: `{ + "company": { + "id": "c1", + "name": "TechCorp", + "departments": [ + { + "name": "Engineering", + "budget": 2000000, + "employees": [ + {"name": "Alice", "role": "Engineer"}, + {"name": "Bob", "role": "Manager"} + ], + "projects": [ + {"title": "Project X", "status": "ongoing"}, + {"title": "Project Y", "status": "completed"} + ] + }, + { + "name": "Sales", + "budget": 300000, + "employees": [ + {"name": "Eve", "role": "Salesperson"}, + {"name": "Mallory", "role": "Manager"} + ], + "projects": [ + {"title": "Project A", "status": "completed"}, + {"title": "Project B", "status": "ongoing"} + ] + } + ], + "locations": [ + {"city": "Athens", "country": "Greece"}, + {"city": "Berlin", "country": "USA"} + ] + } + }`, + }, + { + id: "doc2", + data: `{ + "company" : { + "id": "c2", + "name": "BizInc", + "departments": [ + { + "name": "Marketing", + "budget": 800000, + "employees": [ + {"name": "Eve", "role": "Marketer"}, + {"name": "David", "role": "Manager"} + ], + "projects": [ + {"title": "Project Z", "status": "ongoing"}, + {"title": "Project W", "status": "planned"} + ] + }, + { + "name": "Engineering", + "budget": 800000, + "employees": [ + {"name": "Frank", "role": "Manager"}, + {"name": "Grace", "role": "Engineer"} + ], + "projects": [ + {"title": "Project Alpha", "status": "completed"}, + {"title": "Project Beta", "status": "ongoing"} + ] + } + ], + "locations": [ + {"city": "Athens", "country": "USA"}, + {"city": "London", "country": "UK"} + ] + } + }`, + }, + { + id: "doc3", + data: `{ + "company": { + "id": "c3", + "name": "WebSolutions", + "departments": [ + { + "name": "HR", + "budget": 800000, + "employees": [ + {"name": "Eve", "role": "Manager"}, + {"name": "Frank", "role": "HR"} + ], + "projects": [ + {"title": "Project Beta", "status": "completed"}, + {"title": "Project B", "status": "ongoing"} + ] + }, + { + "name": "Engineering", + "budget": 200000, + "employees": [ + {"name": "Heidi", "role": "Support Engineer"}, + {"name": "Ivan", "role": "Manager"} + ], + "projects": [ + {"title": "Project Helpdesk", "status": "ongoing"}, + {"title": "Project FAQ", "status": "completed"} + ] + } + ], + "locations": [ + {"city": "Edinburgh", "country": "UK"}, + {"city": "London", "country": "Canada"} + ] + } + }`, + }, + } + + for _, doc := range docs { + var dataMap map[string]interface{} + err := json.Unmarshal([]byte(doc.data), &dataMap) + if err != nil { + t.Fatalf("failed to unmarshal document %s: %v", doc.id, err) + } + err = idx.Index(doc.id, dataMap) + if err != nil { + t.Fatalf("failed to index document %s: %v", doc.id, err) + } + } + + var buildReq = func(subQueries []query.Query) *SearchRequest { + rv := NewSearchRequest(query.NewConjunctionQuery(subQueries)) + rv.SortBy([]string{"_id"}) + rv.Fields = []string{"*"} + rv.Highlight = NewHighlightWithStyle(ansi.Name) + return rv + } + + var ( + req *SearchRequest + res *SearchResult + deptNameQuery *query.MatchQuery + deptBudgetQuery *query.NumericRangeQuery + empNameQuery *query.MatchQuery + empRoleQuery *query.MatchQuery + projTitleQuery *query.MatchPhraseQuery + projStatusQuery *query.MatchQuery + countryQuery *query.MatchQuery + cityQuery *query.MatchQuery + ) + + // Test 1: Find companies with a department named "Engineering" AND budget over 900000 + deptNameQuery = query.NewMatchQuery("Engineering") + deptNameQuery.SetField("company.departments.name") + + min := float64(800000) + deptBudgetQuery = query.NewNumericRangeQuery(&min, nil) + deptBudgetQuery.SetField("company.departments.budget") + + req = buildReq([]query.Query{deptNameQuery, deptBudgetQuery}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 2 { + t.Fatalf("expected 2 hit, got %d", len(res.Hits)) + } + if res.Hits[0].ID != "doc1" || res.Hits[1].ID != "doc2" { + t.Fatalf("unexpected hit IDs: %v, %v", res.Hits[0].ID, res.Hits[1].ID) + } + + // Test 2: Find companies with an employee named "Eve" AND project status "completed" + empNameQuery = query.NewMatchQuery("Eve") + empNameQuery.SetField("company.departments.employees.name") + + projStatusQuery = query.NewMatchQuery("completed") + projStatusQuery.SetField("company.departments.projects.status") + + req = buildReq([]query.Query{empNameQuery, projStatusQuery}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 2 { + t.Fatalf("expected 2 hits, got %d", len(res.Hits)) + } + if res.Hits[0].ID != "doc1" || res.Hits[1].ID != "doc3" { + t.Fatalf("unexpected hit IDs: %v, %v", res.Hits[0].ID, res.Hits[1].ID) + } + + // Test 3: Find companies located in "Athens, USA" AND with an Engineering department + countryQuery = query.NewMatchQuery("USA") + countryQuery.SetField("company.locations.country") + + cityQuery = query.NewMatchQuery("Athens") + cityQuery.SetField("company.locations.city") + + locQuery := query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery}) + + deptNameQuery = query.NewMatchQuery("Engineering") + deptNameQuery.SetField("company.departments.name") + + req = buildReq([]query.Query{locQuery, deptNameQuery}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 1 { + t.Fatalf("expected 1 hit, got %d", len(res.Hits)) + } + if res.Hits[0].ID != "doc2" { + t.Fatalf("unexpected hit ID: %v", res.Hits[0].ID) + } + + // Test 4a: Find companies located in "Athens, USA" AND with an Engineering department with a budget over 1M + countryQuery = query.NewMatchQuery("USA") + countryQuery.SetField("company.locations.country") + + cityQuery = query.NewMatchQuery("Athens") + cityQuery.SetField("company.locations.city") + + locQuery = query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery}) + + deptNameQuery = query.NewMatchQuery("Engineering") + deptNameQuery.SetField("company.departments.name") + + min = float64(1000000) + deptBudgetQuery = query.NewNumericRangeQuery(&min, nil) + deptBudgetQuery.SetField("company.departments.budget") + + deptQuery := query.NewConjunctionQuery([]query.Query{deptNameQuery, deptBudgetQuery}) + + req = buildReq([]query.Query{locQuery, deptQuery}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 0 { + t.Fatalf("expected 0 hits, got %d", len(res.Hits)) + } + + // Test 4b: Find companies located in "Athens, Greece" AND with an Engineering department with a budget over 1M + countryQuery = query.NewMatchQuery("Greece") + countryQuery.SetField("company.locations.country") + + cityQuery = query.NewMatchQuery("Athens") + cityQuery.SetField("company.locations.city") + + locQuery = query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery}) + + deptNameQuery = query.NewMatchQuery("Engineering") + deptNameQuery.SetField("company.departments.name") + + min = float64(1000000) + deptBudgetQuery = query.NewNumericRangeQuery(&min, nil) + deptBudgetQuery.SetField("company.departments.budget") + + deptQuery = query.NewConjunctionQuery([]query.Query{deptNameQuery, deptBudgetQuery}) + + req = buildReq([]query.Query{locQuery, deptQuery}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 1 { + t.Fatalf("expected 1 hits, got %d", len(res.Hits)) + } + if res.Hits[0].ID != "doc1" { + t.Fatalf("unexpected hit ID: %v", res.Hits[0].ID) + } + + // Test 5a: Find companies with an employee named "Frank" AND role "Manager" whose department is + // handling a project titled "Project Beta" which is marked as "completed" + empNameQuery = query.NewMatchQuery("Frank") + empNameQuery.SetField("company.departments.employees.name") + + empRoleQuery = query.NewMatchQuery("Manager") + empRoleQuery.SetField("company.departments.employees.role") + + empQuery := query.NewConjunctionQuery([]query.Query{empNameQuery, empRoleQuery}) + + projTitleQuery = query.NewMatchPhraseQuery("Project Beta") + projTitleQuery.SetField("company.departments.projects.title") + + projStatusQuery = query.NewMatchQuery("completed") + projStatusQuery.SetField("company.departments.projects.status") + + projQuery := query.NewConjunctionQuery([]query.Query{projTitleQuery, projStatusQuery}) + + req = buildReq([]query.Query{empQuery, projQuery}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 0 { + t.Fatalf("expected 0 hit, got %d", len(res.Hits)) + } + + // Test 5b: Find companies with an employee named "Frank" AND role "Manager" whose department is + // handling a project titled "Project Beta" which is marked as "ongoing" + empNameQuery = query.NewMatchQuery("Frank") + empNameQuery.SetField("company.departments.employees.name") + + empRoleQuery = query.NewMatchQuery("Manager") + empRoleQuery.SetField("company.departments.employees.role") + + empQuery = query.NewConjunctionQuery([]query.Query{empNameQuery, empRoleQuery}) + + projTitleQuery = query.NewMatchPhraseQuery("Project Beta") + projTitleQuery.SetField("company.departments.projects.title") + + projStatusQuery = query.NewMatchQuery("ongoing") + projStatusQuery.SetField("company.departments.projects.status") + + projQuery = query.NewConjunctionQuery([]query.Query{projTitleQuery, projStatusQuery}) + + req = buildReq([]query.Query{empQuery, projQuery}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 1 { + t.Fatalf("expected 1 hit, got %d", len(res.Hits)) + } + if res.Hits[0].ID != "doc2" { + t.Fatalf("unexpected hit ID: %v", res.Hits[0].ID) + } + + // Test 6a: Find companies with an employee named "Eve" AND role "Manager" + // who is working in a department located in "London, UK" + empNameQuery = query.NewMatchQuery("Eve") + empNameQuery.SetField("company.departments.employees.name") + + empRoleQuery = query.NewMatchQuery("Manager") + empRoleQuery.SetField("company.departments.employees.role") + + empQuery = query.NewConjunctionQuery([]query.Query{empNameQuery, empRoleQuery}) + + countryQuery = query.NewMatchQuery("UK") + countryQuery.SetField("company.locations.country") + + cityQuery = query.NewMatchQuery("London") + cityQuery.SetField("company.locations.city") + + locQuery = query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery}) + + req = buildReq([]query.Query{empQuery, locQuery}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 0 { + t.Fatalf("expected 0 hit, got %d", len(res.Hits)) + } + + // Test 6b: Find companies with an employee named "Eve" AND role "Manager" + // who is working in a department located in "London, Canada" + empNameQuery = query.NewMatchQuery("Eve") + empNameQuery.SetField("company.departments.employees.name") + + empRoleQuery = query.NewMatchQuery("Manager") + empRoleQuery.SetField("company.departments.employees.role") + + empQuery = query.NewConjunctionQuery([]query.Query{empNameQuery, empRoleQuery}) + + countryQuery = query.NewMatchQuery("Canada") + countryQuery.SetField("company.locations.country") + + cityQuery = query.NewMatchQuery("London") + cityQuery.SetField("company.locations.city") + + locQuery = query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery}) + + req = buildReq([]query.Query{empQuery, locQuery}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 1 { + t.Fatalf("expected 1 hit, got %d", len(res.Hits)) + } + if res.Hits[0].ID != "doc3" { + t.Fatalf("unexpected hit ID: %v", res.Hits[0].ID) + } + + // Test 7a: Find companies where Ivan the Manager works London, UK + + empNameQuery = query.NewMatchQuery("Ivan") + empNameQuery.SetField("company.departments.employees.name") + + empRoleQuery = query.NewMatchQuery("Manager") + empRoleQuery.SetField("company.departments.employees.role") + + empQuery = query.NewConjunctionQuery([]query.Query{empNameQuery, empRoleQuery}) + + countryQuery = query.NewMatchQuery("UK") + countryQuery.SetField("company.locations.country") + + cityQuery = query.NewMatchQuery("London") + cityQuery.SetField("company.locations.city") + + locQuery = query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery}) + + req = buildReq([]query.Query{empQuery, locQuery}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 0 { + t.Fatalf("expected 0 hit, got %d", len(res.Hits)) + } + + // Test 7b: Find companies where Ivan the Manager works London, Canada + + empNameQuery = query.NewMatchQuery("Ivan") + empNameQuery.SetField("company.departments.employees.name") + + empRoleQuery = query.NewMatchQuery("Manager") + empRoleQuery.SetField("company.departments.employees.role") + + empQuery = query.NewConjunctionQuery([]query.Query{empNameQuery, empRoleQuery}) + + countryQuery = query.NewMatchQuery("Canada") + countryQuery.SetField("company.locations.country") + + cityQuery = query.NewMatchQuery("London") + cityQuery.SetField("company.locations.city") + + locQuery = query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery}) + + req = buildReq([]query.Query{empQuery, locQuery}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 1 { + t.Fatalf("expected 1 hit, got %d", len(res.Hits)) + } + if res.Hits[0].ID != "doc3" { + t.Fatalf("unexpected hit ID: %v", res.Hits[0].ID) + } + + // Test 8: Find companies where Frank the Manager works in Engineering department located in London, UK + empNameQuery = query.NewMatchQuery("Frank") + empNameQuery.SetField("company.departments.employees.name") + + empRoleQuery = query.NewMatchQuery("Manager") + empRoleQuery.SetField("company.departments.employees.role") + + empQuery = query.NewConjunctionQuery([]query.Query{empNameQuery, empRoleQuery}) + + deptNameQuery = query.NewMatchQuery("Engineering") + deptNameQuery.SetField("company.departments.name") + + deptQuery = query.NewConjunctionQuery([]query.Query{empQuery, deptNameQuery}) + + countryQuery = query.NewMatchQuery("UK") + countryQuery.SetField("company.locations.country") + + cityQuery = query.NewMatchQuery("London") + cityQuery.SetField("company.locations.city") + + locQuery = query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery}) + + req = buildReq([]query.Query{deptQuery, locQuery}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 1 { + t.Fatalf("expected 1 hit, got %d", len(res.Hits)) + } + if res.Hits[0].ID != "doc2" { + t.Fatalf("unexpected hit ID: %v", res.Hits[0].ID) + } + + // Test 9: Match_All query must return only top-level documents + matchAllQuery := query.NewMatchAllQuery() + req = buildReq([]query.Query{matchAllQuery}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 3 { + t.Fatalf("expected 3 hits, got %d", len(res.Hits)) + } + + // Test 10: DocID query must return only top-level documents + docIDQuery := query.NewDocIDQuery([]string{"doc1", "doc2", "doc3", "doc2_$company.locations_$0", "doc3_$company.departments_$0_$company.departments.employees_$0"}) + req = buildReq([]query.Query{docIDQuery}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 3 { + t.Fatalf("expected 3 hits, got %d", len(res.Hits)) + } + + // Test 11: Boolean query in Filter-only mode must return correct top-level documents + empNameQuery = query.NewMatchQuery("Frank") + empNameQuery.SetField("company.departments.employees.name") + + boolQuery := query.NewBooleanQuery(nil, nil, nil) + boolQuery.AddFilter(empNameQuery) + + req = buildReq([]query.Query{boolQuery}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 2 { + t.Fatalf("expected 2 hits, got %d", len(res.Hits)) + } + if res.Hits[0].ID != "doc2" || res.Hits[1].ID != "doc3" { + t.Fatalf("unexpected hit IDs: %v, %v", res.Hits[0].ID, res.Hits[1].ID) + } + + // Test 12: Boolean query Must clause should work in nested context + empNameQuery = query.NewMatchQuery("Ivan") + empNameQuery.SetField("company.departments.employees.name") + + empRoleQuery = query.NewMatchQuery("Manager") + empRoleQuery.SetField("company.departments.employees.role") + + empQuery = query.NewConjunctionQuery([]query.Query{empNameQuery, empRoleQuery}) + + countryQuery = query.NewMatchQuery("Canada") + countryQuery.SetField("company.locations.country") + + cityQuery = query.NewMatchQuery("London") + cityQuery.SetField("company.locations.city") + + locQuery = query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery}) + + boolQuery = query.NewBooleanQuery([]query.Query{empQuery, locQuery}, nil, nil) + req = buildReq([]query.Query{boolQuery}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 1 { + t.Fatalf("expected 1 hit, got %d", len(res.Hits)) + } + if res.Hits[0].ID != "doc3" { + t.Fatalf("unexpected hit ID: %v", res.Hits[0].ID) + } + + // Test 13: Queries targetting _all field should: + // - match only top-level fields when no specific field is set + // - not match nested fields when no specific field is set + // - work correctly when combined with nested field queries, + // returning only top-level documents where both conditions are met + allRootFieldQuery := query.NewMatchQuery("TechCorp") + + req = buildReq([]query.Query{allRootFieldQuery}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 1 { + t.Fatalf("expected 1 hit, got %d", len(res.Hits)) + } + if res.Hits[0].ID != "doc1" { + t.Fatalf("unexpected hit ID: %v", res.Hits[0].ID) + } + + allNestedFieldQuery := query.NewMatchQuery("Alice") + req = buildReq([]query.Query{allNestedFieldQuery}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 0 { + t.Fatalf("expected 0 hits, got %d", len(res.Hits)) + } + + allMixedQuery := buildReq([]query.Query{allRootFieldQuery, allNestedFieldQuery}) + res, err = idx.Search(allMixedQuery) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 0 { + t.Fatalf("expected 0 hits, got %d", len(res.Hits)) + } + + nestedFieldQuery := query.NewMatchQuery("Alice") + nestedFieldQuery.SetField("company.departments.employees.name") + + allMixedQueryWithNested := buildReq([]query.Query{allRootFieldQuery, nestedFieldQuery}) + res, err = idx.Search(allMixedQueryWithNested) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 1 { + t.Fatalf("expected 1 hit, got %d", len(res.Hits)) + } + if res.Hits[0].ID != "doc1" { + t.Fatalf("unexpected hit ID: %v", res.Hits[0].ID) + } + + empNameQuery = query.NewMatchQuery("Frank") + empNameQuery.SetField("company.departments.employees.name") + + empRoleQuery = query.NewMatchQuery("Manager") + empRoleQuery.SetField("company.departments.employees.role") + + empQuery = query.NewConjunctionQuery([]query.Query{empNameQuery, empRoleQuery}) + + deptNameQuery = query.NewMatchQuery("Engineering") + deptNameQuery.SetField("company.departments.name") + + deptQuery = query.NewConjunctionQuery([]query.Query{empQuery, deptNameQuery}) + + countryQuery = query.NewMatchQuery("UK") + countryQuery.SetField("company.locations.country") + + cityQuery = query.NewMatchQuery("London") + cityQuery.SetField("company.locations.city") + + locQuery = query.NewConjunctionQuery([]query.Query{countryQuery, cityQuery}) + + // mixed queries with _all field and _id field should match at root level always + companyNameAllQuery := query.NewMatchQuery("BizInc") + matchAllQuery = query.NewMatchAllQuery() + req = buildReq([]query.Query{deptQuery, locQuery, companyNameAllQuery, matchAllQuery}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 1 { + t.Fatalf("expected 1 hit, got %d", len(res.Hits)) + } + if res.Hits[0].ID != "doc2" { + t.Fatalf("unexpected hit ID: %v", res.Hits[0].ID) + } + + companyNameAllQueryNoMatch := query.NewMatchQuery("WebSolutions") + req = buildReq([]query.Query{deptQuery, locQuery, companyNameAllQueryNoMatch}) + res, err = idx.Search(req) + if err != nil { + t.Fatalf("search failed: %v", err) + } + if len(res.Hits) != 0 { + t.Fatalf("expected 0 hits, got %d", len(res.Hits)) + } +} + +func TestNestedArrayConjunctionQuery(t *testing.T) { + imap := NewIndexMapping() + groupsMapping := mapping.NewNestedDocumentMapping() + + nameField := mapping.NewTextFieldMapping() + groupsMapping.AddFieldMappingsAt("first_name", nameField) + groupsMapping.AddFieldMappingsAt("last_name", nameField) + + imap.DefaultMapping.AddSubDocumentMapping("groups", groupsMapping) + + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + idx, err := New(tmpIndexPath, imap) + if err != nil { + t.Fatal(err) + } + defer func() { + err = idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + docs := []string{ + `{ + "groups": [ + [ + { + "first_name": "Alice", + "last_name": "Smith" + }, + { + "first_name": "Bob", + "last_name": "Johnson" + } + ], + [ + { + "first_name": "Charlie", + "last_name": "Williams" + }, + { + "first_name": "Diana", + "last_name": "Brown" + } + ] + ] + }`, + `{ + "groups": [ + { + "first_name": "Alice", + "last_name": "Smith" + }, + { + "first_name": "Bob", + "last_name": "Johnson" + }, + { + "first_name": "Charlie", + "last_name": "Williams" + }, + { + "first_name": "Diana", + "last_name": "Brown" + } + ] + }`, + } + + for i, doc := range docs { + var dataMap map[string]interface{} + err := json.Unmarshal([]byte(doc), &dataMap) + if err != nil { + t.Fatalf("failed to unmarshal document %d: %v", i, err) + } + err = idx.Index(fmt.Sprintf("%d", i+1), dataMap) + if err != nil { + t.Fatalf("failed to index document %d: %v", i, err) + } + } + + var ( + firstNameQuery *query.MatchQuery + lastNameQuery *query.MatchQuery + conjQuery *query.ConjunctionQuery + searchReq *SearchRequest + res *SearchResult + ) + + // Search for documents where first_name is "Alice" AND last_name is "Johnson" + firstNameQuery = query.NewMatchQuery("Alice") + firstNameQuery.SetField("groups.first_name") + + lastNameQuery = query.NewMatchQuery("Johnson") + lastNameQuery.SetField("groups.last_name") + + conjQuery = query.NewConjunctionQuery([]query.Query{firstNameQuery, lastNameQuery}) + + searchReq = NewSearchRequest(conjQuery) + searchReq.SortBy([]string{"_id"}) + + res, err = idx.Search(searchReq) + if err != nil { + t.Fatalf("search failed: %v", err) + } + + if len(res.Hits) != 0 { + t.Fatalf("expected 0 hits, got %d", len(res.Hits)) + } + + // Search for documents where first_name is "Bob" AND last_name is "Johnson" + firstNameQuery = query.NewMatchQuery("Bob") + firstNameQuery.SetField("groups.first_name") + + lastNameQuery = query.NewMatchQuery("Johnson") + lastNameQuery.SetField("groups.last_name") + + conjQuery = query.NewConjunctionQuery([]query.Query{firstNameQuery, lastNameQuery}) + + searchReq = NewSearchRequest(conjQuery) + searchReq.SortBy([]string{"_id"}) + + res, err = idx.Search(searchReq) + if err != nil { + t.Fatalf("search failed: %v", err) + } + + if len(res.Hits) != 2 { + t.Fatalf("expected 2 hits, got %d", len(res.Hits)) + } + + if res.Hits[0].ID != "1" || res.Hits[1].ID != "2" { + t.Fatalf("unexpected hit IDs: %v, %v", res.Hits[0].ID, res.Hits[1].ID) + } + + // Search for documents where first_name is "Alice" AND last_name is "Williams" + firstNameQuery = query.NewMatchQuery("Alice") + firstNameQuery.SetField("groups.first_name") + + lastNameQuery = query.NewMatchQuery("Williams") + lastNameQuery.SetField("groups.last_name") + + conjQuery = query.NewConjunctionQuery([]query.Query{firstNameQuery, lastNameQuery}) + + searchReq = NewSearchRequest(conjQuery) + searchReq.SortBy([]string{"_id"}) + + res, err = idx.Search(searchReq) + if err != nil { + t.Fatalf("search failed: %v", err) + } + + if len(res.Hits) != 0 { + t.Fatalf("expected 0 hits, got %d", len(res.Hits)) + } + + // Search for documents where first_name is "Diana" AND last_name is "Brown" + firstNameQuery = query.NewMatchQuery("Diana") + firstNameQuery.SetField("groups.first_name") + + lastNameQuery = query.NewMatchQuery("Brown") + lastNameQuery.SetField("groups.last_name") + + conjQuery = query.NewConjunctionQuery([]query.Query{firstNameQuery, lastNameQuery}) + + searchReq = NewSearchRequest(conjQuery) + searchReq.SortBy([]string{"_id"}) + + res, err = idx.Search(searchReq) + if err != nil { + t.Fatalf("search failed: %v", err) + } + + if len(res.Hits) != 2 { + t.Fatalf("expected 2 hits, got %d", len(res.Hits)) + } + + if res.Hits[0].ID != "1" || res.Hits[1].ID != "2" { + t.Fatalf("unexpected hit IDs: %v, %v", res.Hits[0].ID, res.Hits[1].ID) + } +} + +func TestValidNestedMapping(t *testing.T) { + // ensure that top-level mappings - DefaultMapping and any type mappings - cannot be nested mappings + imap := mapping.NewIndexMapping() + nestedMapping := mapping.NewNestedDocumentMapping() + imap.DefaultMapping = nestedMapping + err := imap.Validate() + if err == nil { + t.Fatalf("expected error for nested DefaultMapping, got nil") + } + // invalid nested type mapping + imap = mapping.NewIndexMapping() + imap.AddDocumentMapping("type1", nestedMapping) + err = imap.Validate() + if err == nil { + t.Fatalf("expected error for nested type mapping, got nil") + } + // valid nested mappings within DefaultMapping + imap = mapping.NewIndexMapping() + docMapping := mapping.NewDocumentMapping() + nestedMapping = mapping.NewNestedDocumentMapping() + fieldMapping := mapping.NewTextFieldMapping() + nestedMapping.AddFieldMappingsAt("field1", fieldMapping) + docMapping.AddSubDocumentMapping("nestedField", nestedMapping) + imap.DefaultMapping = docMapping + err = imap.Validate() + if err != nil { + t.Fatalf("expected valid nested mapping, got error: %v", err) + } + // valid nested mappings within type mapping + imap = mapping.NewIndexMapping() + docMapping = mapping.NewDocumentMapping() + nestedMapping = mapping.NewNestedDocumentMapping() + fieldMapping = mapping.NewTextFieldMapping() + nestedMapping.AddFieldMappingsAt("field1", fieldMapping) + docMapping.AddSubDocumentMapping("nestedField", nestedMapping) + imap.AddDocumentMapping("type1", docMapping) + err = imap.Validate() + if err != nil { + t.Fatalf("expected valid nested mapping, got error: %v", err) + } + // some nested type mappings + imap = mapping.NewIndexMapping() + nestedMapping = mapping.NewNestedDocumentMapping() + regularMapping := mapping.NewDocumentMapping() + imap.AddDocumentMapping("non_nested1", regularMapping) + imap.AddDocumentMapping("non_nested2", regularMapping) + imap.AddDocumentMapping("nested1", nestedMapping) + imap.AddDocumentMapping("nested2", nestedMapping) + err = imap.Validate() + if err == nil { + t.Fatalf("expected error for nested type mappings, got nil") + } +} diff --git a/search_no_knn.go b/search_no_knn.go new file mode 100644 index 0000000..f294a47 --- /dev/null +++ b/search_no_knn.go @@ -0,0 +1,240 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !vectors +// +build !vectors + +package bleve + +import ( + "context" + "encoding/json" + "sort" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/collector" + "github.com/blevesearch/bleve/v2/search/query" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" +) + +const supportForVectorSearch = false + +// A SearchRequest describes all the parameters +// needed to search the index. +// Query is required. +// Size/From describe how much and which part of the +// result set to return. +// Highlight describes optional search result +// highlighting. +// Fields describes a list of field values which +// should be retrieved for result documents, provided they +// were stored while indexing. +// Facets describe the set of facets to be computed. +// Explain triggers inclusion of additional search +// result score explanations. +// Sort describes the desired order for the results to be returned. +// Score controls the kind of scoring performed +// SearchAfter supports deep paging by providing a minimum sort key +// SearchBefore supports deep paging by providing a maximum sort key +// sortFunc specifies the sort implementation to use for sorting results. +// +// A special field named "*" can be used to return all fields. +type SearchRequest struct { + ClientContextID string `json:"client_context_id,omitempty"` + Query query.Query `json:"query"` + Size int `json:"size"` + From int `json:"from"` + Highlight *HighlightRequest `json:"highlight,omitempty"` + Fields []string `json:"fields,omitempty"` + Facets FacetsRequest `json:"facets,omitempty"` + Explain bool `json:"explain"` + Sort search.SortOrder `json:"sort"` + IncludeLocations bool `json:"includeLocations"` + Score string `json:"score,omitempty"` + SearchAfter []string `json:"search_after,omitempty"` + SearchBefore []string `json:"search_before,omitempty"` + + // PreSearchData will be a map that will be used + // in the second phase of any 2-phase search, to provide additional + // context to the second phase. This is useful in the case of index + // aliases where the first phase will gather the PreSearchData from all + // the indexes in the alias, and the second phase will use that + // PreSearchData to perform the actual search. + // The currently accepted map configuration is: + // + // "_knn_pre_search_data_key": []*search.DocumentMatch + + PreSearchData map[string]interface{} `json:"pre_search_data,omitempty"` + + Params *RequestParams `json:"params,omitempty"` + + sortFunc func(sort.Interface) +} + +// UnmarshalJSON deserializes a JSON representation of +// a SearchRequest +func (r *SearchRequest) UnmarshalJSON(input []byte) error { + var temp struct { + Q json.RawMessage `json:"query"` + Size *int `json:"size"` + From int `json:"from"` + Highlight *HighlightRequest `json:"highlight"` + Fields []string `json:"fields"` + Facets FacetsRequest `json:"facets"` + Explain bool `json:"explain"` + Sort []json.RawMessage `json:"sort"` + IncludeLocations bool `json:"includeLocations"` + Score string `json:"score"` + SearchAfter []string `json:"search_after"` + SearchBefore []string `json:"search_before"` + PreSearchData OptionalRawMessage `json:"pre_search_data"` + Params OptionalRawMessage `json:"params"` + } + + err := util.UnmarshalJSON(input, &temp) + if err != nil { + return err + } + + if temp.Size == nil { + r.Size = 10 + } else { + r.Size = *temp.Size + } + if temp.Sort == nil { + r.Sort = search.SortOrder{&search.SortScore{Desc: true}} + } else { + r.Sort, err = search.ParseSortOrderJSON(temp.Sort) + if err != nil { + return err + } + } + r.From = temp.From + r.Explain = temp.Explain + r.Highlight = temp.Highlight + r.Fields = temp.Fields + r.Facets = temp.Facets + r.IncludeLocations = temp.IncludeLocations + r.Score = temp.Score + r.SearchAfter = temp.SearchAfter + r.SearchBefore = temp.SearchBefore + r.Query, err = query.ParseQuery(temp.Q) + if err != nil { + return err + } + + if r.Size < 0 { + r.Size = 10 + } + if r.From < 0 { + r.From = 0 + } + + if IsScoreFusionRequested(r) { + if temp.Params == nil { + // If params is not present and it is requires rescoring, assign + // default values + r.Params = NewDefaultParams(r.From, r.Size) + } else { + // if it is a request that requires rescoring, parse the rescoring + // parameters. + params, err := ParseParams(r, temp.Params) + if err != nil { + return err + } + r.Params = params + } + } + + if temp.PreSearchData != nil { + r.PreSearchData, err = query.ParsePreSearchData(temp.PreSearchData) + if err != nil { + return err + } + } + + return nil + +} + +// ----------------------------------------------------------------------------- + +func copySearchRequest(req *SearchRequest, preSearchData map[string]interface{}) *SearchRequest { + rv := SearchRequest{ + Query: req.Query, + Size: req.Size + req.From, + From: 0, + Highlight: req.Highlight, + Fields: req.Fields, + Facets: req.Facets, + Explain: req.Explain, + Sort: req.Sort.Copy(), + IncludeLocations: req.IncludeLocations, + Score: req.Score, + SearchAfter: req.SearchAfter, + SearchBefore: req.SearchBefore, + PreSearchData: preSearchData, + } + return &rv +} + +func validateKNN(req *SearchRequest) error { + return nil +} + +func (i *indexImpl) runKnnCollector(ctx context.Context, req *SearchRequest, reader index.IndexReader, preSearch bool) ([]*search.DocumentMatch, error) { + return nil, nil +} + +func setKnnHitsInCollector(knnHits []*search.DocumentMatch, coll *collector.TopNCollector) { +} + +func requestHasKNN(req *SearchRequest) bool { + return false +} + +func numKNNQueries(req *SearchRequest) int { + return 0 +} + +func addKnnToDummyRequest(dummyReq *SearchRequest, realReq *SearchRequest) { +} + +func validateAndDistributeKNNHits(knnHits []*search.DocumentMatch, indexes []Index) (map[string][]*search.DocumentMatch, error) { + return nil, nil +} + +func isKNNrequestSatisfiedByPreSearch(req *SearchRequest) bool { + return false +} + +func constructKnnPreSearchData(mergedOut map[string]map[string]interface{}, preSearchResult *SearchResult, + indexes []Index) (map[string]map[string]interface{}, error) { + return mergedOut, nil +} + +func finalizeKNNResults(req *SearchRequest, knnHits []*search.DocumentMatch) []*search.DocumentMatch { + return knnHits +} + +func newKnnPreSearchResultProcessor(req *SearchRequest) *knnPreSearchResultProcessor { + return &knnPreSearchResultProcessor{} // equivalent to nil +} + +func (r *rescorer) prepareKnnRequest() { +} + +func (r *rescorer) restoreKnnRequest() { +} diff --git a/size/sizes.go b/size/sizes.go new file mode 100644 index 0000000..0990bf8 --- /dev/null +++ b/size/sizes.go @@ -0,0 +1,59 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package size + +import ( + "reflect" +) + +func init() { + var b bool + SizeOfBool = int(reflect.TypeOf(b).Size()) + var f32 float32 + SizeOfFloat32 = int(reflect.TypeOf(f32).Size()) + var f64 float64 + SizeOfFloat64 = int(reflect.TypeOf(f64).Size()) + var i int + SizeOfInt = int(reflect.TypeOf(i).Size()) + var m map[int]int + SizeOfMap = int(reflect.TypeOf(m).Size()) + var ptr *int + SizeOfPtr = int(reflect.TypeOf(ptr).Size()) + var slice []int + SizeOfSlice = int(reflect.TypeOf(slice).Size()) + var str string + SizeOfString = int(reflect.TypeOf(str).Size()) + var u8 uint8 + SizeOfUint8 = int(reflect.TypeOf(u8).Size()) + var u16 uint16 + SizeOfUint16 = int(reflect.TypeOf(u16).Size()) + var u32 uint32 + SizeOfUint32 = int(reflect.TypeOf(u32).Size()) + var u64 uint64 + SizeOfUint64 = int(reflect.TypeOf(u64).Size()) +} + +var SizeOfBool int +var SizeOfFloat32 int +var SizeOfFloat64 int +var SizeOfInt int +var SizeOfMap int +var SizeOfPtr int +var SizeOfSlice int +var SizeOfString int +var SizeOfUint8 int +var SizeOfUint16 int +var SizeOfUint32 int +var SizeOfUint64 int diff --git a/term_limit_integration_test.go b/term_limit_integration_test.go new file mode 100644 index 0000000..0e0ad2c --- /dev/null +++ b/term_limit_integration_test.go @@ -0,0 +1,146 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/query" +) + +// disjunctionOfTerms builds a disjunction of n distinct term queries over the +// "name" field. Each term query becomes one leaf term searcher at construction +// time, regardless of whether it matches any document. +func disjunctionOfTerms(prefix string, n int) query.Query { + dq := NewDisjunctionQuery() + for i := 0; i < n; i++ { + tq := NewTermQuery(fmt.Sprintf("%s%d", prefix, i)) + tq.SetField("name") + dq.AddQuery(tq) + } + return dq +} + +// conjunctionOfDisjunctions builds a conjunction of `groups` disjunctions, each +// containing `per` distinct term queries -> groups*per leaf term searchers, but +// no single fan-out node has more than `per` children. +func conjunctionOfDisjunctions(groups, per int) query.Query { + cq := NewConjunctionQuery() + for g := 0; g < groups; g++ { + cq.AddQuery(disjunctionOfTerms(fmt.Sprintf("g%d_", g), per)) + } + return cq +} + +// TestBleveMaxTermSearchers exercises the full bleveMaxTerms wiring end-to-end +// through SearchInContext: the per-search counter is installed from the ctx +// limit, every leaf term searcher is counted, and construction fails once the +// running total exceeds the limit. This complements TestRecordTermSearcher +// (which only covers the counter primitive) by validating that the counter is +// actually installed and consulted along the real search path. +func TestBleveMaxTermSearchers(t *testing.T) { + idx, err := NewMemOnly(NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + defer func() { + if cerr := idx.Close(); cerr != nil { + t.Fatal(cerr) + } + }() + + // A couple of real documents so the term searchers open real readers. + if err := idx.Index("doc0", map[string]interface{}{"name": "alpha beta gamma"}); err != nil { + t.Fatal(err) + } + if err := idx.Index("doc1", map[string]interface{}{"name": "delta epsilon zeta"}); err != nil { + t.Fatal(err) + } + + runSearch := func(q query.Query, limit int) error { + ctx := context.WithValue(context.Background(), search.MaxTermSearchersKey, limit) + _, serr := idx.SearchInContext(ctx, NewSearchRequest(q)) + return serr + } + + t.Run("disabled when limit unset (0)", func(t *testing.T) { + if err := runSearch(disjunctionOfTerms("t", 6), 0); err != nil { + t.Fatalf("limit 0 should disable the check, got: %v", err) + } + }) + + t.Run("disabled when limit is negative", func(t *testing.T) { + // A non-positive limit (as produced upstream when a request explicitly + // disables bleveMaxTerms) must be treated as "no limit", not enforced. + if err := runSearch(disjunctionOfTerms("t", 6), -1); err != nil { + t.Fatalf("negative limit should disable the check, got: %v", err) + } + }) + + t.Run("under the limit succeeds", func(t *testing.T) { + // 6 leaf term searchers, limit 100. + if err := runSearch(disjunctionOfTerms("t", 6), 100); err != nil { + t.Fatalf("6 term searchers under limit 100 should succeed, got: %v", err) + } + }) + + t.Run("exactly at the limit succeeds", func(t *testing.T) { + // 6 leaf term searchers, limit 6: 6 is not > 6, so it is allowed. + if err := runSearch(disjunctionOfTerms("t", 6), 6); err != nil { + t.Fatalf("6 term searchers at limit 6 should succeed, got: %v", err) + } + }) + + t.Run("over the limit fails with TooManyTermSearchers", func(t *testing.T) { + // 6 leaf term searchers, limit 5 -> trips at the 6th. + err := runSearch(disjunctionOfTerms("t", 6), 5) + if err == nil { + t.Fatal("6 term searchers over limit 5 should fail, got nil") + } + if !strings.Contains(err.Error(), "TooManyTermSearchers") { + t.Fatalf("expected a TooManyTermSearchers error, got: %v", err) + } + if !strings.Contains(err.Error(), "set to 5") { + t.Fatalf("error should report the effective limit (5), got: %v", err) + } + }) + + t.Run("counts globally across the whole query tree", func(t *testing.T) { + // 2 disjunctions * 4 terms = 8 leaf term searchers, but no single node + // has more than 4 children. With DisjunctionMaxClauseCount unset this + // can only be tripped by the global bleveMaxTerms count, proving the two + // checks are distinct: bleveMaxClauseCount is per-node, bleveMaxTerms is + // the sum across the tree. + err := runSearch(conjunctionOfDisjunctions(2, 4), 5) + if err == nil { + t.Fatal("8 term searchers over limit 5 should fail, got nil") + } + if !strings.Contains(err.Error(), "TooManyTermSearchers") { + t.Fatalf("expected a TooManyTermSearchers error, got: %v", err) + } + if strings.Contains(err.Error(), "TooManyClauses") { + t.Fatalf("should be the global term-count check, not the per-node clause check: %v", err) + } + + // Same query, limit 8 -> allowed (8 is not > 8). + if err := runSearch(conjunctionOfDisjunctions(2, 4), 8); err != nil { + t.Fatalf("8 term searchers at limit 8 should succeed, got: %v", err) + } + }) +} diff --git a/test/integration.go b/test/integration.go new file mode 100644 index 0000000..c45286b --- /dev/null +++ b/test/integration.go @@ -0,0 +1,27 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "github.com/blevesearch/bleve/v2" +) + +type SearchTest struct { + Search *bleve.SearchRequest `json:"search"` + Result *bleve.SearchResult `json:"result"` + Comment string `json:"comment"` +} + +type SearchTests []*SearchTest diff --git a/test/integration_test.go b/test/integration_test.go new file mode 100644 index 0000000..f2605b2 --- /dev/null +++ b/test/integration_test.go @@ -0,0 +1,279 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "encoding/json" + "flag" + "fmt" + "math" + "os" + "path/filepath" + "reflect" + "regexp" + "testing" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/mapping" + + // allow choosing alternate kvstores + _ "github.com/blevesearch/bleve/v2/config" +) + +var dataset = flag.String("dataset", "", "only test datasets matching this regex") +var onlynum = flag.Int("testnum", -1, "only run the test with this number") +var keepIndex = flag.Bool("keepIndex", false, "keep the index after testing") + +var indexType = flag.String("indexType", bleve.Config.DefaultIndexType, "index type to build") +var kvType = flag.String("kvType", bleve.Config.DefaultKVStore, "kv store type to build") +var segType = flag.String("segType", "", "force scorch segment type") +var segVer = flag.Int("segVer", 0, "force scorch segment version") + +func TestIntegration(t *testing.T) { + + flag.Parse() + + t.Logf("using index type %s and kv type %s", *indexType, *kvType) + if *segType != "" { + t.Logf("forcing segment type: %s", *segType) + } + if *segVer != 0 { + t.Logf("forcing segment version: %d", *segVer) + } + + var err error + var datasetRegexp *regexp.Regexp + if *dataset != "" { + datasetRegexp, err = regexp.Compile(*dataset) + if err != nil { + t.Fatal(err) + } + } + + entries, err := os.ReadDir("tests") + if err != nil { + t.Fatal(err) + } + for _, f := range entries { + if datasetRegexp != nil { + if !datasetRegexp.MatchString(f.Name()) { + continue + } + } + if f.IsDir() { + t.Logf("Running test: %s", f.Name()) + runTestDir(t, "tests"+string(filepath.Separator)+f.Name(), f.Name()) + } + } +} + +func runTestDir(t *testing.T, dir, datasetName string) { + // read the mapping + mappingBytes, err := os.ReadFile(dir + string(filepath.Separator) + "mapping.json") + if err != nil { + t.Errorf("error reading mapping: %v", err) + return + } + var mapping mapping.IndexMappingImpl + err = json.Unmarshal(mappingBytes, &mapping) + if err != nil { + t.Errorf("error unmarshalling mapping: %v", err) + return + } + + var index bleve.Index + var cleanup func() + + // if there is a dir named 'data' open single index + _, err = os.Stat(dir + string(filepath.Separator) + "data") + if !os.IsNotExist(err) { + + index, cleanup, err = loadDataSet(t, datasetName, mapping, dir+string(filepath.Separator)+"data") + if err != nil { + t.Errorf("error loading dataset: %v", err) + return + } + defer cleanup() + } else { + // if there is a dir named 'datasets' build alias over each index + _, err = os.Stat(dir + string(filepath.Separator) + "datasets") + if !os.IsNotExist(err) { + index, cleanup, err = loadDataSets(t, datasetName, mapping, dir+string(filepath.Separator)+"datasets") + if err != nil { + t.Errorf("error loading dataset: %v", err) + return + } + defer cleanup() + } + } + + // read the searches + searchBytes, err := os.ReadFile(dir + string(filepath.Separator) + "searches.json") + if err != nil { + t.Errorf("error reading searches: %v", err) + return + } + var searches SearchTests + err = json.Unmarshal(searchBytes, &searches) + if err != nil { + t.Errorf("error unmarshalling searches: %v", err) + return + } + + // run the searches + for testNum, search := range searches { + if *onlynum < 0 || (*onlynum > 0 && testNum == *onlynum) { + res, err := index.Search(search.Search) + if err != nil { + t.Errorf("error running search: %v", err) + } + if res.Total != search.Result.Total { + t.Errorf("test error - %s", search.Comment) + t.Errorf("test %d - expected total: %d got %d", testNum, search.Result.Total, res.Total) + continue + } + if len(res.Hits) != len(search.Result.Hits) { + t.Errorf("test error - %s", search.Comment) + t.Errorf("test %d - expected hits len: %d got %d", testNum, len(search.Result.Hits), len(res.Hits)) + t.Errorf("got hits: %v", res.Hits) + continue + } + for hi, hit := range search.Result.Hits { + if hit.ID != res.Hits[hi].ID { + t.Errorf("test error - %s", search.Comment) + t.Errorf("test %d - expected hit %d to have ID %s got %s", testNum, hi, hit.ID, res.Hits[hi].ID) + } + if hit.Fields != nil { + if !reflect.DeepEqual(hit.Fields, res.Hits[hi].Fields) { + t.Errorf("test error - %s", search.Comment) + t.Errorf("test %d - expected hit %d to have fields %#v got %#v", testNum, hi, hit.Fields, res.Hits[hi].Fields) + } + } + if hit.Fragments != nil { + if !reflect.DeepEqual(hit.Fragments, res.Hits[hi].Fragments) { + t.Errorf("test error - %s", search.Comment) + t.Errorf("test %d - expected hit %d to have fragments %#v got %#v", testNum, hi, hit.Fragments, res.Hits[hi].Fragments) + } + } + if hit.Locations != nil { + if !reflect.DeepEqual(hit.Locations, res.Hits[hi].Locations) { + t.Errorf("test error - %s", search.Comment) + t.Errorf("test %d - expected hit %d to have locations %#v got %#v", testNum, hi, hit.Locations, res.Hits[hi].Locations) + } + } + // assert that none of the scores were NaN,+Inf,-Inf + if math.IsInf(res.Hits[hi].Score, 0) || math.IsNaN(res.Hits[hi].Score) { + t.Errorf("test error - %s", search.Comment) + t.Errorf("test %d - invalid score %f", testNum, res.Hits[hi].Score) + } + } + if search.Result.Facets != nil { + if !reflect.DeepEqual(search.Result.Facets, res.Facets) { + t.Errorf("test error - %s", search.Comment) + t.Errorf("test %d - expected facets: %#v got %#v", testNum, search.Result.Facets, res.Facets) + } + } + if _, ok := index.(bleve.IndexAlias); !ok { + // check that custom index name is in results + for _, hit := range res.Hits { + if hit.Index != datasetName { + t.Fatalf("expected name: %s, got: %s", datasetName, hit.Index) + } + } + } + } + } +} + +func loadDataSet(t *testing.T, datasetName string, mapping mapping.IndexMappingImpl, path string) (bleve.Index, func(), error) { + idxPath := fmt.Sprintf("test-%s.bleve", datasetName) + cfg := map[string]interface{}{} + if *segType != "" { + cfg["forceSegmentType"] = *segType + } + if *segVer != 0 { + cfg["forceSegmentVersion"] = *segVer + } + + index, err := bleve.NewUsing(idxPath, &mapping, *indexType, *kvType, cfg) + if err != nil { + return nil, nil, fmt.Errorf("error creating new index: %v", err) + } + // set a custom index name + index.SetName(datasetName) + + // index data + entries, err := os.ReadDir(path) + if err != nil { + return nil, nil, fmt.Errorf("error reading data dir: %v", err) + } + for _, f := range entries { + fileBytes, err := os.ReadFile(path + string(filepath.Separator) + f.Name()) + if err != nil { + return nil, nil, fmt.Errorf("error reading data file: %v", err) + } + var fileDoc interface{} + err = json.Unmarshal(fileBytes, &fileDoc) + if err != nil { + return nil, nil, fmt.Errorf("error parsing data file as json: %v", err) + } + filename := f.Name() + ext := filepath.Ext(filename) + id := filename[0 : len(filename)-len(ext)] + err = index.Index(id, fileDoc) + if err != nil { + return nil, nil, fmt.Errorf("error indexing data: %v", err) + } + } + cleanup := func() { + err := index.Close() + if err != nil { + t.Fatalf("error closing index: %v", err) + } + if !*keepIndex { + err := os.RemoveAll(idxPath) + if err != nil { + t.Fatalf("error removing index: %v", err) + } + } + } + return index, cleanup, nil +} + +func loadDataSets(t *testing.T, datasetName string, mapping mapping.IndexMappingImpl, path string) (bleve.Index, func(), error) { + entries, err := os.ReadDir(path) + if err != nil { + return nil, nil, fmt.Errorf("error reading datasets dir: %v", err) + } + var cleanups []func() + alias := bleve.NewIndexAlias() + for _, f := range entries { + idx, idxCleanup, err := loadDataSet(t, f.Name(), mapping, path+string(filepath.Separator)+f.Name()) + if err != nil { + return nil, nil, fmt.Errorf("error loading dataset: %v", err) + } + cleanups = append(cleanups, idxCleanup) + alias.Add(idx) + } + alias.SetName(datasetName) + + cleanupAll := func() { + for _, cleanup := range cleanups { + cleanup() + } + } + + return alias, cleanupAll, nil +} diff --git a/test/ip_field_test.go b/test/ip_field_test.go new file mode 100644 index 0000000..04f9762 --- /dev/null +++ b/test/ip_field_test.go @@ -0,0 +1,272 @@ +// Copyright (c) 2021 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "net" + "testing" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/mapping" +) + +type doc struct { + IP string `json:"ip"` +} + +func createIdx(t *testing.T) bleve.Index { + ipIndexed := mapping.NewIPFieldMapping() + ipIndexed.Name = "ip" + + lineMapping := bleve.NewDocumentStaticMapping() + lineMapping.AddFieldMappingsAt("ip", ipIndexed) + + mapping := bleve.NewIndexMapping() + mapping.DefaultMapping = lineMapping + mapping.DefaultAnalyzer = "standard" + + idx, err := bleve.NewMemOnly(mapping) + if err != nil { + t.Fatal(err) + } + return idx +} + +func Test_ipv4CidrQuery(t *testing.T) { + idx := createIdx(t) + defer idx.Close() + + err := idx.Index("id1", doc{"192.168.1.21"}) + if err != nil { + t.Fatal(err) + } + + reqStr := `192.168.1.0/24` + query := bleve.NewIPRangeQuery(reqStr) + query.FieldVal = "ip" + + search := bleve.NewSearchRequest(query) + res, err := idx.Search(search) + if err != nil { + t.Fatal(err) + } + + if res.Total != 1 { + t.Fatalf("failed to find %q, res -> %s", reqStr, res) + } + if res.Hits[0].ID != "id1" { + t.Fatalf("expected %q got %q", "id1", res.Hits[0].Index) + } +} + +func Test_ipv6CidrQuery(t *testing.T) { + idx := createIdx(t) + defer idx.Close() + + err := idx.Index("id1", doc{"2a00:23c8:7283:ff00:1fa8:2af6:9dec:6b19"}) + if err != nil { + t.Fatal(err) + } + + reqStr := `2a00:23c8:7283:ff00:1fa8:0:0:0/80` + query := bleve.NewIPRangeQuery(reqStr) + query.FieldVal = "ip" + + search := bleve.NewSearchRequest(query) + res, err := idx.Search(search) + if err != nil { + t.Fatal(err) + } + + if res.Total != 1 { + t.Fatalf("failed to find %q, res -> %s", reqStr, res) + } + if res.Hits[0].ID != "id1" { + t.Fatalf("expected %q got %q", "id1", res.Hits[0].Index) + } +} + +func Test_MultiIPvr4CidrQuery(t *testing.T) { + idx := createIdx(t) + defer idx.Close() + + err := idx.Index("id1", doc{"192.168.1.0"}) + if err != nil { + t.Fatal(err) + } + err = idx.Index("id2", doc{"192.168.1.255"}) + if err != nil { + t.Fatal(err) + } + err = idx.Index("id3", doc{"192.168.2.22"}) + if err != nil { + t.Fatal(err) + } + + reqStr := `192.168.1.0/24` + query := bleve.NewIPRangeQuery(reqStr) + query.FieldVal = "ip" + + search := bleve.NewSearchRequest(query) + res, err := idx.Search(search) + if err != nil { + t.Fatal(err) + } + + if res.Total != 2 { + t.Fatalf("failed to find %q, res -> %s", reqStr, res) + } + if res.Hits[0].ID != "id1" { + t.Fatalf("expected %q got %q", "id1", res.Hits[0].ID) + } + if res.Hits[1].ID != "id2" { + t.Fatalf("expected %q got %q", "id2", res.Hits[0].Index) + } +} + +func Test_CidrQueryNonDivisibleBy8(t *testing.T) { + idx := createIdx(t) + defer idx.Close() + + err := idx.Index("id1", doc{"192.168.1.1"}) + if err != nil { + t.Fatal(err) + } + err = idx.Index("id2", doc{"192.168.1.2"}) + if err != nil { + t.Fatal(err) + } + err = idx.Index("id3", doc{"192.168.2.5"}) + if err != nil { + t.Fatal(err) + } + err = idx.Index("id4", doc{"192.168.2.6"}) + if err != nil { + t.Fatal(err) + } + + reqStr := `192.168.1.0/30` + query := bleve.NewIPRangeQuery(reqStr) + query.FieldVal = "ip" + + search := bleve.NewSearchRequest(query) + res, err := idx.Search(search) + if err != nil { + t.Fatal(err) + } + + if res.Total != 2 { + t.Fatalf("failed to find %q, res -> %s", reqStr, res) + } + if res.Hits[0].ID != "id1" { + t.Fatalf("expected %q got %q", "id1", res.Hits[0].ID) + } + if res.Hits[1].ID != "id2" { + t.Fatalf("expected %q got %q", "id2", res.Hits[0].Index) + } +} + +func Test_simpleIPv4MatchQuery(t *testing.T) { + idx := createIdx(t) + defer idx.Close() + + err := idx.Index("id1", doc{"192.168.1.21"}) + if err != nil { + t.Fatal(err) + } + + reqStr := `192.168.1.21` + query := bleve.NewIPRangeQuery(reqStr) + query.FieldVal = "ip" + + search := bleve.NewSearchRequest(query) + res, err := idx.Search(search) + if err != nil { + t.Fatal(err) + } + + if res.Total != 1 { + t.Fatalf("failed to find %q, res -> %s", reqStr, res) + } + if res.Hits[0].ID != "id1" { + t.Fatalf("expected %q got %q", "id1", res.Hits[0].Index) + } +} + +func Test_ipv4LiteralData(t *testing.T) { + idx := createIdx(t) + defer idx.Close() + + type stronglyTyped struct { + IP net.IP `json:"ip"` + } + + err := idx.Index("id1", stronglyTyped{net.ParseIP("192.168.1.21")}) + if err != nil { + t.Fatal(err) + } + + reqStr := `192.168.1.0/24` + query := bleve.NewIPRangeQuery(reqStr) + query.FieldVal = "ip" + + search := bleve.NewSearchRequest(query) + res, err := idx.Search(search) + if err != nil { + t.Fatal(err) + } + + if res.Total != 1 { + t.Fatalf("failed to find %q, res -> %s", reqStr, res) + } + if res.Hits[0].ID != "id1" { + t.Fatalf("expected %q got %q", "id1", res.Hits[0].Index) + } +} + +func Test_badIPFmt(t *testing.T) { + idx := createIdx(t) + defer idx.Close() + + reqStr := `192.168.1.` + query := bleve.NewIPRangeQuery(reqStr) + query.FieldVal = "ip" + + search := bleve.NewSearchRequest(query) + _, err := idx.Search(search) + if err == nil { + t.Errorf("%q is not a valid IP", reqStr) + } +} + +func Test_badCIDRFmt(t *testing.T) { + idx := createIdx(t) + defer idx.Close() + + reqStr := `/` + query := bleve.NewIPRangeQuery(reqStr) + query.FieldVal = "ip" + + err := query.Validate() + if err == nil { + t.Errorf("%q is not a valid CIDR", reqStr) + } + + search := bleve.NewSearchRequest(query) + _, err = idx.Search(search) + if err == nil { + t.Errorf("%q is not a valid CIDR", reqStr) + } +} diff --git a/test/tests/alias/datasets/shard0/a.json b/test/tests/alias/datasets/shard0/a.json new file mode 100644 index 0000000..1aeaf2c --- /dev/null +++ b/test/tests/alias/datasets/shard0/a.json @@ -0,0 +1,3 @@ +{ + "name": "a" +} \ No newline at end of file diff --git a/test/tests/alias/datasets/shard0/c.json b/test/tests/alias/datasets/shard0/c.json new file mode 100644 index 0000000..a584a54 --- /dev/null +++ b/test/tests/alias/datasets/shard0/c.json @@ -0,0 +1,3 @@ +{ + "name": "c" +} \ No newline at end of file diff --git a/test/tests/alias/datasets/shard1/b.json b/test/tests/alias/datasets/shard1/b.json new file mode 100644 index 0000000..1c67e0d --- /dev/null +++ b/test/tests/alias/datasets/shard1/b.json @@ -0,0 +1,3 @@ +{ + "name": "b" +} \ No newline at end of file diff --git a/test/tests/alias/datasets/shard1/d.json b/test/tests/alias/datasets/shard1/d.json new file mode 100644 index 0000000..0a478db --- /dev/null +++ b/test/tests/alias/datasets/shard1/d.json @@ -0,0 +1,3 @@ +{ + "name": "d" +} \ No newline at end of file diff --git a/test/tests/alias/mapping.json b/test/tests/alias/mapping.json new file mode 100644 index 0000000..7f9bae7 --- /dev/null +++ b/test/tests/alias/mapping.json @@ -0,0 +1,3 @@ +{ + "default_analyzer": "keyword" +} diff --git a/test/tests/alias/searches.json b/test/tests/alias/searches.json new file mode 100644 index 0000000..fb8dd06 --- /dev/null +++ b/test/tests/alias/searches.json @@ -0,0 +1,76 @@ +[ + { + "comment": "match all across shards", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "match_all": {} + } + }, + "result": { + "total_hits": 4, + "hits": [ + { + "id": "a" + }, + { + "id": "b" + }, + { + "id": "c" + }, + { + "id": "d" + } + ] + } + }, + { + "comment": "search after b (page 2 when size=2)", + "search": { + "from": 0, + "size": 2, + "sort": ["name"], + "search_after": ["b"], + "query": { + "match_all": {} + } + }, + "result": { + "total_hits": 4, + "hits": [ + { + "id": "c" + }, + { + "id": "d" + } + ] + } + }, + { + "comment": "search before c (page 1 when size=2)", + "search": { + "from": 0, + "size": 2, + "sort": ["name"], + "search_before": ["c"], + "query": { + "match_all": {} + } + }, + "result": { + "total_hits": 4, + "hits": [ + { + "id": "a" + }, + { + "id": "b" + } + ] + } + } +] \ No newline at end of file diff --git a/test/tests/basic/data/a.json b/test/tests/basic/data/a.json new file mode 100644 index 0000000..992bbd9 --- /dev/null +++ b/test/tests/basic/data/a.json @@ -0,0 +1,7 @@ +{ + "id": "a", + "name": "marty", + "age": 19, + "title": "mista", + "tags": ["gopher", "belieber"] +} \ No newline at end of file diff --git a/test/tests/basic/data/b.json b/test/tests/basic/data/b.json new file mode 100644 index 0000000..0697272 --- /dev/null +++ b/test/tests/basic/data/b.json @@ -0,0 +1,7 @@ +{ + "id": "b", + "name": "steve has long & complicated name", + "age": 27, + "birthday": "2001-09-09T01:46:40Z", + "title": "missess" +} \ No newline at end of file diff --git a/test/tests/basic/data/c.json b/test/tests/basic/data/c.json new file mode 100644 index 0000000..2b5d308 --- /dev/null +++ b/test/tests/basic/data/c.json @@ -0,0 +1,7 @@ +{ + "id": "c", + "name": "bob walks home", + "age": 64, + "birthday": "2014-05-13T16:53:20Z", + "title": "masta" +} \ No newline at end of file diff --git a/test/tests/basic/data/d.json b/test/tests/basic/data/d.json new file mode 100644 index 0000000..dbc2f51 --- /dev/null +++ b/test/tests/basic/data/d.json @@ -0,0 +1,7 @@ +{ + "id": "d", + "name": "bobbleheaded wings top the phone", + "age": 72, + "birthday": "2014-05-13T16:53:20Z", + "title": "mizz" +} \ No newline at end of file diff --git a/test/tests/basic/mapping.json b/test/tests/basic/mapping.json new file mode 100644 index 0000000..a7a049b --- /dev/null +++ b/test/tests/basic/mapping.json @@ -0,0 +1,27 @@ +{ + "types": { + "person": { + "properties": { + "name": { + "fields": [ + { + "include_term_vectors": true, + "include_in_all": true, + "index": true, + "store": true, + "analyzer": "en", + "type": "text" + } + ], + "dynamic": true, + "enabled": true + }, + "id": { + "dynamic": false, + "enabled": false + } + } + } + }, + "default_type": "person" +} \ No newline at end of file diff --git a/test/tests/basic/searches.json b/test/tests/basic/searches.json new file mode 100644 index 0000000..7ddfce3 --- /dev/null +++ b/test/tests/basic/searches.json @@ -0,0 +1,883 @@ +[ + { + "comment": "test term search, exact match", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "name", + "term": "marti" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "comment": "test term search, no match", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "name", + "term": "noone" + } + }, + "result": { + "total_hits": 0, + "hits": [] + } + }, + { + "comment": "test match phrase search", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "match_phrase": "steve has" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "b" + } + ] + } + }, + { + "comment": "test term search, no match", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "name", + "term": "walking" + } + }, + "result": { + "total_hits": 0, + "hits": [] + } + }, + { + "comment": "test match search, matching due to analysis", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "fuzziness": 0, + "prefix_length": 0, + "field": "name", + "match": "walking" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "c" + } + ] + } + }, + { + "comment": "test term prefix search", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "name", + "prefix": "bobble" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "d" + } + ] + } + }, + { + "comment": "test simple query string", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "query": "+name:phone" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "d" + } + ] + } + }, + { + "comment": "test numeric range, no lower bound", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "age", + "max": 30 + } + }, + "result": { + "total_hits": 2, + "hits": [ + { + "id": "a" + }, + { + "id": "b" + } + ] + } + }, + { + "comment": "test numeric range, upper and lower bounds", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "age", + "max": 30, + "min": 20 + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "b" + } + ] + } + }, + { + "comment": "test conjunction of numeric range, upper and lower bounds", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "conjuncts": [ + { + "boost": 1, + "field": "age", + "min": 20 + }, + { + "boost": 1, + "field": "age", + "max": 30 + } + ] + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "b" + } + ] + } + }, + { + "comment": "test date range, no upper bound", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "birthday", + "start": "2010-01-01" + } + }, + "result": { + "total_hits": 2, + "hits": [ + { + "id": "c" + }, + { + "id": "d" + } + ] + } + }, + { + "comment": "test numeric range, no lower bound", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "birthday", + "end": "2010-01-01" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "b" + } + ] + } + }, + { + "comment": "test term search, matching inside an array", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "tags", + "term": "gopher" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "comment": "test term search, matching another element inside array", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "tags", + "term": "belieber" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "comment": "test term search, not present in array", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "tags", + "term": "notintagsarray" + } + }, + "result": { + "total_hits": 0, + "hits": [] + } + }, + { + "comment": "with size 0, total should be 1, but hits empty", + "search": { + "from": 0, + "size": 0, + "sort": ["-_score", "_id"], + "query": { + "field": "name", + "term": "marti" + } + }, + "result": { + "total_hits": 1, + "hits": [] + } + }, + { + "comment": "a search for doc a that includes tags field, verifies both values come back", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "fields": ["tags"], + "query": { + "field": "name", + "term": "marti" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a", + "fields": { + "tags": ["gopher", "belieber"] + } + } + ] + } + }, + { + "comment": "test fuzzy search, fuzziness 1 with match", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "name", + "term": "msrti", + "fuzziness": 1 + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "comment": "highlight results", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "name", + "match": "long" + }, + "highlight": { + "fields": ["name"] + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "b", + "fragments": { + "name": ["steve has <a> long & complicated name"] + } + } + ] + } + }, + { + "comment": "highlight results without specifying fields", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "name", + "match": "long" + }, + "highlight": {} + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "b", + "fragments": { + "name": ["steve has <a> long & complicated name"] + } + } + ] + } + }, + { + "comment": "request fields", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "fields": ["age","birthday"], + "query": { + "field": "name", + "match": "long" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "b", + "fields": { + "age": 27, + "birthday": "2001-09-09T01:46:40Z" + } + } + ] + } + }, + { + "comment": "tests query string only containing MUST NOT clause, bug #193", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "query": "-title:mista" + } + }, + "result": { + "total_hits": 3, + "hits": [ + { + "id": "b" + }, + { + "id": "c" + }, + { + "id": "d" + } + ] + } + }, + { + "comment": "highlight results including non-matching field (which should be produced in its entirety, though unhighlighted)", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "name", + "match": "long" + }, + "highlight": { + "fields": ["name", "title"] + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "b", + "fragments": { + "name": ["steve has <a> long & complicated name"], + "title": ["missess"] + } + } + ] + } + }, + { + "comment": "search and highlight an array field", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "tags", + "match": "gopher" + }, + "highlight": { + "fields": ["tags"] + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a", + "fragments": { + "tags": ["gopher"] + } + } + ] + } + }, + { + "comment": "reproduce bug in prefix search", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "title", + "prefix": "miss" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "b" + } + ] + } + }, + { + "comment": "test match none", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "match_none": {} + } + }, + "result": { + "total_hits": 0, + "hits": [] + } + }, + { + "comment": "test match all", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "match_all": {} + } + }, + "result": { + "total_hits": 4, + "hits": [ + { + "id": "a" + }, + { + "id": "b" + }, + { + "id": "c" + }, + { + "id": "d" + } + ] + } + }, + { + "comment": "test doc id query", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "ids": ["b", "c"] + } + }, + "result": { + "total_hits": 2, + "hits": [ + { + "id": "b" + }, + { + "id": "c" + } + ] + } + }, + { + "comment": "test query string MUST and SHOULD", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "query": "+age:>20 missess" + } + }, + "result": { + "total_hits": 3, + "hits": [ + { + "id": "b" + }, + { + "id": "c" + }, + { + "id": "d" + } + ] + } + }, + { + "comment": "test regexp matching term", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "name", + "regexp": "mar.*" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "comment": "test regexp that should not match when properly anchored", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "name", + "regexp": "mar." + } + }, + "result": { + "total_hits": 0, + "hits": [] + } + }, + { + "comment": "test wildcard matching term", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "name", + "wildcard": "mar*" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "comment": "test boost - term query", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "disjuncts": [ + { + "field": "name", + "term": "marti", + "boost": 1.0 + }, + { + "field": "name", + "term": "steve", + "boost": 5.0 + } + ] + } + }, + "result": { + "total_hits": 2, + "hits": [ + { + "id": "b" + }, + { + "id": "a" + } + ] + } + }, + { + "comment": "test boost - term query", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "disjuncts": [ + { + "field": "name", + "term": "marti", + "boost": 1.0 + }, + { + "fuzziness": 1, + "field": "name", + "term": "steve", + "boost": 5.0 + } + ] + } + }, + "result": { + "total_hits": 2, + "hits": [ + { + "id": "b" + }, + { + "id": "a" + } + ] + } + }, + { + "comment": "test boost - numeric range query", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "disjuncts": [ + { + "field": "name", + "term": "marti", + "boost": 1.0 + }, + { + "field": "age", + "min": 25, + "max": 29, + "boost": 50.0 + } + ] + } + }, + "result": { + "total_hits": 2, + "hits": [ + { + "id": "b" + }, + { + "id": "a" + } + ] + } + }, + { + "comment": "test boost - regexp query", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "disjuncts": [ + { + "field": "name", + "term": "marti", + "boost": 1.0 + }, + { + "field": "name", + "regexp": "stev.*", + "boost": 5.0 + } + ] + } + }, + "result": { + "total_hits": 2, + "hits": [ + { + "id": "b" + }, + { + "id": "a" + } + ] + } + }, + { + "comment": "test wildcard inside query string", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "query": "name:mar*" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "comment": "test regexp inside query string", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "query": "name:/mar.*/" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "comment": "test term range", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "title", + "max": "miz", + "min": "mis" + } + }, + "result": { + "total_hits": 2, + "hits": [ + { + "id": "a" + }, + { + "id": "b" + } + ] + } + } +] diff --git a/test/tests/employee/data/emp10508560.json b/test/tests/employee/data/emp10508560.json new file mode 100644 index 0000000..b8f7548 --- /dev/null +++ b/test/tests/employee/data/emp10508560.json @@ -0,0 +1,44 @@ +{ + "salary": 104561.8, + "_type": "emp", + "name": "Deirdre Reed", + "mutated": 0, + "is_manager": true, + "dept": "Accounts", + "join_date": "2003-05-28T21:29:00", + "manages": { + "team_size": 9, + "reports": [ + "Gallia Julián", + "Duvessa Nicolás", + "Beryl Thomas", + "Deirdre Julián", + "Antonia Gerónimo", + "Ciara Young", + "Riona Richardson IX", + "Severin Jr.", + "Perdita Morgan" + ] + }, + "languages_known": [ + "English", + "Spanish", + "German", + "Italian", + "French", + "Arabic", + "Africans", + "Hindi", + "Vietnamese", + "Urdu", + "Dutch", + "Quechua", + "Japanese", + "Chinese", + "Nepalese", + "Thai", + "Malay" + ], + "emp_id": "10508560", + "email": "deirdre@mcdiabetes.com" +} \ No newline at end of file diff --git a/test/tests/employee/mapping.json b/test/tests/employee/mapping.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/test/tests/employee/mapping.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/test/tests/employee/searches.json b/test/tests/employee/searches.json new file mode 100644 index 0000000..d4db280 --- /dev/null +++ b/test/tests/employee/searches.json @@ -0,0 +1,41 @@ +[ + { + "comment": "test array position output", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "manages.reports", + "term": "julián" + }, + "includeLocations": true + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "emp10508560", + "locations": { + "manages.reports": { + "julián": [ + { + "pos": 2, + "start": 7, + "end": 14, + "array_positions":[0] + }, + { + "pos": 2, + "start": 8, + "end": 15, + "array_positions":[3] + } + ] + } + } + } + ] + } + } +] \ No newline at end of file diff --git a/test/tests/facet/data/a.json b/test/tests/facet/data/a.json new file mode 100644 index 0000000..777b21f --- /dev/null +++ b/test/tests/facet/data/a.json @@ -0,0 +1,6 @@ +{ + "category": "inventory", + "type": "book", + "rating": 2, + "updated": "2014-11-25" +} \ No newline at end of file diff --git a/test/tests/facet/data/b.json b/test/tests/facet/data/b.json new file mode 100644 index 0000000..c3c2c69 --- /dev/null +++ b/test/tests/facet/data/b.json @@ -0,0 +1,6 @@ +{ + "category": "inventory", + "type": "book", + "rating": 7, + "updated": "2013-07-25" +} \ No newline at end of file diff --git a/test/tests/facet/data/c.json b/test/tests/facet/data/c.json new file mode 100644 index 0000000..da3542c --- /dev/null +++ b/test/tests/facet/data/c.json @@ -0,0 +1,6 @@ +{ + "category": "inventory", + "type": "book", + "rating": 1, + "updated": "2014-03-03" +} \ No newline at end of file diff --git a/test/tests/facet/data/d.json b/test/tests/facet/data/d.json new file mode 100644 index 0000000..20aaef1 --- /dev/null +++ b/test/tests/facet/data/d.json @@ -0,0 +1,6 @@ +{ + "category": "inventory", + "type": "book", + "rating": 9, + "updated": "2014-09-16" +} \ No newline at end of file diff --git a/test/tests/facet/data/e.json b/test/tests/facet/data/e.json new file mode 100644 index 0000000..8dbd28a --- /dev/null +++ b/test/tests/facet/data/e.json @@ -0,0 +1,6 @@ +{ + "category": "inventory", + "type": "book", + "rating": 5, + "updated": "2014-11-15" +} \ No newline at end of file diff --git a/test/tests/facet/data/f.json b/test/tests/facet/data/f.json new file mode 100644 index 0000000..74e8cd2 --- /dev/null +++ b/test/tests/facet/data/f.json @@ -0,0 +1,6 @@ +{ + "category": "inventory", + "type": "movie", + "rating": 3, + "updated": "2017-06-05" +} \ No newline at end of file diff --git a/test/tests/facet/data/g.json b/test/tests/facet/data/g.json new file mode 100644 index 0000000..ea5f29f --- /dev/null +++ b/test/tests/facet/data/g.json @@ -0,0 +1,6 @@ +{ + "category": "inventory", + "type": "movie", + "rating": 9, + "updated": "2011-10-03" +} \ No newline at end of file diff --git a/test/tests/facet/data/h.json b/test/tests/facet/data/h.json new file mode 100644 index 0000000..f91c0d7 --- /dev/null +++ b/test/tests/facet/data/h.json @@ -0,0 +1,6 @@ +{ + "category": "inventory", + "type": "movie", + "rating": 9, + "updated": "2019-08-26" +} \ No newline at end of file diff --git a/test/tests/facet/data/i.json b/test/tests/facet/data/i.json new file mode 100644 index 0000000..e46e5b7 --- /dev/null +++ b/test/tests/facet/data/i.json @@ -0,0 +1,6 @@ +{ + "category": "inventory", + "type": "movie", + "rating": 1, + "updated": "2014-12-14" +} \ No newline at end of file diff --git a/test/tests/facet/data/j.json b/test/tests/facet/data/j.json new file mode 100644 index 0000000..263f07b --- /dev/null +++ b/test/tests/facet/data/j.json @@ -0,0 +1,6 @@ +{ + "category": "inventory", + "type": "game", + "rating": 9, + "updated": "2013-10-20" +} \ No newline at end of file diff --git a/test/tests/facet/mapping.json b/test/tests/facet/mapping.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/test/tests/facet/mapping.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/test/tests/facet/searches.json b/test/tests/facet/searches.json new file mode 100644 index 0000000..6752282 --- /dev/null +++ b/test/tests/facet/searches.json @@ -0,0 +1,144 @@ +[ + { + "search": { + "from": 0, + "size": 0, + "query": { + "field": "category", + "term": "inventory" + }, + "facets": { + "types": { + "size": 3, + "field": "type" + } + } + }, + "result": { + "total_hits": 10, + "hits": [], + "facets": { + "types": { + "field": "type", + "total": 10, + "missing": 0, + "other": 0, + "terms": [ + { + "term": "book", + "count": 5 + }, + { + "term": "movie", + "count": 4 + }, + { + "term": "game", + "count": 1 + } + ] + } + } + } + }, + { + "search": { + "from": 0, + "size": 0, + "query": { + "field": "category", + "term": "inventory" + }, + "facets": { + "types": { + "size": 3, + "field": "rating", + "numeric_ranges": [ + { + "name": "low", + "max": 5 + }, + { + "name": "high", + "min": 5 + } + ] + } + } + }, + "result": { + "total_hits": 10, + "hits": [], + "facets": { + "types": { + "field": "rating", + "total": 10, + "missing": 0, + "other": 0, + "numeric_ranges": [ + { + "name": "high", + "count": 6, + "min": 5 + }, + { + "name": "low", + "count": 4, + "max": 5 + } + ] + } + } + } + }, + { + "search": { + "from": 0, + "size": 0, + "query": { + "field": "category", + "term": "inventory" + }, + "facets": { + "types": { + "size": 3, + "field": "updated", + "date_ranges": [ + { + "name": "old", + "end": "2012-01-01" + }, + { + "name": "new", + "start": "2012-01-01" + } + ] + } + } + }, + "result": { + "total_hits": 10, + "hits": [], + "facets": { + "types": { + "field": "updated", + "total": 10, + "missing": 0, + "other": 0, + "date_ranges": [ + { + "name": "new", + "count": 9, + "start": "2012-01-01T00:00:00Z" + }, + { + "name": "old", + "count": 1, + "end": "2012-01-01T00:00:00Z" + } + ] + } + } + } + } +] \ No newline at end of file diff --git a/test/tests/fosdem/data/3311@FOSDEM15@fosdem.org.json b/test/tests/fosdem/data/3311@FOSDEM15@fosdem.org.json new file mode 100644 index 0000000..62918db --- /dev/null +++ b/test/tests/fosdem/data/3311@FOSDEM15@fosdem.org.json @@ -0,0 +1,4 @@ +{ + "description": "From Prolog to Erlang to Haskell to Lisp to TLC and then back to Prolog I have journeyed, and I'd like to share some of the beautiful", + "category": "Word" +} \ No newline at end of file diff --git a/test/tests/fosdem/data/3492@FOSDEM15@fosdem.org.json b/test/tests/fosdem/data/3492@FOSDEM15@fosdem.org.json new file mode 100644 index 0000000..874f60e --- /dev/null +++ b/test/tests/fosdem/data/3492@FOSDEM15@fosdem.org.json @@ -0,0 +1,4 @@ +{ + "description": "different cats", + "category": "Perl" +} \ No newline at end of file diff --git a/test/tests/fosdem/data/3496@FOSDEM15@fosdem.org.json b/test/tests/fosdem/data/3496@FOSDEM15@fosdem.org.json new file mode 100644 index 0000000..1acdf34 --- /dev/null +++ b/test/tests/fosdem/data/3496@FOSDEM15@fosdem.org.json @@ -0,0 +1,4 @@ +{ + "description": "many cats", + "category": "Perl" +} \ No newline at end of file diff --git a/test/tests/fosdem/data/3505@FOSDEM15@fosdem.org.json b/test/tests/fosdem/data/3505@FOSDEM15@fosdem.org.json new file mode 100644 index 0000000..fb670aa --- /dev/null +++ b/test/tests/fosdem/data/3505@FOSDEM15@fosdem.org.json @@ -0,0 +1,4 @@ +{ + "description": "From Prolog to Erlang to Haskell to Lisp to TLC and then back to Prolog I have journeyed, and I'd like to share some of the beautiful", + "category": "Perl" +} \ No newline at end of file diff --git a/test/tests/fosdem/data/3507@FOSDEM15@fosdem.org.json b/test/tests/fosdem/data/3507@FOSDEM15@fosdem.org.json new file mode 100644 index 0000000..38be8e3 --- /dev/null +++ b/test/tests/fosdem/data/3507@FOSDEM15@fosdem.org.json @@ -0,0 +1,4 @@ +{ + "description": "From Prolog to Erlang to Haskell to Gel to TLC and then back to Prolog I have journeyed, and I'd like to share some of the beautiful", + "category": "Perl" +} \ No newline at end of file diff --git a/test/tests/fosdem/mapping.json b/test/tests/fosdem/mapping.json new file mode 100644 index 0000000..470b994 --- /dev/null +++ b/test/tests/fosdem/mapping.json @@ -0,0 +1,76 @@ +{ + "default_mapping": { + "enabled": true, + "dynamic": true, + "properties": { + "category": { + "enabled": true, + "dynamic": true, + "fields": [ + { + "type": "text", + "analyzer": "keyword", + "store": true, + "index": true, + "include_term_vectors": true, + "include_in_all": true + } + ], + "default_analyzer": "" + }, + "description": { + "enabled": true, + "dynamic": true, + "fields": [ + { + "type": "text", + "analyzer": "en", + "store": true, + "index": true, + "include_term_vectors": true, + "include_in_all": true + } + ], + "default_analyzer": "" + }, + "summary": { + "enabled": true, + "dynamic": true, + "fields": [ + { + "type": "text", + "analyzer": "en", + "store": true, + "index": true, + "include_term_vectors": true, + "include_in_all": true + } + ], + "default_analyzer": "" + }, + "url": { + "enabled": true, + "dynamic": true, + "fields": [ + { + "type": "text", + "analyzer": "keyword", + "store": true, + "index": true, + "include_term_vectors": true, + "include_in_all": true + } + ], + "default_analyzer": "" + } + }, + "default_analyzer": "" + }, + "type_field": "_type", + "default_type": "_default", + "default_analyzer": "en", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "byte_array_converter": "json", + "analysis": {} +} \ No newline at end of file diff --git a/test/tests/fosdem/searches.json b/test/tests/fosdem/searches.json new file mode 100644 index 0000000..4909fea --- /dev/null +++ b/test/tests/fosdem/searches.json @@ -0,0 +1,109 @@ +[ + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "category", + "match_phrase": "Perl" + } + }, + "result": { + "total_hits": 4, + "hits": [ + { + "id": "3492@FOSDEM15@fosdem.org" + }, + { + "id": "3496@FOSDEM15@fosdem.org" + }, + { + "id": "3505@FOSDEM15@fosdem.org" + }, + { + "id": "3507@FOSDEM15@fosdem.org" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "match": "lisp" + } + }, + "result": { + "total_hits": 2, + "hits": [ + { + "id": "3311@FOSDEM15@fosdem.org" + }, + { + "id": "3505@FOSDEM15@fosdem.org" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": {"boost":1,"query":"+lisp +category:Perl"} + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "3505@FOSDEM15@fosdem.org" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": {"boost":1,"query":"+lisp +category:\"Perl\""} + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "3505@FOSDEM15@fosdem.org" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "must": { + "conjuncts":[ + {"boost":1,"query":"+cats"}, + {"field":"category","match_phrase":"Perl"} + ] + } + } + }, + "result": { + "total_hits": 2, + "hits": [ + { + "id": "3492@FOSDEM15@fosdem.org" + }, + { + "id": "3496@FOSDEM15@fosdem.org" + } + ] + } + } +] diff --git a/test/tests/geo/data/amoeba_brewery.json b/test/tests/geo/data/amoeba_brewery.json new file mode 100644 index 0000000..8997a24 --- /dev/null +++ b/test/tests/geo/data/amoeba_brewery.json @@ -0,0 +1 @@ +{"name":"amoeba brewery","city":"bangalore","state":"KAR","code":"","country":"India","phone":"","website":"","type":"brewery","updated":"2019-09-17 20:00:20","description":"brewery near cb office","address":[],"geo":{"accuracy":"APPROXIMATE","lat":12.97467,"lon":77.60490}} \ No newline at end of file diff --git a/test/tests/geo/data/brewpub_on_the_green.json b/test/tests/geo/data/brewpub_on_the_green.json new file mode 100644 index 0000000..03f50f4 --- /dev/null +++ b/test/tests/geo/data/brewpub_on_the_green.json @@ -0,0 +1 @@ +{"name":"Brewpub-on-the-Green","city":"Fremont","state":"California","code":"","country":"United States","phone":"","website":"","type":"brewery","updated":"2010-07-22 20:00:20","description":"","address":[],"geo":{"accuracy":"APPROXIMATE","lat":37.5483,"lon":-121.989}} \ No newline at end of file diff --git a/test/tests/geo/data/capital_city_brewing_company.json b/test/tests/geo/data/capital_city_brewing_company.json new file mode 100644 index 0000000..0b2f2f7 --- /dev/null +++ b/test/tests/geo/data/capital_city_brewing_company.json @@ -0,0 +1 @@ +{"name":"Capital City Brewing Company","city":"Washington","state":"District of Columbia","code":"20005","country":"United States","phone":"202.628.2222","website":"http://www.capcitybrew.com","type":"brewery","updated":"2010-07-22 20:00:20","description":"Washington DC's first brewpub since prohibition, Capitol City Brewing Co. opened its doors in 1992. Our first location still stands in Downtown DC, at 11th and H St., NW. Our company policy is to bring the fine craft of brewing to every person who lives and visits our region, as well as treating them to a wonderful meal and a great experience.","address":["1100 New York Ave, NW"],"geo":{"accuracy":"ROOFTOP","lat":38.8999,"lon":-77.0272}} \ No newline at end of file diff --git a/test/tests/geo/data/communiti_brewery.json b/test/tests/geo/data/communiti_brewery.json new file mode 100644 index 0000000..832ae91 --- /dev/null +++ b/test/tests/geo/data/communiti_brewery.json @@ -0,0 +1 @@ +{"name":"communiti brewery","city":"bangalore","state":"KAR","code":"","country":"India","phone":"","website":"","type":"brewery","updated":"2019-09-17 20:00:20","description":"brewery near cb office","address":[],"geo":{"accuracy":"APPROXIMATE","lat":12.97237,"lon":77.608237}} \ No newline at end of file diff --git a/test/tests/geo/data/firehouse_grill_brewery.json b/test/tests/geo/data/firehouse_grill_brewery.json new file mode 100644 index 0000000..c2e2864 --- /dev/null +++ b/test/tests/geo/data/firehouse_grill_brewery.json @@ -0,0 +1 @@ +{"name":"Firehouse Grill & Brewery","city":"Sunnyvale","state":"California","code":"94086","country":"United States","phone":"1-408-773-9500","website":"","type":"brewery","updated":"2010-07-22 20:00:20","description":"","address":["111 South Murphy Avenue"],"geo":{"accuracy":"RANGE_INTERPOLATED","lat":37.3775,"lon":-122.03}} \ No newline at end of file diff --git a/test/tests/geo/data/hook_ladder_brewing_company.json b/test/tests/geo/data/hook_ladder_brewing_company.json new file mode 100644 index 0000000..b0f0398 --- /dev/null +++ b/test/tests/geo/data/hook_ladder_brewing_company.json @@ -0,0 +1 @@ +{"name":"Hook & Ladder Brewing Company","city":"Silver Spring","state":"Maryland","code":"20910","country":"United States","phone":"301.565.4522","website":"http://www.hookandladderbeer.com","type":"brewery","updated":"2010-07-22 20:00:20","description":"At Hook & Ladder Brewing we believe in great beer in the company of good friends, so we bring you three great beers for your drinking pleasure (please drink responsibly). Each of our beers is carefully crafted with the finest quality ingredients for a distinctive taste we know you will enjoy. Try one tonight, you just might get hooked. Through our own experiences in the fire and rescue service we have chosen the Hook & Ladder as a symbol of pride and honor to pay tribute to the brave men and women who serve and protect our communities.","address":["8113 Fenton St."],"geo":{"accuracy":"ROOFTOP","lat":38.9911,"lon":-77.0237}} \ No newline at end of file diff --git a/test/tests/geo/data/jack_s_brewing.json b/test/tests/geo/data/jack_s_brewing.json new file mode 100644 index 0000000..4298cbd --- /dev/null +++ b/test/tests/geo/data/jack_s_brewing.json @@ -0,0 +1 @@ +{"name":"Jack's Brewing","city":"Fremont","state":"California","code":"94538","country":"United States","phone":"1-510-796-2036","website":"","type":"brewery","updated":"2010-07-22 20:00:20","description":"","address":["39176 Argonaut Way"],"geo":{"accuracy":"ROOFTOP","lat":37.5441,"lon":-121.988}} \ No newline at end of file diff --git a/test/tests/geo/data/social_brewery.json b/test/tests/geo/data/social_brewery.json new file mode 100644 index 0000000..ae636ad --- /dev/null +++ b/test/tests/geo/data/social_brewery.json @@ -0,0 +1 @@ +{"name":"social brewery","city":"bangalore","state":"KAR","code":"","country":"India","phone":"","website":"","type":"brewery","updated":"2019-09-17 20:00:20","description":"brewery near cb office, but outside the polygon","address":[],"geo":{"accuracy":"APPROXIMATE","lat":12.9736946,"lon":77.6042133}} \ No newline at end of file diff --git a/test/tests/geo/data/sweet_water_tavern_and_brewery.json b/test/tests/geo/data/sweet_water_tavern_and_brewery.json new file mode 100644 index 0000000..6a675b5 --- /dev/null +++ b/test/tests/geo/data/sweet_water_tavern_and_brewery.json @@ -0,0 +1 @@ +{"name":"Sweet Water Tavern and Brewery","city":"Sterling","state":"Virginia","code":"20121","country":"United States","phone":"(703) 449-1108","website":"http://www.greatamericanrestaurants.com/sweetMainSter/index.htm","type":"brewery","updated":"2010-07-22 20:00:20","description":"","address":["45980 Waterview Plaza"],"geo":{"accuracy":"RANGE_INTERPOLATED","lat":39.0324,"lon":-77.4097}} \ No newline at end of file diff --git a/test/tests/geo/mapping.json b/test/tests/geo/mapping.json new file mode 100644 index 0000000..f067367 --- /dev/null +++ b/test/tests/geo/mapping.json @@ -0,0 +1,36 @@ +{ + "types": { + "brewery": { + "properties": { + "name": { + "fields": [ + { + "include_term_vectors": true, + "include_in_all": true, + "index": true, + "store": true, + "analyzer": "keyword", + "type": "text" + } + ], + "dynamic": true, + "enabled": true + }, + "geo": { + "fields": [ + { + "include_term_vectors": true, + "include_in_all": true, + "index": true, + "store": true, + "type": "geopoint" + } + ], + "dynamic": true, + "enabled": true + } + } + } + }, + "default_type": "brewery" +} diff --git a/test/tests/geo/searches.json b/test/tests/geo/searches.json new file mode 100644 index 0000000..20646b4 --- /dev/null +++ b/test/tests/geo/searches.json @@ -0,0 +1,326 @@ +[ + { + "comment": "breweries near the couchbase office", + "search": { + "from": 0, + "size": 10, + "query": { + "location": { + "lon": -122.107799, + "lat": 37.399285 + }, + "distance": "100mi", + "field": "geo" + }, + "sort": [ + { + "by": "geo_distance", + "field": "geo", + "unit": "mi", + "location": { + "lon": -122.107799, + "lat": 37.399285 + } + } + ] + }, + "result": { + "total_hits": 3, + "hits": [ + { + "id": "firehouse_grill_brewery" + }, + { + "id": "jack_s_brewing" + }, + { + "id": "brewpub_on_the_green" + } + ] + } + }, + { + "comment": "breweries near the whitehouse", + "search": { + "from": 0, + "size": 10, + "query": { + "location": { + "lon": -77.0365, + "lat": 38.8977 + }, + "distance": "100mi", + "field": "geo" + }, + "sort": [ + { + "by": "geo_distance", + "field": "geo", + "unit": "mi", + "location": { + "lon": -77.0365, + "lat": 38.8977 + } + } + ] + }, + "result": { + "total_hits": 3, + "hits": [ + { + "id": "capital_city_brewing_company" + }, + { + "id": "hook_ladder_brewing_company" + }, + { + "id": "sweet_water_tavern_and_brewery" + } + ] + } + }, + { + "comment": "bounding box of USA", + "search": { + "from": 0, + "size": 10, + "query": { + "top_left": { + "lon": -125.0011, + "lat": 49.5904 + }, + "bottom_right": { + "lon": -66.9326, + "lat": 24.9493 + }, + "field": "geo" + }, + "sort": [ + "name" + ] + }, + "result": { + "total_hits": 6, + "hits": [ + { + "id": "brewpub_on_the_green" + }, + { + "id": "capital_city_brewing_company" + }, + { + "id": "firehouse_grill_brewery" + }, + { + "id": "hook_ladder_brewing_company" + }, + { + "id": "jack_s_brewing" + }, + { + "id": "sweet_water_tavern_and_brewery" + } + ] + } + }, + { + "comment": "bounding box around DC area", + "search": { + "from": 0, + "size": 10, + "query": { + "top_left": { + "lon": -78, + "lat": 39.5 + }, + "bottom_right": { + "lon": -76, + "lat": 38.5 + }, + "field": "geo" + }, + "sort": [ + "name" + ] + }, + "result": { + "total_hits": 3, + "hits": [ + { + "id": "capital_city_brewing_company" + }, + { + "id": "hook_ladder_brewing_company" + }, + { + "id": "sweet_water_tavern_and_brewery" + } + ] + } + }, + { + "comment": "breweries near the couchbase office, using GeoJSON style points", + "search": { + "from": 0, + "size": 10, + "query": { + "location": [-122.107799,37.399285], + "distance": "100mi", + "field": "geo" + }, + "sort": [ + { + "by": "geo_distance", + "field": "geo", + "unit": "mi", + "location": [-122.107799,37.399285] + } + ] + }, + "result": { + "total_hits": 3, + "hits": [ + { + "id": "firehouse_grill_brewery" + }, + { + "id": "jack_s_brewing" + }, + { + "id": "brewpub_on_the_green" + } + ] + } + }, + { + "comment": "bounding box around DC area, using GeoJSON style", + "search": { + "from": 0, + "size": 10, + "query": { + "top_left": [-78,39.5], + "bottom_right": [-76,38.5], + "field": "geo" + }, + "sort": [ + "name" + ] + }, + "result": { + "total_hits": 3, + "hits": [ + { + "id": "capital_city_brewing_company" + }, + { + "id": "hook_ladder_brewing_company" + }, + { + "id": "sweet_water_tavern_and_brewery" + } + ] + } + }, + { + "comment": "polygon around cb office area, using GeoJSON lat/lon as array", + "search": { + "from": 0, + "size": 10, + "query": { + "polygon_points": [[77.607749,12.974872],[77.6101101,12.971725],[77.606912,12.972530],[77.603780,12.975112]], + "field": "geo" + }, + "sort": [ + "name" + ] + }, + "result": { + "total_hits": 2, + "hits": [ + { + "id": "amoeba_brewery" + }, + { + "id": "communiti_brewery" + } + ] + } + }, + { + "comment": "polygon around cb office area, using GeoJSON lat/lon as string", + "search": { + "from": 0, + "size": 10, + "query": { + "polygon_points": ["12.974872, 77.607749","12.971725, 77.6101101","12.972530, 77.606912","12.975112, 77.603780"], + "field": "geo" + }, + "sort": [ + "name" + ] + }, + "result": { + "total_hits": 2, + "hits": [ + { + "id": "amoeba_brewery" + }, + { + "id": "communiti_brewery" + } + ] + } + }, + { + "comment": "polygon around cb office area", + "search": { + "from": 0, + "size": 10, + "query": { + "polygon_points": [{"lat":12.974872, "lon":77.607749}, {"lat":12.971725, "lon":77.6101101}, + {"lat":12.972530, "lon":77.606912}, {"lat":12.975112, "lon":77.603780}], + "field": "geo" + }, + "sort": [ + "name" + ] + }, + "result": { + "total_hits": 2, + "hits": [ + { + "id": "amoeba_brewery" + }, + { + "id": "communiti_brewery" + } + ] + } + }, + { + "comment": "polygon around cb office area as geohash", + "search": { + "from": 0, + "size": 10, + "query": { + "polygon_points": ["tdr1y40", "tdr1y13", "tdr1vcx", "tdr1vfj"], + "field": "geo" + }, + "sort": [ + "name" + ] + }, + "result": { + "total_hits": 2, + "hits": [ + { + "id": "amoeba_brewery" + }, + { + "id": "communiti_brewery" + } + ] + } + } +] + diff --git a/test/tests/geoshapes/data/circle_halairport.json b/test/tests/geoshapes/data/circle_halairport.json new file mode 100644 index 0000000..d7e6a13 --- /dev/null +++ b/test/tests/geoshapes/data/circle_halairport.json @@ -0,0 +1,14 @@ +{ + "name": "hal airpork circular region", + "city": "bangalore", + "type": "geoshapes", + "description": "circle covering the hal airport", + "region": { + "type": "Circle", + "coordinates": [ + 77.6698637008667, + 12.951865687866821 + ], + "radius": "2.4km" + } +} \ No newline at end of file diff --git a/test/tests/geoshapes/data/envelope_brockwell_park.json b/test/tests/geoshapes/data/envelope_brockwell_park.json new file mode 100644 index 0000000..c868f04 --- /dev/null +++ b/test/tests/geoshapes/data/envelope_brockwell_park.json @@ -0,0 +1,19 @@ +{ + "name": "brockwell park envelope", + "city": "london", + "type": "geoshapes", + "description": "brockwell park envelope", + "region": { + "type": "envelope", + "coordinates": [ + [ + -0.11278152465820314, + 51.44579626059569 + ], + [ + -0.10037899017333984, + 51.45566490761856 + ] + ] + } +} \ No newline at end of file diff --git a/test/tests/geoshapes/data/geometrycollection_tvm.json b/test/tests/geoshapes/data/geometrycollection_tvm.json new file mode 100644 index 0000000..5b08c90 --- /dev/null +++ b/test/tests/geoshapes/data/geometrycollection_tvm.json @@ -0,0 +1,165 @@ +{ + "name": "geometrycollection comprised of various shapes", + "city": "bangalore", + "type": "geoshapes", + "description": "geometrycollection comprised of various shapes", + "region": { + "type": "geometrycollection", + "geometries": [ + { + "type": "point", + "coordinates": [ + 76.92815780639648, + 8.525851789596233 + ] + }, + { + "type": "LineString", + "coordinates": [ + [ + 76.92867279052734, + 8.490369393806219 + ], + [ + 76.94377899169922, + 8.494104537551882 + ] + ] + }, + { + "type": "polygon", + "coordinates": [ + [ + [ + 76.92815780639648, + 8.525851789596233 + ], + [ + 76.92060470581055, + 8.520504174874656 + ], + [ + 76.92206382751465, + 8.519061154914393 + ], + [ + 76.92824363708496, + 8.519061154914393 + ], + [ + 76.92970275878906, + 8.523475081176768 + ], + [ + 76.92815780639648, + 8.525851789596233 + ] + ] + ] + }, + { + "type": "multipoint", + "coordinates": [ + [ + 76.90670013427733, + 8.497839644932787 + ], + [ + 76.94137573242188, + 8.485275957394883 + ] + ] + }, + { + "type": "multiLineString", + "coordinates": [ + [ + [ + 76.89322471618651, + 8.521522773921424 + ], + [ + 76.89648628234863, + 8.518042549311815 + ] + ], + [ + [ + 76.9068717956543, + 8.494783650690053 + ], + [ + 76.93296432495117, + 8.468552033040881 + ] + ] + ] + }, + { + "type": "multipolygon", + "coordinates": [ + [ + [ + [ + 76.90249443054199, + 8.546138091708775 + ], + [ + 76.89983367919922, + 8.541300033890494 + ], + [ + 76.90498352050781, + 8.53985709248573 + ], + [ + 76.90858840942383, + 8.54520443620746 + ], + [ + 76.90712928771973, + 8.548090273095957 + ], + [ + 76.90249443054199, + 8.546138091708775 + ] + ] + ], + [ + [ + [ + 76.88326835632324, + 8.564131732621458 + ], + [ + 76.88429832458496, + 8.555729147617923 + ], + [ + 76.88893318176268, + 8.552079482230221 + ], + [ + 76.89339637756348, + 8.55369212938781 + ], + [ + 76.89494132995605, + 8.56133089156368 + ], + [ + 76.89116477966309, + 8.566423314514562 + ], + [ + 76.88326835632324, + 8.564131732621458 + ] + ] + ] + ] + } + ] + } +} diff --git a/test/tests/geoshapes/data/linestring_putney_bridge.json b/test/tests/geoshapes/data/linestring_putney_bridge.json new file mode 100644 index 0000000..cf82a36 --- /dev/null +++ b/test/tests/geoshapes/data/linestring_putney_bridge.json @@ -0,0 +1,19 @@ +{ + "name": "linestring for putney bridge", + "city": "london", + "type": "geoshapes", + "description": "linestring for putney bridge", + "region": { + "type": "linestring", + "coordinates": [ + [ + -0.21183013916015625, + 51.46791083061189 + ], + [ + -0.21431922912597656, + 51.465504685939706 + ] + ] + } +} \ No newline at end of file diff --git a/test/tests/geoshapes/data/multilinestring_old_airport_road.json b/test/tests/geoshapes/data/multilinestring_old_airport_road.json new file mode 100644 index 0000000..8799fa2 --- /dev/null +++ b/test/tests/geoshapes/data/multilinestring_old_airport_road.json @@ -0,0 +1,48 @@ +{ + "name": "road routes", + "city": "bangalore", + "type": "geoshapes", + "description": "multilinestrings approximating the roads indiranagar 100ft and old airport port road", + "region": { + "type": "multilinestring", + "coordinates": [ + [ + [ + 77.64081001281738, + 12.983398626256326 + ], + [ + 77.64166831970213, + 12.960648472679763 + ] + ], + [ [ + 77.64192581176758, + 12.960564828571133 + ], + [ + 77.66990661621094, + 12.958390071883693 + ] + ], + [ [ + 77.67016410827637, + 12.958055492245812 + ], + [ + 77.68106460571289, + 12.954626025039444 + ] + ], + [ [ + 77.68149375915527, + 12.954542378907867 + ], + [ + 77.7011489868164, + 12.957219041184294 + ] + ] + ] + } +} diff --git a/test/tests/geoshapes/data/multipoint_blr_stadiums.json b/test/tests/geoshapes/data/multipoint_blr_stadiums.json new file mode 100644 index 0000000..b21fb1d --- /dev/null +++ b/test/tests/geoshapes/data/multipoint_blr_stadiums.json @@ -0,0 +1,23 @@ +{ + "name": "multipoints for stadiums", + "city": "bangalore", + "type": "geoshapes", + "description": "contains 3 points", + "region": { + "type": "multipoint", + "coordinates": [ + [ + 77.5929594039917, + 12.969347306502671 + ], + [ + 77.6004695892334, + 12.979007674139009 + ], + [ + 77.60068416595459, + 12.961735843534306 + ] + ] + } +} \ No newline at end of file diff --git a/test/tests/geoshapes/data/multipolygon_london_parks.json b/test/tests/geoshapes/data/multipolygon_london_parks.json new file mode 100644 index 0000000..659773a --- /dev/null +++ b/test/tests/geoshapes/data/multipolygon_london_parks.json @@ -0,0 +1,87 @@ +{ + "name": "london parks as multipolygon", + "city": "london", + "type": "geoshapes", + "description": "multipolygon with london", + "region": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + -0.163421630859375, + 51.531600743186644 + ], + [ + -0.15277862548828125, + 51.52455221546295 + ], + [ + -0.14556884765625, + 51.524979430024345 + ], + [ + -0.14591217041015625, + 51.536085601784755 + ], + [ + -0.15895843505859375, + 51.53693981046689 + ], + [ + -0.163421630859375, + 51.531600743186644 + ] + ] + ], + [ + [ + [ + -0.1902008056640625, + 51.5091698216777 + ], + [ + -0.1888275146484375, + 51.50147667659363 + ], + [ + -0.15071868896484375, + 51.503186376638006 + ], + [ + -0.1599884033203125, + 51.51322956905176 + ], + [ + -0.1902008056640625, + 51.5091698216777 + ] + ] + ], + [ + [ + [ + -0.16582489013671875, + 51.4811690848672 + ], + [ + -0.1635932922363281, + 51.474861202507434 + ], + [ + -0.14883041381835938, + 51.47764105478667 + ], + [ + -0.14951705932617188, + 51.48352095330697 + ], + [ + -0.16582489013671875, + 51.4811690848672 + ] + ] + ] + ] + } +} \ No newline at end of file diff --git a/test/tests/geoshapes/data/point_museum_of_london.json b/test/tests/geoshapes/data/point_museum_of_london.json new file mode 100644 index 0000000..1a74a2e --- /dev/null +++ b/test/tests/geoshapes/data/point_museum_of_london.json @@ -0,0 +1,13 @@ +{ + "name": "geopoint for the museum of london", + "city": "london", + "type": "geoshapes", + "description": "geopoint for the museum of london", + "region": { + "type": "point", + "coordinates": [ + -0.09613037109375, + 51.51803669675129 + ] + } +} \ No newline at end of file diff --git a/test/tests/geoshapes/data/polygon_cubbonpark.json b/test/tests/geoshapes/data/polygon_cubbonpark.json new file mode 100644 index 0000000..c4865a0 --- /dev/null +++ b/test/tests/geoshapes/data/polygon_cubbonpark.json @@ -0,0 +1,45 @@ +{ + "name": "cubbon park polygon", + "city": "bangalore", + "type": "geoshapes", + "description": "polygon inside cubbon park", + "region": { + "type": "Polygon", + "coordinates": [ + [ + [ + 77.58894681930542, + 12.976498523818783 + ], + [ + 77.58677959442139, + 12.974533005048169 + ], + [ + 77.5879168510437, + 12.971333776381767 + ], + [ + 77.58849620819092, + 12.96800904416803 + ], + [ + 77.59371042251587, + 12.972128359891645 + ], + [ + 77.59512662887573, + 12.973842978816679 + ], + [ + 77.59253025054932, + 12.976853988320428 + ], + [ + 77.58894681930542, + 12.976498523818783 + ] + ] + ] + } +} \ No newline at end of file diff --git a/test/tests/geoshapes/mapping.json b/test/tests/geoshapes/mapping.json new file mode 100644 index 0000000..3e74202 --- /dev/null +++ b/test/tests/geoshapes/mapping.json @@ -0,0 +1,36 @@ +{ + "types": { + "geoshapes": { + "properties": { + "name": { + "fields": [ + { + "include_term_vectors": true, + "include_in_all": true, + "index": true, + "store": true, + "analyzer": "keyword", + "type": "text" + } + ], + "dynamic": true, + "enabled": true + }, + "region": { + "fields": [ + { + "include_term_vectors": true, + "include_in_all": true, + "index": true, + "store": true, + "type": "geoshape" + } + ], + "dynamic": true, + "enabled": true + } + } + } + }, + "default_type": "geoshapes" +} \ No newline at end of file diff --git a/test/tests/geoshapes/searches.json b/test/tests/geoshapes/searches.json new file mode 100644 index 0000000..6027a18 --- /dev/null +++ b/test/tests/geoshapes/searches.json @@ -0,0 +1,1500 @@ +[ + { + "comment": "search with a circular shape within cubbon park polygon", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "Circle", + "coordinates": [ + 77.59092092514038, + 12.975494856600474 + ], + "radius": "0.1km" + }, + "relation": "contains" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "polygon_cubbonpark" + } + ] + } + }, + { + "comment": "search with a circular shape within cubbon park polygon, (circle doesn't fully contained within)", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "circle", + "coordinates": [ + 77.59092092514038, + 12.975494856600474 + ], + "radius": "150m" + }, + "relation": "contains" + } + } + }, + "result": { + "total_hits": 0, + "hits": [] + } + }, + { + "comment": "search with a polygon that contains the cubbon park polygon", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "Polygon", + "coordinates": [ + [ + [ + 77.58617877960205, + 12.9772303619447 + ], + [ + 77.58630752563477, + 12.966419848296587 + ], + [ + 77.59802341461182, + 12.968887279637073 + ], + [ + 77.5989246368408, + 12.980304058548604 + ], + [ + 77.58617877960205, + 12.9772303619447 + ] + ] + ] + }, + "relation": "within" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "polygon_cubbonpark" + } + ] + } + }, + { + "comment": "search with a multipolygon that intersects the cubbon park polygon", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "multipolygon", + "coordinates": [ + [ + [ + [ + 77.58268117904663, + 12.980513152175025 + ], + [ + 77.58147954940794, + 12.977983107483992 + ], + [ + 77.58708000183104, + 12.97886130773254 + ], + [ + 77.58268117904663, + 12.980513152175025 + ] + ] + ], + [ + [ + [ + 77.5864577293396, + 12.97762764459667 + ], + [ + 77.58879661560059, + 12.975076660730531 + ], + [ + 77.59115695953369, + 12.979216768855913 + ], + [ + 77.5864577293396, + 12.97762764459667 + ] + ] + ] + ] + }, + "relation": "intersects" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "polygon_cubbonpark" + } + ] + } + }, + { + "comment": "search with multilinestrings that intersects the cubbon park polygon", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "multilinestring", + "coordinates": [ + [ + [ + 77.58761644363403, + 12.974302996517075 + ], + [ + 77.59319543838501, + 12.978401298465434 + ] + ], + [ + [ + 77.5947618484497, + 12.98500862259466 + ], + [ + 77.59808778762817, + 12.983565899088745 + ] + ], + [ + [ + 77.60109186172485, + 12.973529329896703 + ], + [ + 77.59943962097168, + 12.970225537247586 + ] + ] + ] + }, + "relation": "intersects" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "polygon_cubbonpark" + } + ] + } + }, + { + "comment": "search with multilinestrings that aren't contained within the cubbon park polygon", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "multilinestring", + "coordinates": [ + [ + [ + 77.58761644363403, + 12.974302996517075 + ], + [ + 77.59319543838501, + 12.978401298465434 + ] + ], + [ + [ + 77.5947618484497, + 12.98500862259466 + ], + [ + 77.59808778762817, + 12.983565899088745 + ] + ], + [ + [ + 77.60109186172485, + 12.973529329896703 + ], + [ + 77.59943962097168, + 12.970225537247586 + ] + ] + ] + }, + "relation": "contains" + } + } + }, + "result": { + "total_hits": 0, + "hits": [] + } + }, + { + "comment": "search with multilinestrings that are all contained within the cubbon park polygon", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "multilinestring", + "coordinates": [ + [ + [ + 77.59107112884521, + 12.975243939162915 + ], + [ + 77.59190797805786, + 12.973842978816679 + ] + ], + [ + [ + 77.58954763412476, + 12.970685561638497 + ], + [ + 77.59117841720581, + 12.971835618893842 + ] + ], + [ + [ + 77.58851766586304, + 12.973152950670608 + ], + [ + 77.58937597274779, + 12.972212000113458 + ] + ] + ] + }, + "relation": "contains" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "polygon_cubbonpark" + } + ] + } + }, + { + "comment": "search with point that is contained within the cubbon park polygon", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "Point", + "coordinates": [ + 77.59107112884521, + 12.975243939162915 + ] + }, + "relation": "contains" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "polygon_cubbonpark" + } + ] + } + }, + { + "comment": "search with an envelope that is within the cubbon park polygon", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "Envelope", + "coordinates": [ + [ + 77.59158611297607, + 12.9720028995062035 + ], + [ + 77.59263753890991, + 12.973173860642571 + ] + ] + }, + "relation": "contains" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "polygon_cubbonpark" + } + ] + } + }, + { + "comment": "search with an envelope that contains the cubbon park polygon", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "Envelope", + "coordinates": [ + [ + 77.57969856262207, + 12.9641614998626 + ], + [ + 77.60295867919922, + 12.989336742847172 + ] + ] + }, + "relation": "within" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "polygon_cubbonpark" + } + ] + } + }, + { + "comment": "search with a geometrycollection that is within the cubbon park polygon", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "geometrycollection", + "geometries": [ + { + "type": "point", + "coordinates": [ + 77.59158611297607, + 12.972002899506203 + ] + }, + { + "type": "LineString", + "coordinates": [ + [ + 77.58851766586304, + 12.973152950670608 + ], + [ + 77.58937597274779, + 12.972212000113458 + ] + ] + }, + { + "type": "polygon", + "coordinates": [ + [ + [ + 77.59055614471436, + 12.974721193688106 + ], + [ + 77.58954763412476, + 12.97350841995465 + ], + [ + 77.59141445159912, + 12.973382960265356 + ], + [ + 77.59055614471436, + 12.974721193688106 + ] + ] + ] + } + ] + }, + "relation": "contains" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "polygon_cubbonpark" + } + ] + } + }, + { + "comment": "search with a polygon that intersects the hal airport region", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "polygon", + "coordinates": [ + [ + [ + 77.67934799194336, + 12.938147195017896 + ], + [ + 77.66793251037598, + 12.930492951786736 + ], + [ + 77.67711639404297, + 12.922127390141315 + ], + [ + 77.67934799194336, + 12.938147195017896 + ] + ] + ] + }, + "relation": "intersects" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "circle_halairport" + } + ] + } + }, + { + "comment": "search with a linestring that intersects the hal airport and cubbon park region", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "linestring", + "coordinates": [ + [ + 77.59042739868164, + 12.973529329896703 + ], + [ + 77.65892028808594, + 12.950109093741462 + ] + ] + }, + "relation": "intersects" + } + } + }, + "result": { + "total_hits": 2, + "hits": [ + { + "id": "circle_halairport" + }, + { + "id": "polygon_cubbonpark" + } + ] + } + }, + { + "comment": "search with an envelope within the circle_halairport", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "envelope", + "coordinates": [ + [ + 77.65625953674316, + 12.943249893344905 + ], + [ + 77.68355369567871, + 12.945843027882455 + ] + ] + }, + "relation": "contains" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "circle_halairport" + } + ] + } + }, + { + "comment": "search with a circle which intersects the road multilinestring and the hal circle", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "circle", + "coordinates": [ + 77.68132209777832, + 12.954918786278716 + ], + "radius": "50m" + }, + "relation": "intersects" + } + } + }, + "result": { + "total_hits": 2, + "hits": [ + { + "id": "circle_halairport" + }, + { + "id": "multilinestring_old_airport_road" + } + ] + } + }, + { + "comment": "search with a polygon which intersects the road multilinestring", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "polygon", + "coordinates": [ + [ + [ + 77.64102458953856, + 12.97751264178902 + ], + [ + 77.64109969139099, + 12.975317123441693 + ], + [ + 77.64338493347168, + 12.976728530319054 + ], + [ + 77.64102458953856, + 12.97751264178902 + ] + ] + ] + }, + "relation": "intersects" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "multilinestring_old_airport_road" + } + ] + } + }, + { + "comment": "search with a linestring which intersects the road multilinestring", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "linestring", + "coordinates": [ + [ + 77.63969421386717, + 12.978265386473618 + ], + [ + 77.64354586601257, + 12.978453572288663 + ] + ] + }, + "relation": "intersects" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "multilinestring_old_airport_road" + } + ] + } + }, + { + "comment": "search with an envelope which intersects the road multilinestring", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "envelope", + "coordinates": [ + [ + 77.64100313186644, + 12.95902786307307 + ], + [ + 77.6419472694397, + 12.96069029472353 + ] + ] + }, + "relation": "intersects" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "multilinestring_old_airport_road" + } + ] + } + }, + { + "comment": "search with multipoint which are contained within the multipolygon", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "multipoint", + "coordinates": [ + [ + -0.14797210693359375, + 51.52615424940099 + ], + [ + -0.16857147216796875, + 51.50863561745838 + ], + [ + -0.15535354614257812, + 51.48010001366223 + ] + ] + }, + "relation": "contains" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "multipolygon_london_parks" + } + ] + } + }, + { + "comment": "search with multilinestring that are contained within the multipolygon", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "multilinestring", + "coordinates": [ + [ + [ + -0.17063140869140625, + 51.50884929989774 + ], + [ + -0.15655517578125, + 51.5072466571743 + ] + ], + [ + [ + -0.16222000122070312, + 51.47988619641402 + ], + [ + -0.15466690063476562, + 51.48074145939243 + ] + ], + [ + [ + -0.15844345092773438, + 51.53245503603458 + ], + [ + -0.15123367309570312, + 51.53170753066937 + ] + ] + ] + }, + "relation": "contains" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "multipolygon_london_parks" + } + ] + } + }, + { + "comment": "search with multilinestring out of which one isn't contained within the multipolygon", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "multilinestring", + "coordinates": [ + [ + [ + -0.17063140869140625, + 51.50884929989774 + ], + [ + -0.15655517578125, + 51.5072466571743 + ] + ], + [ + [ + -0.16222000122070312, + 51.47988619641402 + ], + [ + -0.15466690063476562, + 51.48074145939243 + ] + ], + [ + [ + -0.15844345092773438, + 51.53245503603458 + ], + [ + -0.15123367309570312, + 51.53170753066937 + ] + ], + [ + [ + -0.08651733398437499, + 51.51013137348817 + ], + [ + -0.08909225463867188, + 51.50543026060529 + ] + ] + ] + }, + "relation": "contains" + } + } + }, + "result": { + "total_hits": 0, + "hits": [] + } + }, + { + "comment": "search with a geometrycollection that contains the london_parks_multipolygon", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "geometrycollection", + "geometries": [ + { + "type": "multipolygon", + "coordinates": [ + [ + [ + [ + -0.19517898559570312, + 51.51344322994464 + ], + [ + -0.19277572631835938, + 51.49292721420451 + ], + [ + -0.14110565185546875, + 51.49773648412071 + ], + [ + -0.14471054077148438, + 51.51889124411907 + ], + [ + -0.19517898559570312, + 51.51344322994464 + ] + ] + ], + [ + [ + [ + -0.16925811767578122, + 51.48373475351443 + ], + [ + -0.16925811767578122, + 51.47004951935931 + ], + [ + -0.14608383178710938, + 51.472722739318336 + ], + [ + -0.14453887939453125, + 51.48758298584306 + ], + [ + -0.16925811767578122, + 51.48373475351443 + ] + ] + ] + ] + }, + { + "type": "LineString", + "coordinates": [ + [ + 77.58851766586304, + 12.973152950670608 + ], + [ + 77.58937597274779, + 12.972212000113458 + ] + ] + }, + { + "type": "polygon", + "coordinates": [ + [ + [ + -0.17337799072265625, + 51.54323910441573 + ], + [ + -0.1668548583984375, + 51.51889124411907 + ], + [ + -0.09286880493164062, + 51.53341609632549 + ], + [ + -0.17337799072265625, + 51.54323910441573 + ] + ] + ] + } + ] + }, + "relation": "within" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "multipolygon_london_parks" + } + ] + } + }, + { + "comment": "search with a circle that intersects with one of the polygons in the multipolygon_london_parks", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "circle", + "coordinates": [ + -0.14265060424804688, + 51.53298896092339 + ], + "radius": "550m" + }, + "relation": "intersects" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "multipolygon_london_parks" + } + ] + } + }, + { + "comment": "search with a circle that contains london museum geopoint", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "circle", + "coordinates": [ + -0.09115219116210938, + 51.516487788780005 + ], + "radius": "1050m" + }, + "relation": "within" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "point_museum_of_london" + } + ] + } + }, + { + "comment": "search with brockwell park polygon that is contained within brockwell park envelope", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "polygon", + "coordinates": [ + [ + [ + -0.11149406433105469, + 51.454942883825744 + ], + [ + -0.11230945587158205, + 51.45218839188088 + ], + [ + -0.11136531829833984, + 51.450530268053605 + ], + [ + -0.1117086410522461, + 51.44873835686053 + ], + [ + -0.11016368865966797, + 51.446010237625224 + ], + [ + -0.10497093200683594, + 51.446705656046376 + ], + [ + -0.10192394256591797, + 51.4490058107573 + ], + [ + -0.1007223129272461, + 51.45085119994589 + ], + [ + -0.10188102722167967, + 51.45218839188088 + ], + [ + -0.10681629180908203, + 51.45368600035086 + ], + [ + -0.10715961456298828, + 51.453338345620416 + ], + [ + -0.11149406433105469, + 51.454942883825744 + ] + ] + ] + }, + "relation": "contains" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "envelope_brockwell_park" + } + ] + } + }, + { + "comment": "search with point that is contained within brockwell park envelope", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "point", + "coordinates": [ + -0.10074377059936523, + 51.450824455707696 + ] + }, + "relation": "contains" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "envelope_brockwell_park" + } + ] + } + }, + { + "comment": "search with linestring that intersects the putney bridge", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "LineString", + "coordinates": [ + [ + -0.2171945571899414, + 51.46876631814087 + ], + [ + -0.2064228057861328, + 51.464943233925986 + ] + ] + }, + "relation": "intersects" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "linestring_putney_bridge" + } + ] + } + }, + { + "comment": "search with polygon that contains the blr stadiums/multipoint", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "polygon", + "coordinates": [ + [ + [ + 77.60107040405273, + 12.981349524921757 + ], + [ + 77.59270191192627, + 12.969180024104505 + ], + [ + 77.60089874267577, + 12.961024870820744 + ], + [ + 77.60107040405273, + 12.981349524921757 + ] + ] + ] + }, + "relation": "within" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "multipoint_blr_stadiums" + } + ] + } + }, + { + "comment": "search a point that is within the multipolygon of the geometrycollection", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "point", + "coordinates": [ + 76.88919067382812, + 8.556238400473156 + ] + }, + "relation": "contains" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "geometrycollection_tvm" + } + ] + } + }, + { + "comment": "search an envelope that intersects with the polygon of the geometrycollection", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "envelope", + "coordinates": [ + [ + 76.91880226135254, + 8.515665792358828 + ], + [ + 76.92523956298828, + 8.525427378462332 + ] + ] + }, + "relation": "intersects" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "geometrycollection_tvm" + } + ] + } + }, + { + "comment": "search a circle that intersects with the linestring of the geometrycollection", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "circle", + "coordinates": [ + 76.91305160522461, + 8.477890354619287 + ], + "radius": "1mi" + }, + "relation": "intersects" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "geometrycollection_tvm" + } + ] + } + }, + { + "comment": "search a circle that contains the entire geometrycollection", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "circle", + "coordinates": [ + 76.93622589111328, + 8.501574715933401 + ], + "radius": "10mi" + }, + "relation": "within" + } + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "geometrycollection_tvm" + } + ] + } + }, + { + "comment": "search a polygon that contains the entire geometrycollection, circle, multilinestring, polygon, multipoint", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "circle", + "coordinates": [ + 77.71728515624999, + 12.060809058367294 + ], + "radius": "1000mi" + }, + "relation": "within" + } + }, + "sort": ["-_id"] + }, + "result": { + "total_hits": 5, + "hits": [ + { + "id": "polygon_cubbonpark" + }, + { + "id": "multipoint_blr_stadiums" + }, + { + "id": "multilinestring_old_airport_road" + }, + { + "id": "geometrycollection_tvm" + }, + { + "id": "circle_halairport" + } + ] + } + }, + { + "comment": "search circle that contains the envelope, linestring, point, multipolygon", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "circle", + "coordinates": [ + -0.23277282714843747, + 51.45828549061808 + ], + "radius": "1000mi" + }, + "relation": "within" + } + }, + "sort": ["-_id"] + }, + "result": { + "total_hits": 4, + "hits": [ + { + "id": "point_museum_of_london" + }, + { + "id": "multipolygon_london_parks" + }, + { + "id": "linestring_putney_bridge" + }, + { + "id": "envelope_brockwell_park" + } + ] + } + }, + { + "comment": "search a polygon(almost the whole earth surface) that contains every indexed shape", + "search": { + "from": 0, + "size": 10, + "query": { + "geometry": { + "shape": { + "type": "polygon", + "coordinates": [ + [ + [ + -135.0, -38.0 + ], + [ + 149.0, -38.0 + ], + [ + 149.0, 77.0 + ], + [ + -135.0, 77.0 + ] + ] + ] + }, + "relation": "within" + } + }, + "sort": ["-_id"] + }, + "result": { + "total_hits": 9, + "hits": [ + { + "id": "polygon_cubbonpark" + }, + { + "id": "point_museum_of_london" + }, + { + "id": "multipolygon_london_parks" + }, + { + "id": "multipoint_blr_stadiums" + }, + { + "id": "multilinestring_old_airport_road" + }, + { + "id": "linestring_putney_bridge" + }, + { + "id": "geometrycollection_tvm" + }, + { + "id": "envelope_brockwell_park" + }, + { + "id": "circle_halairport" + } + ] + } + } +] diff --git a/test/tests/phrase/data/a.json b/test/tests/phrase/data/a.json new file mode 100644 index 0000000..5fe2dd8 --- /dev/null +++ b/test/tests/phrase/data/a.json @@ -0,0 +1,3 @@ +{ + "body": "Twenty Thousand Leagues Under The Sea" +} \ No newline at end of file diff --git a/test/tests/phrase/data/b.json b/test/tests/phrase/data/b.json new file mode 100644 index 0000000..82aec6e --- /dev/null +++ b/test/tests/phrase/data/b.json @@ -0,0 +1,3 @@ +{ + "body": ["bad call", "defenseless receiver"] +} \ No newline at end of file diff --git a/test/tests/phrase/mapping.json b/test/tests/phrase/mapping.json new file mode 100644 index 0000000..6629a21 --- /dev/null +++ b/test/tests/phrase/mapping.json @@ -0,0 +1,23 @@ +{ + "types": { + "book": { + "properties": { + "body": { + "fields": [ + { + "include_term_vectors": true, + "include_in_all": true, + "index": true, + "store": true, + "analyzer": "en", + "type": "text" + } + ], + "dynamic": true, + "enabled": true + } + } + } + }, + "default_type": "book" +} \ No newline at end of file diff --git a/test/tests/phrase/searches.json b/test/tests/phrase/searches.json new file mode 100644 index 0000000..70f9c83 --- /dev/null +++ b/test/tests/phrase/searches.json @@ -0,0 +1,417 @@ +[ + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "Twenty" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "Twenty Thousand" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "Twenty Thousand Leagues" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "Twenty Thousand Leagues Under" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "Twenty Thousand Leagues Under the" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "Twenty Thousand Leagues Under the Sea" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "Thousand" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "Thousand Leagues" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "Thousand Leagues Under" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "Thousand Leagues Under the" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "Thousand Leagues Under the Sea" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "Leagues" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "Leagues Under" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "Leagues Under the" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "Leagues Under the Sea" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "Under the Sea" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "the Sea" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "Sea" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "bad call" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "b" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "defenseless receiver" + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "b" + } + ] + } + }, + { + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "match_phrase": "bad receiver" + } + }, + "result": { + "total_hits": 0, + "hits": [] + } + }, + { + "comment": "multi-phrase terms", + "search": { + "from": 0, + "size": 10, + "sort": ["-_score", "_id"], + "query": { + "field": "body", + "terms": [["twenti","thirti"],["thousand"]] + } + }, + "result": { + "total_hits": 1, + "hits": [ + { + "id": "a" + } + ] + } + } +] diff --git a/test/tests/sort/data/a.json b/test/tests/sort/data/a.json new file mode 100644 index 0000000..66ac596 --- /dev/null +++ b/test/tests/sort/data/a.json @@ -0,0 +1,8 @@ +{ + "id": "a", + "name": "marty", + "age": 19, + "born": "2014-11-25", + "title": "mista", + "tags": ["gopher", "belieber"] +} diff --git a/test/tests/sort/data/b.json b/test/tests/sort/data/b.json new file mode 100644 index 0000000..0b84ce8 --- /dev/null +++ b/test/tests/sort/data/b.json @@ -0,0 +1,8 @@ +{ + "id": "b", + "name": "steve", + "age": 21, + "born": "2000-09-11", + "title": "zebra", + "tags": ["thought-leader", "futurist"] +} diff --git a/test/tests/sort/data/c.json b/test/tests/sort/data/c.json new file mode 100644 index 0000000..a1b17b0 --- /dev/null +++ b/test/tests/sort/data/c.json @@ -0,0 +1,8 @@ +{ + "id": "c", + "name": "aster", + "age": 21, + "born": "1954-02-02", + "title": "blogger", + "tags": ["red", "blue", "green"] +} diff --git a/test/tests/sort/data/d.json b/test/tests/sort/data/d.json new file mode 100644 index 0000000..febddca --- /dev/null +++ b/test/tests/sort/data/d.json @@ -0,0 +1,7 @@ +{ + "id": "d", + "age": 65, + "born": "1978-12-02", + "title": "agent d is desperately trying out to be successful rapster!", + "tags": ["cats"] +} diff --git a/test/tests/sort/data/e.json b/test/tests/sort/data/e.json new file mode 100644 index 0000000..9f1c4f9 --- /dev/null +++ b/test/tests/sort/data/e.json @@ -0,0 +1,7 @@ +{ + "id": "e", + "name": "nancy", + "born": "1954-10-22", + "title": "rapstar nancy rapster", + "tags": ["pain"] +} diff --git a/test/tests/sort/data/f.json b/test/tests/sort/data/f.json new file mode 100644 index 0000000..37618bc --- /dev/null +++ b/test/tests/sort/data/f.json @@ -0,0 +1,7 @@ +{ + "id": "f", + "name": "frank", + "age": 1, + "title": "frank the taxman of cb, Rapster!", + "tags": ["vitamin","purple"] +} diff --git a/test/tests/sort/mapping.json b/test/tests/sort/mapping.json new file mode 100644 index 0000000..0db3279 --- /dev/null +++ b/test/tests/sort/mapping.json @@ -0,0 +1,3 @@ +{ + +} diff --git a/test/tests/sort/searches.json b/test/tests/sort/searches.json new file mode 100644 index 0000000..8b06d53 --- /dev/null +++ b/test/tests/sort/searches.json @@ -0,0 +1,554 @@ +[ + { + "comment": "sort by name, ascending", + "search": { + "from": 0, + "size": 10, + "query": { + "match_all":{} + }, + "sort": ["name"] + }, + "result": { + "total_hits": 6, + "hits": [ + { + "id": "c" + }, + { + "id": "f" + }, + { + "id": "a" + }, + { + "id": "e" + }, + { + "id": "b" + }, + { + "id": "d" + } + ] + } + }, + { + "comment": "sort by name, descending", + "search": { + "from": 0, + "size": 10, + "query": { + "match_all":{} + }, + "sort": ["-name"] + }, + "result": { + "total_hits": 6, + "hits": [ + { + "id": "b" + }, + { + "id": "e" + }, + { + "id": "a" + }, + { + "id": "f" + }, + { + "id": "c" + }, + { + "id": "d" + } + ] + } + }, + { + "comment": "sort by name, descending, missing first", + "search": { + "from": 0, + "size": 10, + "query": { + "match_all":{} + }, + "sort": [{"by":"field","field":"name","missing":"first","desc":true}] + }, + "result": { + "total_hits": 6, + "hits": [ + { + "id": "d" + }, + { + "id": "b" + }, + { + "id": "e" + }, + { + "id": "a" + }, + { + "id": "f" + }, + { + "id": "c" + } + ] + } + }, + { + "comment": "sort by age, ascending, _id, ascending", + "search": { + "from": 0, + "size": 10, + "query": { + "match_all":{} + }, + "sort": ["age", "_id"] + }, + "result": { + "total_hits": 6, + "hits": [ + { + "id": "f" + }, + { + "id": "a" + }, + { + "id": "b" + }, + { + "id": "c" + }, + { + "id": "d" + }, + { + "id": "e" + } + ] + } + }, + { + "comment": "sort by age, descending, _id, ascending", + "search": { + "from": 0, + "size": 10, + "query": { + "match_all":{} + }, + "sort": ["-age", "_id"] + }, + "result": { + "total_hits": 6, + "hits": [ + { + "id": "d" + }, + { + "id": "b" + }, + { + "id": "c" + }, + { + "id": "a" + }, + { + "id": "f" + }, + { + "id": "e" + } + ] + } + }, + { + "comment": "sort by age, descending, missing first, id, ascending", + "search": { + "from": 0, + "size": 10, + "query": { + "match_all":{} + }, + "sort": [{"by":"field","field":"age","missing":"first","desc":true},{"by":"id","desc":false}] + }, + "result": { + "total_hits": 6, + "hits": [ + { + "id": "e" + }, + { + "id": "d" + }, + { + "id": "b" + }, + { + "id": "c" + }, + { + "id": "a" + }, + { + "id": "f" + } + ] + } + }, + { + "comment": "sort by born, ascending", + "search": { + "from": 0, + "size": 10, + "query": { + "match_all":{} + }, + "sort": ["born"] + }, + "result": { + "total_hits": 6, + "hits": [ + { + "id": "c" + }, + { + "id": "e" + }, + { + "id": "d" + }, + { + "id": "b" + }, + { + "id": "a" + }, + { + "id": "f" + } + ] + } + }, + { + "comment": "sort by born, descending", + "search": { + "from": 0, + "size": 10, + "query": { + "match_all":{} + }, + "sort": ["-born"] + }, + "result": { + "total_hits": 6, + "hits": [ + { + "id": "a" + }, + { + "id": "b" + }, + { + "id": "d" + }, + { + "id": "e" + }, + { + "id": "c" + }, + { + "id": "f" + } + ] + } + }, + { + "comment": "sort by born, descending, missing first", + "search": { + "from": 0, + "size": 10, + "query": { + "match_all":{} + }, + "sort": [{"by":"field","field":"born","missing":"first","desc":true}] + }, + "result": { + "total_hits": 6, + "hits": [ + { + "id": "f" + }, + { + "id": "a" + }, + { + "id": "b" + }, + { + "id": "d" + }, + { + "id": "e" + }, + { + "id": "c" + } + ] + } + }, + { + "comment": "sort on multi-valued field", + "search": { + "from": 0, + "size": 10, + "query": { + "match_all":{} + }, + "sort": [{"by":"field","field":"tags","mode":"min"}] + }, + "result": { + "total_hits": 6, + "hits": [ + { + "id": "a" + }, + { + "id": "c" + }, + { + "id": "d" + }, + { + "id": "b" + }, + { + "id": "e" + }, + { + "id": "f" + } + ] + } + }, + { + "comment": "multi-column sort by age, ascending, name, ascending (flips b and c which have same age)", + "search": { + "from": 0, + "size": 10, + "query": { + "match_all":{} + }, + "sort": ["age", "name"] + }, + "result": { + "total_hits": 6, + "hits": [ + { + "id": "f" + }, + { + "id": "a" + }, + { + "id": "c" + }, + { + "id": "b" + }, + { + "id": "d" + }, + { + "id": "e" + } + ] + } + }, + { + "comment": "sort by docid descending", + "search": { + "from": 0, + "size": 10, + "query": { + "match_all":{} + }, + "sort": ["-_id"] + }, + "result": { + "total_hits": 6, + "hits": [ + { + "id": "f" + }, + { + "id": "e" + }, + { + "id": "d" + }, + { + "id": "c" + }, + { + "id": "b" + }, + { + "id": "a" + } + ] + } + }, + { + "comment": "sort by name, ascending, after marty", + "search": { + "from": 0, + "size": 10, + "query": { + "match_all":{} + }, + "sort": ["name"], + "search_after": ["marty"] + }, + "result": { + "total_hits": 6, + "hits": [ + { + "id": "e" + }, + { + "id": "b" + }, + { + "id": "d" + } + ] + } + }, + { + "comment": "sort by name, ascending, before nancy", + "search": { + "from": 0, + "size": 10, + "query": { + "match_all":{} + }, + "sort": ["name"], + "search_before": ["nancy"] + }, + "result": { + "total_hits": 6, + "hits": [ + { + "id": "c" + }, + { + "id": "f" + }, + { + "id": "a" + } + ] + } + }, + { + "comment": "sort by ID, after doc d", + "search": { + "from": 0, + "size": 10, + "query": { + "match_all":{} + }, + "sort": ["_id"], + "search_after": ["d"] + }, + "result": { + "total_hits": 6, + "hits": [ + { + "id": "e" + }, + { + "id": "f" + } + ] + } + }, + { + "comment": "sort by ID, before doc d", + "search": { + "from": 0, + "size": 10, + "query": { + "match_all":{} + }, + "sort": ["_id"], + "search_before": ["d"] + }, + "result": { + "total_hits": 6, + "hits": [ + { + "id": "a" + }, + { + "id": "b" + }, + { + "id": "c" + } + ] + } + }, + { + "comment": "sort by score, after score 0.286889[ e(299646) > f(286889) > d(222224)]", + "search": { + "from": 0, + "size": 10, + "query": { + "query":"rapster" + }, + "sort": ["_score"], + "search_after": ["0.286889"] + }, + "result": { + "total_hits": 3, + "hits": [ + { + "id": "f" + }, + { + "id": "e" + } + ] + } + }, + { + "comment": "sort by score, before score f/0.286889[ e(299646) > f(286889) > d(222224)]", + "search": { + "from": 0, + "size": 10, + "query": { + "query":"rapster" + }, + "sort": ["_score"], + "search_before": ["0.286889"] + }, + "result": { + "total_hits": 3, + "hits": [ + { + "id": "d" + } + ] + } + } +] diff --git a/test/versus_score_test.go b/test/versus_score_test.go new file mode 100644 index 0000000..18c32bf --- /dev/null +++ b/test/versus_score_test.go @@ -0,0 +1,140 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "os" + "strconv" + "testing" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/document" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/index/upsidedown" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +func TestDisjunctionSearchScoreIndexWithCompositeFields(t *testing.T) { + upHits := disjunctionQueryiOnIndexWithCompositeFields(upsidedown.Name, t) + scHits := disjunctionQueryiOnIndexWithCompositeFields(scorch.Name, t) + + if upHits[0].ID != scHits[0].ID || upHits[1].ID != scHits[1].ID { + t.Errorf("upsidedown, scorch returned different docs;\n"+ + "upsidedown: (%s, %s), scorch: (%s, %s)\n", + upHits[0].ID, upHits[1].ID, scHits[0].ID, scHits[1].ID) + } + + if scHits[0].Score != upHits[0].Score || scHits[1].Score != upHits[1].Score { + t.Errorf("upsidedown, scorch showing different scores;\n"+ + "upsidedown: (%+v, %+v), scorch: (%+v, %+v)\n", + *upHits[0].Expl, *upHits[1].Expl, *scHits[0].Expl, *scHits[1].Expl) + } +} + +func disjunctionQueryiOnIndexWithCompositeFields(indexName string, + t *testing.T, +) []*search.DocumentMatch { + tmpIndexPath, err := os.MkdirTemp("", "bleve-testidx") + if err != nil { + t.Fatalf("error creating temp dir: %v", err) + } + defer func() { + err := os.RemoveAll(tmpIndexPath) + if err != nil { + t.Fatalf("error removing temp dir: %v", err) + } + }() + // create an index + idxMapping := mapping.NewIndexMapping() + idx, err := bleve.NewUsing(tmpIndexPath, idxMapping, indexName, + bleve.Config.DefaultKVStore, nil) + if err != nil { + t.Error(err) + } + + defer func() { + err = idx.Close() + if err != nil { + t.Error(err) + } + }() + + // create and insert documents as a batch + batch := idx.NewBatch() + docs := []struct { + field1 string + field2 int + }{ + { + field1: "one", + field2: 1, + }, + { + field1: "two", + field2: 2, + }, + } + + for i := 0; i < len(docs); i++ { + doc := document.NewDocument(strconv.Itoa(docs[i].field2)) + doc.Fields = []document.Field{ + document.NewTextField("field1", []uint64{}, []byte(docs[i].field1)), + document.NewNumericField("field2", []uint64{}, float64(docs[i].field2)), + } + doc.CompositeFields = []*document.CompositeField{ + document.NewCompositeFieldWithIndexingOptions( + "_all", true, []string{"field1"}, []string{}, + index.IndexField|index.IncludeTermVectors), + } + if err = batch.IndexAdvanced(doc); err != nil { + t.Error(err) + } + } + if err = idx.Batch(batch); err != nil { + t.Error(err) + } + + /* + Query: + DISJ + / \ + CONJ TERM(two) + / + TERM(one) + */ + + tq1 := bleve.NewTermQuery("one") + tq1.SetBoost(2) + tq2 := bleve.NewTermQuery("two") + tq2.SetBoost(3) + + cq := bleve.NewConjunctionQuery(tq1) + cq.SetBoost(4) + + q := bleve.NewDisjunctionQuery(tq1, tq2) + sr := bleve.NewSearchRequestOptions(q, 2, 0, true) + res, err := idx.Search(sr) + if err != nil { + t.Error(err) + } + + if len(res.Hits) != 2 { + t.Errorf("indexType: %s Expected 2 hits, but got: %v", indexName, len(res.Hits)) + } + + return res.Hits +} diff --git a/test/versus_test.go b/test/versus_test.go new file mode 100644 index 0000000..119c626 --- /dev/null +++ b/test/versus_test.go @@ -0,0 +1,519 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "bytes" + "encoding/json" + "fmt" + "math" + "math/rand" + "os" + "reflect" + "strconv" + "strings" + "testing" + "text/template" + "time" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/index/upsidedown" + "github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" +) + +// Tests scorch indexer versus upsidedown/bolt indexer against various +// templated queries. Example usage from the bleve top-level directory... +// +// go test -v -run TestScorchVersusUpsideDownBolt ./test +// VERBOSE=1 FOCUS=Trista go test -v -run TestScorchVersusUpsideDownBolt ./test +// + +func init() { + // override for tests + scorch.DefaultPersisterNapTimeMSec = 1 +} + +func TestScorchVersusUpsideDownBoltAll(t *testing.T) { + (&VersusTest{ + t: t, + NumDocs: 1000, + MaxWordsPerDoc: 20, + NumWords: 10, + BatchSize: 1000, + NumAttemptsPerSearch: 100, + }).run(scorch.Name, boltdb.Name, upsidedown.Name, boltdb.Name, nil, nil) +} + +func TestScorchVersusUpsideDownBoltSmallMNSAM(t *testing.T) { + (&VersusTest{ + t: t, + Focus: "must-not-same-as-must", + NumDocs: 5, + MaxWordsPerDoc: 2, + NumWords: 1, + BatchSize: 1, + NumAttemptsPerSearch: 1, + }).run(scorch.Name, boltdb.Name, upsidedown.Name, boltdb.Name, nil, nil) +} + +func TestScorchVersusUpsideDownBoltSmallCMP11(t *testing.T) { + (&VersusTest{ + t: t, + Focus: "conjuncts-match-phrase-1-1", + NumDocs: 30, + MaxWordsPerDoc: 8, + NumWords: 2, + BatchSize: 1, + NumAttemptsPerSearch: 1, + }).run(scorch.Name, boltdb.Name, upsidedown.Name, boltdb.Name, nil, nil) +} + +// ------------------------------------------------------- + +// Templates used to compare search results in the "versus" tests. +var testVersusSearchTemplates = []string{ + `{ + "about": "expected to return zero hits", + "query": { + "query": "title:notARealTitle" + } + }`, + `{ + "about": "try straight word()'s", + "query": { + "query": "body:{{word}}" + } + }`, + `{ + "about": "conjuncts on same term", + "query": { + "conjuncts": [ + { "field": "body", "term": "{{word}}", "boost": 1.0 }, + { "field": "body", "term": "{{word}}", "boost": 1.0 } + ] + } + }`, + `{ + "about": "disjuncts on same term", + "query": { + "disjuncts": [ + { "field": "body", "term": "{{word}}", "boost": 1.0 }, + { "field": "body", "term": "{{word}}", "boost": 1.0 } + ] + } + }`, + `{ + "about": "never-matching-title-conjuncts", + "query": { + "conjuncts": [ + {"field": "body", "match": "{{word}}"}, + {"field": "body", "match": "{{word}}"}, + {"field": "title", "match": "notAnActualTitle"} + ] + } + }`, + `{ + "about": "never-matching-title-disjuncts", + "query": { + "disjuncts": [ + {"field": "body", "match": "{{word}}"}, + {"field": "body", "match": "{{word}}"}, + {"field": "title", "match": "notAnActualTitle"} + ] + } + }`, + `{ + "about": "must-not-never-matches", + "query": { + "must_not": {"disjuncts": [ + {"field": "title", "match": "notAnActualTitle"} + ]}, + "should": {"disjuncts": [ + {"field": "body", "match": "{{word}}"} + ]} + } + }`, + `{ + "about": "must-not-only", + "query": { + "must_not": {"disjuncts": [ + {"field": "body", "term": "{{word}}"} + ]} + } + }`, + `{ + "about": "must-not-same-as-must -- see: MB-27291", + "query": { + "must_not": {"disjuncts": [ + {"field": "body", "match": "{{word}}"} + ]}, + "must": {"conjuncts": [ + {"field": "body", "match": "{{word}}"} + ]} + } + }`, + `{ + "about": "must-not-same-as-should", + "query": { + "must_not": {"disjuncts": [ + {"field": "body", "match": "{{word}}"} + ]}, + "should": {"disjuncts": [ + {"field": "body", "match": "{{word}}"} + ]} + } + }`, + `{ + "about": "inspired by testrunner RQG issue -- see: MB-27291", + "query": { + "must_not": {"disjuncts": [ + {"field": "title", "match": "Trista Allen"}, + {"field": "body", "match": "{{word}}"} + ]}, + "should": {"disjuncts": [ + {"field": "title", "match": "Kallie Safiya Amara"}, + {"field": "body", "match": "{{word}}"} + ]} + } + }`, + `{ + "about": "conjuncts-match-phrase-1-1 inspired by testrunner RQG issue -- see: MB-27291", + "query": { + "conjuncts": [ + {"field": "body", "match": "{{bodyWord 0}}"}, + {"field": "body", "match_phrase": "{{bodyWord 1}} {{bodyWord 1}}"} + ] + } + }`, + `{ + "about": "conjuncts-match-phrase-1-2 inspired by testrunner RQG issue -- see: MB-27291 -- FAILS!!", + "query": { + "conjuncts": [ + {"field": "body", "match": "{{bodyWord 0}}"}, + {"field": "body", "match_phrase": "{{bodyWord 1}} {{bodyWord 2}}"} + ] + } + }`, +} + +// ------------------------------------------------------- + +type VersusTest struct { + t *testing.T + + // Use environment variable VERBOSE= that's > 0 for more + // verbose output. + Verbose int + + // Allow user to focus on particular search templates, where + // where the search template must contain the Focus string. + Focus string + + NumDocs int // Number of docs to insert. + MaxWordsPerDoc int // Max number words in each doc's Body field. + NumWords int // Total number of words in the dictionary. + BatchSize int // Batch size when inserting docs. + NumAttemptsPerSearch int // For each search template, number of searches to try. + + // The Bodies is an array with length NumDocs, where each entry + // is the words in a doc's Body field. + Bodies [][]string + + CurAttempt int + TotAttempts int +} + +// ------------------------------------------------------- + +func testVersusSearches(vt *VersusTest, searchTemplates []string, idxA, idxB bleve.Index) { + t := vt.t + + funcMap := template.FuncMap{ + // Returns a word. The word may or may not be in any + // document's body. + "word": func() string { + return vt.genWord(vt.CurAttempt % vt.NumWords) + }, + // Picks a document and returns the i'th word in that + // document's body. You can use this in searches to + // definitely find at least one document. + "bodyWord": func(i int) string { + body := vt.Bodies[vt.CurAttempt%len(vt.Bodies)] + if len(body) == 0 { + return "" + } + return body[i%len(body)] + }, + } + + // Optionally allow call to focus on a particular search templates, + // where the search template must contain the vt.Focus string. + if vt.Focus == "" { + vt.Focus = os.Getenv("FOCUS") + } + + for i, searchTemplate := range searchTemplates { + if vt.Focus != "" && !strings.Contains(searchTemplate, vt.Focus) { + continue + } + + tmpl, err := template.New("search").Funcs(funcMap).Parse(searchTemplate) + if err != nil { + t.Fatalf("could not parse search template: %s, err: %v", searchTemplate, err) + } + for j := 0; j < vt.NumAttemptsPerSearch; j++ { + vt.CurAttempt = j + + var buf bytes.Buffer + err = tmpl.Execute(&buf, vt) + if err != nil { + t.Fatalf("could not execute search template: %s, err: %v", searchTemplate, err) + } + + bufBytes := buf.Bytes() + + if vt.Verbose > 0 { + fmt.Printf(" %s\n", bufBytes) + } + + var search bleve.SearchRequest + err = json.Unmarshal(bufBytes, &search) + if err != nil { + t.Fatalf("could not unmarshal search: %s, err: %v", bufBytes, err) + } + + search.Size = vt.NumDocs * 10 // Crank up limit to get all results. + + searchA := search + searchB := search + + resA, errA := idxA.Search(&searchA) + resB, errB := idxB.Search(&searchB) + if errA != errB { + t.Errorf("search: (%d) %s,\n err mismatch, errA: %v, errB: %v", + i, bufBytes, errA, errB) + } + + // Scores might have float64 vs float32 wobbles, so truncate precision. + resA.MaxScore = math.Trunc(resA.MaxScore*1000.0) / 1000.0 + resB.MaxScore = math.Trunc(resB.MaxScore*1000.0) / 1000.0 + + // Timings may be different between A & B, so force equality. + resA.Took = resB.Took + + // Hits might have different ordering since some indexers + // (like upsidedown) have a natural secondary sort on id + // while others (like scorch) don't. So, we compare by + // putting the hits from A & B into maps. + hitsA := hitsById(resA) + hitsB := hitsById(resB) + for id, hitA := range hitsA { + hitB := hitsB[id] + if len(hitA.FieldTermLocations) == 0 { + hitA.FieldTermLocations = nil + } + if len(hitB.FieldTermLocations) == 0 { + hitB.FieldTermLocations = nil + } + if !reflect.DeepEqual(hitA, hitB) { + t.Errorf("\n driving from hitsA\n hitA: %#v,\n hitB: %#v", hitA, hitB) + idx, _ := strconv.Atoi(id) + t.Errorf("\n doc: %d, body: %s", idx, strings.Join(vt.Bodies[idx], " ")) + } + } + for id, hitB := range hitsB { + hitA := hitsA[id] + if len(hitA.FieldTermLocations) == 0 { + hitA.FieldTermLocations = nil + } + if len(hitB.FieldTermLocations) == 0 { + hitB.FieldTermLocations = nil + } + if !reflect.DeepEqual(hitA, hitB) { + t.Errorf("\n driving from hitsB\n hitA: %#v,\n hitB: %#v", hitA, hitB) + idx, _ := strconv.Atoi(id) + t.Errorf("\n doc: %d, body: %s", idx, strings.Join(vt.Bodies[idx], " ")) + } + } + + if !reflect.DeepEqual(hitsA, hitsB) { + t.Errorf("=========\nsearch: (%d) %s,\n res hits mismatch,\n len(hitsA): %d,\n len(hitsB): %d", + i, bufBytes, len(hitsA), len(hitsB)) + t.Errorf("\n hitsA: %#v,\n hitsB: %#v", + hitsA, hitsB) + } + + resA.Hits = nil + resB.Hits = nil + resA.Cost = 0 + resB.Cost = 0 + + if !reflect.DeepEqual(resA, resB) { + resAj, _ := json.Marshal(resA) + resBj, _ := json.Marshal(resB) + t.Errorf("search: (%d) %s,\n res mismatch,\n resA: %s,\n resB: %s", + i, bufBytes, resAj, resBj) + } + + if vt.Verbose > 0 { + fmt.Printf(" Total: (%t) %d\n", resA.Total == resB.Total, resA.Total) + } + + vt.TotAttempts++ + } + } +} + +// Organizes the hits into a map keyed by id. +func hitsById(res *bleve.SearchResult) map[string]*search.DocumentMatch { + rv := make(map[string]*search.DocumentMatch, len(res.Hits)) + + for _, hit := range res.Hits { + // Clear out or truncate precision of hit fields that might be + // different across different indexer implementations. + hit.Index = "" + hit.Score = math.Trunc(hit.Score*1000.0) / 1000.0 + hit.IndexInternalID = nil + hit.HitNumber = 0 + + rv[hit.ID] = hit + } + + return rv +} + +// ------------------------------------------------------- + +func (vt *VersusTest) run(indexTypeA, kvStoreA, indexTypeB, kvStoreB string, + cb func(versusTest *VersusTest, searchTemplates []string, idxA, idxB bleve.Index), + searchTemplates []string, +) { + if cb == nil { + cb = testVersusSearches + } + + if searchTemplates == nil { + searchTemplates = testVersusSearchTemplates + } + + if vt.Verbose <= 0 { + vt.Verbose, _ = strconv.Atoi(os.Getenv("VERBOSE")) + } + + dirA := "/tmp/bleve-versus-test-a" + dirB := "/tmp/bleve-versus-test-b" + + defer func() { + _ = os.RemoveAll(dirA) + _ = os.RemoveAll(dirB) + }() + + _ = os.RemoveAll(dirA) + _ = os.RemoveAll(dirB) + + imA := vt.makeIndexMapping() + imB := vt.makeIndexMapping() + + kvConfigA := map[string]interface{}{} + kvConfigB := map[string]interface{}{} + + idxA, err := bleve.NewUsing(dirA, imA, indexTypeA, kvStoreA, kvConfigA) + if err != nil || idxA == nil { + vt.t.Fatalf("new using err: %v", err) + } + defer func() { _ = idxA.Close() }() + + idxB, err := bleve.NewUsing(dirB, imB, indexTypeB, kvStoreB, kvConfigB) + if err != nil || idxB == nil { + vt.t.Fatalf("new using err: %v", err) + } + defer func() { _ = idxB.Close() }() + + if vt.Bodies == nil { + vt.Bodies = vt.genBodies() + } + + vt.insertBodies(idxA) + vt.insertBodies(idxB) + + cb(vt, searchTemplates, idxA, idxB) +} + +// ------------------------------------------------------- + +func (vt *VersusTest) makeIndexMapping() mapping.IndexMapping { + standardFM := bleve.NewTextFieldMapping() + standardFM.Store = false + standardFM.IncludeInAll = false + standardFM.IncludeTermVectors = true + standardFM.Analyzer = "standard" + + dm := bleve.NewDocumentMapping() + dm.AddFieldMappingsAt("title", standardFM) + dm.AddFieldMappingsAt("body", standardFM) + + im := bleve.NewIndexMapping() + im.DefaultMapping = dm + im.DefaultAnalyzer = "standard" + + return im +} + +func (vt *VersusTest) insertBodies(idx bleve.Index) { + batch := idx.NewBatch() + for i, bodyWords := range vt.Bodies { + title := fmt.Sprintf("%d", i) + body := strings.Join(bodyWords, " ") + err := batch.Index(title, map[string]interface{}{"title": title, "body": body}) + if err != nil { + vt.t.Fatalf("batch.Index err: %v", err) + } + if i%vt.BatchSize == 0 { + err = idx.Batch(batch) + if err != nil { + vt.t.Fatalf("batch err: %v", err) + } + batch.Reset() + } + } + err := idx.Batch(batch) + if err != nil { + vt.t.Fatalf("last batch err: %v", err) + } +} + +func (vt *VersusTest) genBodies() (rv [][]string) { + for i := 0; i < vt.NumDocs; i++ { + rv = append(rv, vt.genBody()) + } + return rv +} + +func (vt *VersusTest) genBody() (rv []string) { + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + + m := rng.Intn(vt.MaxWordsPerDoc) + for j := 0; j < m; j++ { + rv = append(rv, vt.genWord(rng.Intn(vt.NumWords))) + } + return rv +} + +func (vt *VersusTest) genWord(i int) string { + return fmt.Sprintf("%x", i) +} diff --git a/util/bitset.go b/util/bitset.go new file mode 100644 index 0000000..418ada2 --- /dev/null +++ b/util/bitset.go @@ -0,0 +1,130 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "math/bits" + + "github.com/RoaringBitmap/roaring/v2" +) + +type Bitset struct { + data []uint64 + numBits int + exclude *roaring.Bitmap +} + +// NewBitset initializes a bitset capable of holding numbers up to maxVal +func NewBitset(maxVal int, exclude *roaring.Bitmap) *Bitset { + // We need (maxVal / 64) + 1 buckets to hold up to maxVal + size := (maxVal / 64) + 1 + return &Bitset{ + data: make([]uint64, size), + numBits: maxVal, + exclude: exclude, + } +} + +// Add inserts a value into the bitset (safely handles duplicates) +func (b *Bitset) Add(val int) { + if b.exclude != nil && b.exclude.Contains(uint32(val)) { + return + } + bucket := val >> 6 // Equivalent to val / 64 + bit := uint(val & 63) // Equivalent to val % 64 + + // Set the bit to 1 using bitwise OR + b.data[bucket] |= (1 << bit) +} + +// Remove deletes a value from the bitset +func (b *Bitset) Remove(val int) { + bucket := val >> 6 + bit := uint(val & 63) + + // Set the bit to 0 using bitwise AND with the complement + b.data[bucket] &^= (1 << bit) +} + +// Contains checks if a value exists in the bitset +func (b *Bitset) Contains(val int) bool { + bucket := val >> 6 + bit := uint(val & 63) + + return (b.data[bucket] & (1 << bit)) != 0 +} + +// Invert flips all bits in the bitset, +// effectively turning all 1s to 0s and vice versa +func (b *Bitset) Invert() { + for i := range b.data { + b.data[i] = ^b.data[i] + } + // the flip above sets the trailing bits beyond numBits in the last + // bucket(s), which do not correspond to valid values - clear them so + // that Iterate and Count never see them + lastBucket := b.numBits >> 6 + if lastBucket < len(b.data) { + b.data[lastBucket] &= (1 << uint(b.numBits&63)) - 1 + for i := lastBucket + 1; i < len(b.data); i++ { + b.data[i] = 0 + } + } + if b.exclude != nil { + it := b.exclude.Iterator() + for it.HasNext() { + bit := uint64(it.Next()) + word := bit / 64 + if word < uint64(len(b.data)) { + b.data[word] &^= uint64(1) << (bit % 64) + } + } + } +} + +// Iterate calls the provided function for every integer recorded in the bitset, in ascending order +func (b *Bitset) Iterate(f func(int)) { + for bucketIdx, bucket := range b.data { + // If the entire 64-bit block is 0, skip it entirely for speed + if bucket == 0 { + continue + } + + // Check all 64 bits in this bucket + for bitIdx := 0; bitIdx < 64; bitIdx++ { + if (bucket & (1 << uint(bitIdx))) != 0 { + // Reconstruct the original integer + originalVal := (bucketIdx << 6) + bitIdx + f(originalVal) + } + } + } +} + +func (b *Bitset) Count() int { + count := 0 + + for _, word := range b.data { + count += bits.OnesCount64(word) + } + + return count +} + +func (b *Bitset) Clear() { + for i := range b.data { + b.data[i] = 0 + } +} diff --git a/util/bitset_test.go b/util/bitset_test.go new file mode 100644 index 0000000..0d26f6d --- /dev/null +++ b/util/bitset_test.go @@ -0,0 +1,205 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "reflect" + "testing" + + "github.com/RoaringBitmap/roaring/v2" +) + +func TestBitsetAddContains(t *testing.T) { + b := NewBitset(100, nil) + + // values not yet added must not be present + for _, v := range []int{0, 1, 63, 64, 65, 100} { + if b.Contains(v) { + t.Fatalf("expected %d to be absent from an empty bitset", v) + } + } + + // exercise word boundaries explicitly: 63 is the last bit of word 0, + // 64 is the first bit of word 1, 65 the second + added := []int{0, 1, 63, 64, 65, 100} + for _, v := range added { + b.Add(v) + } + for _, v := range added { + if !b.Contains(v) { + t.Fatalf("expected %d to be present after Add", v) + } + } + + // a value between two set bits must remain absent + if b.Contains(50) { + t.Fatalf("expected 50 to be absent") + } +} + +func TestBitsetAddDuplicate(t *testing.T) { + b := NewBitset(100, nil) + b.Add(42) + b.Add(42) + if got := b.Count(); got != 1 { + t.Fatalf("expected duplicate Add to keep count at 1, got %d", got) + } + if !b.Contains(42) { + t.Fatalf("expected 42 to be present") + } +} + +func TestBitsetRemove(t *testing.T) { + b := NewBitset(100, nil) + b.Add(10) + b.Add(64) + b.Remove(10) + if b.Contains(10) { + t.Fatalf("expected 10 to be absent after Remove") + } + if !b.Contains(64) { + t.Fatalf("expected 64 to still be present") + } + // removing an absent value must be a no-op + b.Remove(99) + if b.Count() != 1 { + t.Fatalf("expected count 1 after removing an absent value, got %d", b.Count()) + } +} + +func TestBitsetCountAndClear(t *testing.T) { + b := NewBitset(200, nil) + for _, v := range []int{0, 5, 63, 64, 128, 200} { + b.Add(v) + } + if got := b.Count(); got != 6 { + t.Fatalf("expected count 6, got %d", got) + } + b.Clear() + if got := b.Count(); got != 0 { + t.Fatalf("expected count 0 after Clear, got %d", got) + } +} + +func TestBitsetIterateAscending(t *testing.T) { + b := NewBitset(200, nil) + // add out of order to confirm Iterate returns ascending order + for _, v := range []int{130, 0, 64, 63, 7} { + b.Add(v) + } + var got []int + b.Iterate(func(v int) { + got = append(got, v) + }) + want := []int{0, 7, 63, 64, 130} + if !reflect.DeepEqual(got, want) { + t.Fatalf("expected Iterate to yield %v in ascending order, got %v", want, got) + } +} + +func TestBitsetExcludeBlocksAdd(t *testing.T) { + exclude := roaring.New() + exclude.AddInt(5) + exclude.AddInt(70) + + b := NewBitset(100, exclude) + b.Add(5) // excluded, must be ignored + b.Add(6) // allowed + b.Add(70) // excluded, must be ignored + + if b.Contains(5) { + t.Fatalf("expected excluded value 5 to be blocked by Add") + } + if b.Contains(70) { + t.Fatalf("expected excluded value 70 to be blocked by Add") + } + if !b.Contains(6) { + t.Fatalf("expected non-excluded value 6 to be present") + } +} + +func TestBitsetInvertRespectsNumBits(t *testing.T) { + // numDocs = 10 means the valid doc IDs are 0..9. NewBitset allocates a + // full 64-bit word, so bits 10..63 are unused and must never surface. + numDocs := 10 + b := NewBitset(numDocs, nil) + b.Add(3) + b.Add(7) + + b.Invert() + + var got []int + b.Iterate(func(v int) { + got = append(got, v) + }) + + // after inverting, exactly the doc IDs in [0, 10) that were NOT set + // should be present - and nothing >= 10 + want := []int{0, 1, 2, 4, 5, 6, 8, 9} + if !reflect.DeepEqual(got, want) { + t.Fatalf("expected inverted bitset to yield %v, got %v", want, got) + } + if got := b.Count(); got != len(want) { + t.Fatalf("expected inverted count %d, got %d", len(want), got) + } +} + +func TestBitsetInvertWordBoundaries(t *testing.T) { + // exercise numBits at and around 64-bit word boundaries to make sure + // the trailing-bit mask is computed correctly + for _, numDocs := range []int{1, 63, 64, 65, 128, 129} { + b := NewBitset(numDocs, nil) + b.Invert() + max := -1 + b.Iterate(func(v int) { + if v > max { + max = v + } + }) + // every valid doc ID [0, numDocs) should be present after inverting + // an empty bitset, and none at or beyond numDocs + if got := b.Count(); got != numDocs { + t.Fatalf("numDocs=%d: expected inverted-empty count %d, got %d", + numDocs, numDocs, got) + } + if max >= numDocs { + t.Fatalf("numDocs=%d: Invert surfaced out-of-range value %d", + numDocs, max) + } + } +} + +func TestBitsetInvertClearsExcluded(t *testing.T) { + // excluded docs must remain unset even after Invert, since they are + // never valid hits + exclude := roaring.New() + exclude.AddInt(2) + exclude.AddInt(8) + + b := NewBitset(10, exclude) + b.Add(3) + b.Invert() + + if b.Contains(2) || b.Contains(8) { + t.Fatalf("expected excluded docs to stay unset after Invert") + } + if b.Contains(3) { + t.Fatalf("expected the originally-set doc 3 to be cleared after Invert") + } + // a normal, non-excluded, originally-unset doc should now be set + if !b.Contains(0) { + t.Fatalf("expected doc 0 to be set after Invert") + } +} diff --git a/util/bolt.go b/util/bolt.go new file mode 100644 index 0000000..d05500b --- /dev/null +++ b/util/bolt.go @@ -0,0 +1,170 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "fmt" + "os" + + bolt "go.etcd.io/bbolt" +) + +// All of the bolt impls provide a layer of indirection to allow for processing +// of values as they are read/written to bolt depending on the key or bucket name +// This is used to allow better support for file callbacks + +// wrapper around bolt.DB +type RootBoltImpl struct { + *bolt.DB +} + +// wrapper around bolt.Tx +type BoltTxImpl struct { + *bolt.Tx +} + +// wrapper around bolt.Bucket +type BoltBucketImpl struct { + *bolt.Bucket + + name string // store the name of the bucket during creation +} + +func OpenBolt(path string, mode os.FileMode, options *bolt.Options) (*RootBoltImpl, error) { + db, err := bolt.Open(path, mode, options) + if err != nil { + return nil, err + } + return &RootBoltImpl{DB: db}, nil +} + +func (r *RootBoltImpl) Begin(writable bool) (*BoltTxImpl, error) { + tx, err := r.DB.Begin(writable) + if err != nil { + return nil, err + } + return &BoltTxImpl{Tx: tx}, nil +} + +func (r *RootBoltImpl) View(fn func(*BoltTxImpl) error) error { + return r.DB.View(func(tx *bolt.Tx) error { + return fn(&BoltTxImpl{Tx: tx}) + }) +} + +func (r *RootBoltImpl) Update(fn func(*BoltTxImpl) error) error { + return r.DB.Update(func(tx *bolt.Tx) error { + return fn(&BoltTxImpl{Tx: tx}) + }) +} + +func (tx *BoltTxImpl) CreateBucketIfNotExists(name []byte) (*BoltBucketImpl, error) { + bucket, err := tx.Tx.CreateBucketIfNotExists(name) + if err != nil { + return nil, err + } + return &BoltBucketImpl{ + name: string(name), + Bucket: bucket, + }, nil +} + +func (tx *BoltTxImpl) Bucket(name []byte) *BoltBucketImpl { + bucket := tx.Tx.Bucket(name) + if bucket == nil { + return nil + } + return &BoltBucketImpl{ + name: string(name), + Bucket: bucket, + } +} + +func (b *BoltBucketImpl) GetBucket(name []byte) *BoltBucketImpl { + bucket := b.Bucket.Bucket(name) + if bucket == nil { + return nil + } + return &BoltBucketImpl{ + name: string(name), + Bucket: bucket, + } +} + +func (b *BoltBucketImpl) CreateBucketIfNotExists(name []byte) (*BoltBucketImpl, error) { + bucket, err := b.Bucket.CreateBucketIfNotExists(name) + if err != nil { + return nil, err + } + return &BoltBucketImpl{ + name: string(name), + Bucket: bucket, + }, nil +} + +// Process values during ForEach if the bucket name or key is in the boltKeysProcessed map +func (b *BoltBucketImpl) ForEach(fn func(key []byte, value []byte) error, reader FileReader) error { + _, ok1 := boltKeysProcessed[b.name] + return b.Bucket.ForEach(func(k, v []byte) error { + v = append([]byte(nil), v...) + if _, ok2 := boltKeysProcessed[string(k)]; ok1 || ok2 { + if reader == nil { + return fmt.Errorf("reader callback is required for bucket %s", b.name) + } + processedValue, err := reader.Process(v) + if err != nil { + return err + } + return fn(k, processedValue) + } + return fn(k, v) + }) +} + +// Process values during Put/Get if the bucket name or key is in the boltKeysProcessed map +func (b *BoltBucketImpl) Put(key []byte, value []byte, writer FileWriter) error { + _, ok1 := boltKeysProcessed[string(key)] + _, ok2 := boltKeysProcessed[b.name] + value = append([]byte(nil), value...) + if ok1 || ok2 { + if writer == nil { + return fmt.Errorf("writer callback is required for key %s", string(key)) + } + processedValue := writer.Process(value) + return b.Bucket.Put(key, processedValue) + } + return b.Bucket.Put(key, value) +} + +// Process values during Put/Get if the bucket name or key is in the boltKeysProcessed map +func (b *BoltBucketImpl) Get(key []byte, reader FileReader) ([]byte, error) { + _, ok1 := boltKeysProcessed[string(key)] + _, ok2 := boltKeysProcessed[b.name] + if ok1 || ok2 { + if reader == nil { + return nil, fmt.Errorf("reader callback is required for key %s", string(key)) + } + val := b.Bucket.Get(key) + if val == nil { + return nil, nil + } + processedVal, err := reader.Process(val) + if err != nil { + return nil, err + } + return processedVal, nil + } + return b.Bucket.Get(key), nil +} diff --git a/util/extract.go b/util/extract.go new file mode 100644 index 0000000..0d3decf --- /dev/null +++ b/util/extract.go @@ -0,0 +1,66 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "math" + "reflect" +) + +// extract numeric value (if possible) and returns a float64 +func ExtractNumericValFloat64(v interface{}) (float64, bool) { + val := reflect.ValueOf(v) + if !val.IsValid() { + return 0, false + } + + switch { + case val.CanFloat(): + return val.Float(), true + case val.CanInt(): + return float64(val.Int()), true + case val.CanUint(): + return float64(val.Uint()), true + } + + return 0, false +} + +// extract numeric value (if possible) and returns a float32 +func ExtractNumericValFloat32(v interface{}) (float32, bool) { + val := reflect.ValueOf(v) + if !val.IsValid() { + return 0, false + } + + switch { + case val.CanFloat(): + floatVal := val.Float() + if !IsValidFloat32(floatVal) { + return 0, false + } + return float32(floatVal), true + case val.CanInt(): + return float32(val.Int()), true + case val.CanUint(): + return float32(val.Uint()), true + } + + return 0, false +} + +func IsValidFloat32(val float64) bool { + return !math.IsNaN(val) && !math.IsInf(val, 0) && val <= math.MaxFloat32 +} diff --git a/util/file_callbacks.go b/util/file_callbacks.go new file mode 100644 index 0000000..c485006 --- /dev/null +++ b/util/file_callbacks.go @@ -0,0 +1,129 @@ +// Copyright (c) 2026 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "fmt" + + index "github.com/blevesearch/bleve_index_api" +) + +// This file provides a mechanism for users of bleve to provide callbacks +// that can process data before it is written to disk, and after it is read +// from disk. This can be used for things like encryption, compression, etc. + +// The user is responsible for ensuring that the writer and reader callbacks +// are compatible with each other, and that any state needed by the callbacks +// is managed appropriately. For example, if the writer callback uses a +// unique key or nonce per write, the reader callback must be able to +// determine the correct key or nonce to use for each read. + +// The callbacks are identified by an id string, which is returned by the +// WriterHook. The same id string is passed to the ReaderHook when creating a reader. +// This allows the reader to determine which callback to use for a given file. + +// Support for identifying all callbacks used by a given index and to remove +// selected callbacks associated with ids is provided via index.WriterIdsInUse() +// and index.DropWriterIds(). + +const DefaultFileCallbackId = "" + +// FileWriter and FileReader interfaces are wrappers around the callback functions +// provided by the user. They provide a convenient way to apply the callbacks to data +// being written to or read from a file. They also store the id the callbacks, +// which can be useful for managing state across multiple reads and writes. +type FileWriter interface { + Process(data []byte) []byte + Id() string +} +type fileWriterImpl struct { + id string + processor func(data []byte) []byte +} + +func NewFileWriter(context []byte) (FileWriter, error) { + rv := &fileWriterImpl{} + + if index.WriterHook != nil { + var err error + rv.id, rv.processor, err = index.WriterHook(context) + if err != nil { + return nil, err + } + } + + return rv, nil +} + +func (w *fileWriterImpl) Process(data []byte) []byte { + if w.processor != nil { + return w.processor(data) + } + return data +} + +func (w *fileWriterImpl) Id() string { + return w.id +} + +type FileReader interface { + Process(data []byte) ([]byte, error) + Id() string +} + +type fileReaderImpl struct { + id string + processor func(data []byte) ([]byte, error) +} + +func NewFileReader(id string, context []byte) (FileReader, error) { + rv := &fileReaderImpl{ + id: id, + } + + if index.ReaderHook != nil { + var err error + rv.processor, err = index.ReaderHook(id, context) + if err != nil { + return nil, err + } + } else if id != "" { + return nil, fmt.Errorf("reader callback id %s provided but no ReaderHook is set", id) + } + + return rv, nil +} + +func (r *fileReaderImpl) Process(data []byte) ([]byte, error) { + if r.processor != nil { + return r.processor(data) + } + return data, nil +} + +func (r *fileReaderImpl) Id() string { + return r.id +} + +// ----------------------------------------------------------------------- + +// set of bolt keys and bucket names that require processing by the reader +// and writer callbacks. +var boltKeysProcessed = map[string]struct{}{ + string(BoltDeletedKey): {}, + string(BoltInternalKey): {}, + string(BoltStatsKey): {}, + string(BoltUpdatedFieldsKey): {}, +} diff --git a/util/json.go b/util/json.go new file mode 100644 index 0000000..8745dd8 --- /dev/null +++ b/util/json.go @@ -0,0 +1,25 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "encoding/json" +) + +// Should only be overwritten during process init()'ialization. +var ( + MarshalJSON = json.Marshal + UnmarshalJSON = json.Unmarshal +) diff --git a/util/keys.go b/util/keys.go new file mode 100644 index 0000000..fb321eb --- /dev/null +++ b/util/keys.go @@ -0,0 +1,36 @@ +// Copyright (c) 2025 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +var ( + // Bolt keys + BoltSnapshotsBucket = []byte{'s'} + BoltTrainerKey = []byte{'t'} + BoltTrainCompleteKey = []byte{'c'} + BoltTrainedSamplesKey = []byte{'n'} + BoltPathKey = []byte{'p'} + BoltDeletedKey = []byte{'d'} + BoltInternalKey = []byte{'i'} + BoltMetaDataKey = []byte{'m'} + BoltMetaDataSegmentTypeKey = []byte("type") + BoltMetaDataSegmentVersionKey = []byte("version") + BoltMetaDataTimeStamp = []byte("timeStamp") + BoltStatsKey = []byte("stats") + BoltUpdatedFieldsKey = []byte("fields") + TotBytesWrittenKey = []byte("TotBytesWritten") + BoltMetaDataFileWriterIDKey = []byte("fileWriterID") + + MappingInternalKey = []byte("_mapping") +)