snapshot of https://github.com/blevesearch/bleve.git (master, 2026-09-15; blobs > 131072B omitted)

analysis/char/asciifolding/asciifolding_test.go

@@ -0,0 +1,124 @@
+//  Copyright (c) 2018 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package asciifolding
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+func TestAsciiFoldingFilter(t *testing.T) {
+	tests := []struct {
+		input  []byte
+		output []byte
+	}{
+		{
+			// empty input passes
+			input:  []byte(``),
+			output: []byte(``),
+		},
+		{
+			// no modification for plain ASCII
+			input:  []byte(`The quick brown fox jumps over the lazy dog`),
+			output: []byte(`The quick brown fox jumps over the lazy dog`),
+		},
+		{
+			// Umlauts are folded to plain ASCII
+			input:  []byte(`The quick bröwn fox jümps over the läzy dog`),
+			output: []byte(`The quick brown fox jumps over the lazy dog`),
+		},
+		{
+			// composite unicode runes are folded to more than one ASCII rune
+			input:  []byte(`ÆꜴ`),
+			output: []byte(`AEAO`),
+		},
+		{
+			// apples from https://issues.couchbase.com/browse/MB-33486
+			input:  []byte(`Ápple Àpple Äpple Âpple Ãpple Åpple`),
+			output: []byte(`Apple Apple Apple Apple Apple Apple`),
+		},
+		{
+			// Fix ASCII folding of \u24A2
+			input:  []byte(`⒢`),
+			output: []byte(`(g)`),
+		},
+		{
+			// Test folding of \u2053 (SWUNG DASH)
+			input:  []byte(`a⁓b`),
+			output: []byte(`a~b`),
+		},
+		{
+			// Test folding of \uFF5E (FULLWIDTH TILDE)
+			input:  []byte(`c～d`),
+			output: []byte(`c~d`),
+		},
+		{
+			// Test folding of \uFF3F (FULLWIDTH LOW LINE) - case before tilde
+			input:  []byte(`e＿f`),
+			output: []byte(`e_f`),
+		},
+		{
+			// Test mix including tilde and default fallthrough (using a character not explicitly folded)
+			input:  []byte(`a⁓b✅c～d`),
+			output: []byte(`a~b✅c~d`),
+		},
+		{
+			// Test start of 'A' fallthrough block
+			input:  []byte(`ÀBC`),
+			output: []byte(`ABC`),
+		},
+		{
+			// Test end of 'A' fallthrough block
+			input:  []byte(`DEFẶ`),
+			output: []byte(`DEFA`),
+		},
+		{
+			// Test start of 'AE' fallthrough block
+			input:  []byte(`Æ`),
+			output: []byte(`AE`),
+		},
+		{
+			// Test end of 'AE' fallthrough block
+			input:  []byte(`ᴁ`),
+			output: []byte(`AE`),
+		},
+		{
+			// Test 'DZ' multi-rune output
+			input:  []byte(`DŽebra`),
+			output: []byte(`DZebra`),
+		},
+		{
+			// Test start of 'a' fallthrough block
+			input:  []byte(`àbc`),
+			output: []byte(`abc`),
+		},
+		{
+			// Test end of 'a' fallthrough block
+			input:  []byte(`defａ`),
+			output: []byte(`defa`),
+		},
+	}
+
+	for _, test := range tests {
+		filter := New()
+		t.Run(fmt.Sprintf("on %s", test.input), func(t *testing.T) {
+			output := filter.Filter(test.input)
+			if !reflect.DeepEqual(output, test.output) {
+				t.Errorf("\nExpected:\n`%s`\ngot:\n`%s`\n", string(test.output), string(output))
+			}
+		})
+	}
+}

analysis/char/html/html.go

@@ -0,0 +1,57 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package html
+
+import (
+	"bytes"
+	"regexp"
+
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/registry"
+)
+
+const Name = "html"
+
+var htmlCharFilterRegexp = regexp.MustCompile(`</?[!\w]+((\s+\w+(\s*=\s*(?:".*?"|'.*?'|[^'">\s]+))?)+\s*|\s*)/?>`)
+
+type CharFilter struct {
+	r           *regexp.Regexp
+	replacement []byte
+}
+
+func New() *CharFilter {
+	return &CharFilter{
+		r:           htmlCharFilterRegexp,
+		replacement: []byte(" "),
+	}
+}
+
+func (s *CharFilter) Filter(input []byte) []byte {
+	return s.r.ReplaceAllFunc(
+		input, func(in []byte) []byte {
+			return bytes.Repeat(s.replacement, len(in))
+		})
+}
+
+func CharFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.CharFilter, error) {
+	return New(), nil
+}
+
+func init() {
+	err := registry.RegisterCharFilter(Name, CharFilterConstructor)
+	if err != nil {
+		panic(err)
+	}
+}

analysis/char/regexp/regexp.go

@@ -0,0 +1,65 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package regexp
+
+import (
+	"fmt"
+	"regexp"
+
+	"github.com/blevesearch/bleve/v2/analysis"
+	"github.com/blevesearch/bleve/v2/registry"
+)
+
+const Name = "regexp"
+
+type CharFilter struct {
+	r           *regexp.Regexp
+	replacement []byte
+}
+
+func New(r *regexp.Regexp, replacement []byte) *CharFilter {
+	return &CharFilter{
+		r:           r,
+		replacement: replacement,
+	}
+}
+
+func (s *CharFilter) Filter(input []byte) []byte {
+	return s.r.ReplaceAll(input, s.replacement)
+}
+
+func CharFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.CharFilter, error) {
+	regexpStr, ok := config["regexp"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify regexp")
+	}
+	r, err := regexp.Compile(regexpStr)
+	if err != nil {
+		return nil, fmt.Errorf("unable to build regexp char filter: %v", err)
+	}
+	replaceBytes := []byte(" ")
+	replaceStr, ok := config["replace"].(string)
+	if ok {
+		replaceBytes = []byte(replaceStr)
+	}
+	return New(r, replaceBytes), nil
+}
+
+func init() {
+	err := registry.RegisterCharFilter(Name, CharFilterConstructor)
+	if err != nil {
+		panic(err)
+	}
+}

analysis/char/regexp/regexp_test.go

@@ -0,0 +1,88 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package regexp
+
+import (
+	"fmt"
+	"reflect"
+	"regexp"
+	"testing"
+)
+
+func TestRegexpCharFilter(t *testing.T) {
+
+	tests := []struct {
+		regexStr string
+		replace  []byte
+		input    []byte
+		output   []byte
+	}{
+		{
+			regexStr: `</?[!\w]+((\s+\w+(\s*=\s*(?:".*?"|'.*?'|[^'">\s]+))?)+\s*|\s*)/?>`,
+			replace:  []byte{' '},
+			input:    []byte(`<html>test</html>`),
+			output:   []byte(` test `),
+		},
+		{
+			regexStr: `\x{200C}`,
+			replace:  []byte{' '},
+			input:    []byte("water\u200Cunder\u200Cthe\u200Cbridge"),
+			output:   []byte("water under the bridge"),
+		},
+		{
+			regexStr: `([a-z])\s+(\d)`,
+			replace:  []byte(`$1-$2`),
+			input:    []byte(`temp 1`),
+			output:   []byte(`temp-1`),
+		},
+		{
+			regexStr: `foo.?`,
+			replace:  []byte(`X`),
+			input:    []byte(`seafood, fool`),
+			output:   []byte(`seaX, X`),
+		},
+		{
+			regexStr: `def`,
+			replace:  []byte(`_`),
+			input:    []byte(`abcdefghi`),
+			output:   []byte(`abc_ghi`),
+		},
+		{
+			regexStr: `456`,
+			replace:  []byte(`000000`),
+			input:    []byte(`123456789`),
+			output:   []byte(`123000000789`),
+		},
+		{
+			regexStr: `“|”`,
+			replace:  []byte(`"`),
+			input:    []byte(`“hello”`),
+			output:   []byte(`"hello"`),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("match %s replace %s", test.regexStr, string(test.replace)), func(t *testing.T) {
+			regex := regexp.MustCompile(test.regexStr)
+			filter := New(regex, test.replace)
+
+			output := filter.Filter(test.input)
+			if !reflect.DeepEqual(test.output, output) {
+				t.Errorf("Expected: `%s`, Got: `%s`\n", string(test.output), string(output))
+			}
+		})
+
+	}
+}

analysis/char/zerowidthnonjoiner/zerowidthnonjoiner.go

@@ -0,0 +1,39 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package zerowidthnonjoiner
+
+import (
+	"regexp"
+
+	"github.com/blevesearch/bleve/v2/analysis"
+	regexpCharFilter "github.com/blevesearch/bleve/v2/analysis/char/regexp"
+	"github.com/blevesearch/bleve/v2/registry"
+)
+
+const Name = "zero_width_spaces"
+
+var zeroWidthNonJoinerRegexp = regexp.MustCompile(`\x{200C}`)
+
+func CharFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.CharFilter, error) {
+	replaceBytes := []byte(" ")
+	return regexpCharFilter.New(zeroWidthNonJoinerRegexp, replaceBytes), nil
+}
+
+func init() {
+	err := registry.RegisterCharFilter(Name, CharFilterConstructor)
+	if err != nil {
+		panic(err)
+	}
+}