snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/pkg/envoyx/csv/decoder_test.go

@@ -0,0 +1,215 @@
+package csv
+
+import (
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoyx/datasource"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDecoder(t *testing.T) {
+	req := require.New(t)
+
+	t.Run("init & meta", func(t *testing.T) {
+		dc, err := Decoder(testReader(), "test.csv", nil)
+		req.NoError(err)
+
+		req.Equal(uint64(3), dc.Count())
+	})
+
+	t.Run("fields", func(t *testing.T) {
+		dc, err := Decoder(testReader(), "test.csv", nil)
+		req.NoError(err)
+
+		hh := dc.Fields()
+		req.Contains(hh, "f1")
+		req.Contains(hh, "f2")
+		req.Contains(hh, "f3")
+	})
+
+	t.Run("iterate", func(t *testing.T) {
+		dc, err := Decoder(testReader(), "test.csv", nil)
+		req.NoError(err)
+
+		aux := make(datasource.RawRecord)
+		var more bool
+
+		more, err = dc.Next(nil, aux)
+		req.NoError(err)
+		req.True(more)
+		req.Equal("r1f1", aux["f1"].Values[0])
+		req.Equal("r1f2", aux["f2"].Values[0])
+		req.Equal("r1f3", aux["f3"].Values[0])
+
+		more, err = dc.Next(nil, aux)
+		req.NoError(err)
+		req.True(more)
+		req.Equal("r2f1", aux["f1"].Values[0])
+		req.Equal("r2f2", aux["f2"].Values[0])
+		req.Equal("r2f3", aux["f3"].Values[0])
+
+		more, err = dc.Next(nil, aux)
+		req.NoError(err)
+		req.True(more)
+		req.Equal("r3f1", aux["f1"].Values[0])
+		req.Equal("r3f2", aux["f2"].Values[0])
+		req.Equal("r3f3", aux["f3"].Values[0])
+
+		more, err = dc.Next(nil, aux)
+		req.NoError(err)
+		req.False(more)
+	})
+}
+
+func TestParseComplexCSVCell(t *testing.T) {
+	t.Run("plain value", func(t *testing.T) {
+		d := &decoder{}
+
+		out := d.parseComplexCSVCell("cell")
+		require.Equal(t, []string{"cell"}, out)
+	})
+
+	t.Run("multi value without brackets", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ";",
+		}
+
+		out := d.parseComplexCSVCell("v1;v2")
+		require.Equal(t, []string{"v1", "v2"}, out)
+	})
+
+	t.Run("multi value with brackets", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ";",
+			multiValueBrackets:  true,
+		}
+
+		out := d.parseComplexCSVCell("[v1;v2]")
+		require.Equal(t, []string{"v1", "v2"}, out)
+	})
+
+	t.Run("multi value wrong delimiter", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ",",
+			multiValueBrackets:  true,
+		}
+
+		out := d.parseComplexCSVCell("[v1;v2]")
+		require.Equal(t, []string{"v1;v2"}, out)
+	})
+
+	t.Run("json", func(t *testing.T) {
+		d := &decoder{}
+
+		out := d.parseComplexCSVCell("{}")
+		require.Equal(t, []string{"{}"}, out)
+	})
+
+	t.Run("multi value json", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ";",
+			multiValueBrackets:  true,
+		}
+
+		out := d.parseComplexCSVCell("[{};{}]")
+		require.Equal(t, []string{"{}", "{}"}, out)
+	})
+
+	t.Run("single value geometry json", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ";",
+		}
+
+		out := d.parseComplexCSVCell(`{"coordinates":[47.754098,1.9335938]}`)
+		require.Equal(t, []string{`{"coordinates":[47.754098,1.9335938]}`}, out)
+	})
+
+	t.Run("multi value geometry json without brackets", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ";",
+		}
+
+		out := d.parseComplexCSVCell(`{"coordinates":[47.754098,1.9335938]};{"coordinates":[9.7956776,23.3789063]}`)
+		require.Equal(t, []string{
+			`{"coordinates":[47.754098,1.9335938]}`,
+			`{"coordinates":[9.7956776,23.3789063]}`,
+		}, out)
+	})
+
+	t.Run("multi value geometry json with brackets", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ";",
+			multiValueBrackets:  true,
+		}
+
+		out := d.parseComplexCSVCell(`[{"coordinates":[47.754098,1.9335938]};{"coordinates":[9.7956776,23.3789063]}]`)
+		require.Equal(t, []string{
+			`{"coordinates":[47.754098,1.9335938]}`,
+			`{"coordinates":[9.7956776,23.3789063]}`,
+		}, out)
+	})
+
+	t.Run("single value geometry json with comma delimiter", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ",",
+		}
+
+		// commas inside the JSON must not split the value
+		out := d.parseComplexCSVCell(`{"coordinates":[47.754098,1.9335938]}`)
+		require.Equal(t, []string{`{"coordinates":[47.754098,1.9335938]}`}, out)
+	})
+
+	t.Run("single value json with nested objects", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ",",
+		}
+
+		// "},{" inside the JSON must not split the value
+		out := d.parseComplexCSVCell(`{"points":[{"x":1},{"y":2}]}`)
+		require.Equal(t, []string{`{"points":[{"x":1},{"y":2}]}`}, out)
+	})
+
+	t.Run("multi value json with nested objects and braces in strings", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ";",
+		}
+
+		out := d.parseComplexCSVCell(`{"a":[{"x":1},{"y":2}],"s":"};{"};{"b":2}`)
+		require.Equal(t, []string{
+			`{"a":[{"x":1},{"y":2}],"s":"};{"}`,
+			`{"b":2}`,
+		}, out)
+	})
+
+	t.Run("multi value json split is delimiter agnostic", func(t *testing.T) {
+		// exports default to ";" while an unconfigured decoder falls back to
+		// ","; complete-object splitting covers both
+		d := &decoder{}
+
+		out := d.parseComplexCSVCell(`{"a":1};{"b":2}`)
+		require.Equal(t, []string{`{"a":1}`, `{"b":2}`}, out)
+
+		out = d.parseComplexCSVCell(`{"a":1},{"b":2}`)
+		require.Equal(t, []string{`{"a":1}`, `{"b":2}`}, out)
+	})
+
+	t.Run("malformed json cell returned whole", func(t *testing.T) {
+		d := &decoder{
+			multiValueDelimiter: ";",
+		}
+
+		out := d.parseComplexCSVCell(`{"a":{1};{"b"}`)
+		require.Equal(t, []string{`{"a":{1};{"b"}`}, out)
+	})
+}
+
+func testReader() io.Reader {
+	src := `f1,f2,f3
+r1f1,r1f2,r1f3
+r2f1,r2f2,r2f3
+r3f1,r3f2,r3f3`
+
+	return strings.NewReader(src)
+}