upstream-corteza/server/pkg/envoyx/csv/decoder_test.go

package csv

import (
	"io"
	"strings"
	"testing"

	"github.com/cortezaproject/corteza/server/pkg/envoyx/datasource"
	"github.com/stretchr/testify/require"
)

func TestDecoder(t *testing.T) {
	req := require.New(t)

	t.Run("init & meta", func(t *testing.T) {
		dc, err := Decoder(testReader(), "test.csv", nil)
		req.NoError(err)

		req.Equal(uint64(3), dc.Count())
	})

	t.Run("fields", func(t *testing.T) {
		dc, err := Decoder(testReader(), "test.csv", nil)
		req.NoError(err)

		hh := dc.Fields()
		req.Contains(hh, "f1")
		req.Contains(hh, "f2")
		req.Contains(hh, "f3")
	})

	t.Run("iterate", func(t *testing.T) {
		dc, err := Decoder(testReader(), "test.csv", nil)
		req.NoError(err)

		aux := make(datasource.RawRecord)
		var more bool

		more, err = dc.Next(nil, aux)
		req.NoError(err)
		req.True(more)
		req.Equal("r1f1", aux["f1"].Values[0])
		req.Equal("r1f2", aux["f2"].Values[0])
		req.Equal("r1f3", aux["f3"].Values[0])

		more, err = dc.Next(nil, aux)
		req.NoError(err)
		req.True(more)
		req.Equal("r2f1", aux["f1"].Values[0])
		req.Equal("r2f2", aux["f2"].Values[0])
		req.Equal("r2f3", aux["f3"].Values[0])

		more, err = dc.Next(nil, aux)
		req.NoError(err)
		req.True(more)
		req.Equal("r3f1", aux["f1"].Values[0])
		req.Equal("r3f2", aux["f2"].Values[0])
		req.Equal("r3f3", aux["f3"].Values[0])

		more, err = dc.Next(nil, aux)
		req.NoError(err)
		req.False(more)
	})
}

func TestParseComplexCSVCell(t *testing.T) {
	t.Run("plain value", func(t *testing.T) {
		d := &decoder{}

		out := d.parseComplexCSVCell("cell")
		require.Equal(t, []string{"cell"}, out)
	})

	t.Run("multi value without brackets", func(t *testing.T) {
		d := &decoder{
			multiValueDelimiter: ";",
		}

		out := d.parseComplexCSVCell("v1;v2")
		require.Equal(t, []string{"v1", "v2"}, out)
	})

	t.Run("multi value with brackets", func(t *testing.T) {
		d := &decoder{
			multiValueDelimiter: ";",
			multiValueBrackets:  true,
		}

		out := d.parseComplexCSVCell("[v1;v2]")
		require.Equal(t, []string{"v1", "v2"}, out)
	})

	t.Run("multi value wrong delimiter", func(t *testing.T) {
		d := &decoder{
			multiValueDelimiter: ",",
			multiValueBrackets:  true,
		}

		out := d.parseComplexCSVCell("[v1;v2]")
		require.Equal(t, []string{"v1;v2"}, out)
	})

	t.Run("json", func(t *testing.T) {
		d := &decoder{}

		out := d.parseComplexCSVCell("{}")
		require.Equal(t, []string{"{}"}, out)
	})

	t.Run("multi value json", func(t *testing.T) {
		d := &decoder{
			multiValueDelimiter: ";",
			multiValueBrackets:  true,
		}

		out := d.parseComplexCSVCell("[{};{}]")
		require.Equal(t, []string{"{}", "{}"}, out)
	})

	t.Run("single value geometry json", func(t *testing.T) {
		d := &decoder{
			multiValueDelimiter: ";",
		}

		out := d.parseComplexCSVCell(`{"coordinates":[47.754098,1.9335938]}`)
		require.Equal(t, []string{`{"coordinates":[47.754098,1.9335938]}`}, out)
	})

	t.Run("multi value geometry json without brackets", func(t *testing.T) {
		d := &decoder{
			multiValueDelimiter: ";",
		}

		out := d.parseComplexCSVCell(`{"coordinates":[47.754098,1.9335938]};{"coordinates":[9.7956776,23.3789063]}`)
		require.Equal(t, []string{
			`{"coordinates":[47.754098,1.9335938]}`,
			`{"coordinates":[9.7956776,23.3789063]}`,
		}, out)
	})

	t.Run("multi value geometry json with brackets", func(t *testing.T) {
		d := &decoder{
			multiValueDelimiter: ";",
			multiValueBrackets:  true,
		}

		out := d.parseComplexCSVCell(`[{"coordinates":[47.754098,1.9335938]};{"coordinates":[9.7956776,23.3789063]}]`)
		require.Equal(t, []string{
			`{"coordinates":[47.754098,1.9335938]}`,
			`{"coordinates":[9.7956776,23.3789063]}`,
		}, out)
	})

	t.Run("single value geometry json with comma delimiter", func(t *testing.T) {
		d := &decoder{
			multiValueDelimiter: ",",
		}

		// commas inside the JSON must not split the value
		out := d.parseComplexCSVCell(`{"coordinates":[47.754098,1.9335938]}`)
		require.Equal(t, []string{`{"coordinates":[47.754098,1.9335938]}`}, out)
	})

	t.Run("single value json with nested objects", func(t *testing.T) {
		d := &decoder{
			multiValueDelimiter: ",",
		}

		// "},{" inside the JSON must not split the value
		out := d.parseComplexCSVCell(`{"points":[{"x":1},{"y":2}]}`)
		require.Equal(t, []string{`{"points":[{"x":1},{"y":2}]}`}, out)
	})

	t.Run("multi value json with nested objects and braces in strings", func(t *testing.T) {
		d := &decoder{
			multiValueDelimiter: ";",
		}

		out := d.parseComplexCSVCell(`{"a":[{"x":1},{"y":2}],"s":"};{"};{"b":2}`)
		require.Equal(t, []string{
			`{"a":[{"x":1},{"y":2}],"s":"};{"}`,
			`{"b":2}`,
		}, out)
	})

	t.Run("multi value json split is delimiter agnostic", func(t *testing.T) {
		// exports default to ";" while an unconfigured decoder falls back to
		// ","; complete-object splitting covers both
		d := &decoder{}

		out := d.parseComplexCSVCell(`{"a":1};{"b":2}`)
		require.Equal(t, []string{`{"a":1}`, `{"b":2}`}, out)

		out = d.parseComplexCSVCell(`{"a":1},{"b":2}`)
		require.Equal(t, []string{`{"a":1}`, `{"b":2}`}, out)
	})

	t.Run("malformed json cell returned whole", func(t *testing.T) {
		d := &decoder{
			multiValueDelimiter: ";",
		}

		out := d.parseComplexCSVCell(`{"a":{1};{"b"}`)
		require.Equal(t, []string{`{"a":{1};{"b"}`}, out)
	})
}

func testReader() io.Reader {
	src := `f1,f2,f3
r1f1,r1f2,r1f3
r2f1,r2f2,r2f3
r3f1,r3f2,r3f3`

	return strings.NewReader(src)
}