snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/pkg/envoyx/json/decoder.go

@@ -0,0 +1,170 @@
+package json
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoyx/datasource"
+	"github.com/gabriel-vasile/mimetype"
+)
+
+type (
+	decoder struct {
+		ident string
+
+		src    *os.File
+		reader *json.Decoder
+
+		header []string
+		count  uint64
+	}
+)
+
+func CanDecodeFile(f io.Reader) bool {
+	m, err := mimetype.DetectReader(f)
+	if err != nil {
+		return false
+	}
+
+	return CanDecodeExt(m.Extension())
+}
+
+func CanDecodeMime(m string) bool {
+	return m == "application/json" || m == "application/jsonlines"
+}
+
+func CanDecodeExt(ext string) bool {
+	pt := strings.Split(ext, ".")
+	ext = strings.TrimSpace(pt[len(pt)-1])
+	return ext == "jsonl" || ext == "json" || ext == "ndjson"
+}
+
+// Decoder inits a new csv decoder from the given reader
+//
+// @todo hold small files in mem to avoid needles disc access
+func Decoder(r io.Reader, ident string) (out *decoder, err error) {
+	out = &decoder{
+		ident: ident,
+	}
+
+	out.src, err = ioutil.TempFile(os.TempDir(), "*.ndjson")
+	if err != nil {
+		return
+	}
+
+	err = out.flushTemp(r)
+	defer out.src.Seek(0, 0)
+	if err != nil {
+		return
+	}
+
+	out.reader = json.NewDecoder(out.src)
+
+	seenHeader := make(map[string]bool)
+
+	var aux datasource.RawRecord
+	for out.reader.More() {
+		err = out.reader.Decode(&aux)
+		if err == io.EOF {
+			return out, nil
+		} else if err != nil {
+			return
+		}
+
+		for f := range aux {
+			if seenHeader[f] {
+				continue
+			}
+			seenHeader[f] = true
+
+			out.header = append(out.header, f)
+		}
+
+		// Entry count
+		out.count++
+	}
+
+	return
+}
+
+func (d *decoder) SetConfigs(config map[string]any) (err error) {
+	// @todo when needed
+	return
+}
+
+// Cleanup should be called before we stop using the decoder
+func (d *decoder) Cleanup() error {
+	return os.Remove(d.src.Name())
+}
+
+// SetIdent overwrites the system defined identifier
+func (d *decoder) SetIdent(ident string) {
+	d.ident = ident
+}
+
+// Ident returns the assigned identifier
+func (d *decoder) Ident() string {
+	return d.ident
+}
+
+// Fields returns every available field in this dataset
+func (d *decoder) Fields() []string {
+	return d.header
+}
+
+// Reset resets the decoder to the start
+func (d *decoder) Reset(_ context.Context) error {
+	_, err := d.src.Seek(0, 0)
+	if err != nil {
+		return err
+	}
+
+	d.reader = json.NewDecoder(d.src)
+	return nil
+}
+
+// Next returns the field: value mapping for the next row
+func (d *decoder) Next(_ context.Context, out datasource.RawRecord) (more bool, err error) {
+	tmp := make(datasource.RawRecord)
+	err = d.reader.Decode(&tmp)
+
+	if err == io.EOF {
+		return false, nil
+	} else if err != nil {
+		return false, err
+	}
+
+	// Empty out missing fields to keep consistent with CSV
+	for _, h := range d.header {
+		if _, ok := tmp[h]; !ok {
+			vv := tmp[h]
+			vv.Values = make([]string, 0)
+			tmp[h] = vv
+		}
+	}
+
+	for k, v := range tmp {
+		out[k] = v
+	}
+
+	return true, nil
+}
+
+// Count returns the total number of rows in the dataset
+func (d *decoder) Count() uint64 {
+	return d.count
+}
+
+func (d *decoder) flushTemp(r io.Reader) (err error) {
+	_, err = io.Copy(d.src, r)
+	if err != nil {
+		return
+	}
+
+	d.src.Seek(0, 0)
+	return nil
+}

server/pkg/envoyx/json/decoder_test.go

@@ -0,0 +1,72 @@
+package json
+
+import (
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoyx/datasource"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDecoder(t *testing.T) {
+	req := require.New(t)
+
+	t.Run("init & meta", func(t *testing.T) {
+		dc, err := Decoder(testReader(), "test.csv")
+		req.NoError(err)
+
+		req.Equal(uint64(3), dc.Count())
+	})
+
+	t.Run("fields", func(t *testing.T) {
+		dc, err := Decoder(testReader(), "test.csv")
+		req.NoError(err)
+
+		hh := dc.Fields()
+		req.Contains(hh, "f1")
+		req.Contains(hh, "f2")
+		req.Contains(hh, "f3")
+	})
+
+	t.Run("iterate", func(t *testing.T) {
+		dc, err := Decoder(testReader(), "test.csv")
+		req.NoError(err)
+
+		aux := make(datasource.RawRecord)
+		var more bool
+
+		more, err = dc.Next(nil, aux)
+		req.NoError(err)
+		req.True(more)
+		req.Equal("r1f1", aux["f1"].Values[0])
+		req.Equal("r1f2", aux["f2"].Values[0])
+		req.Equal("r1f3", aux["f3"].Values[0])
+
+		more, err = dc.Next(nil, aux)
+		req.NoError(err)
+		req.True(more)
+		req.Equal("r2f1", aux["f1"].Values[0])
+		req.Equal("r2f2", aux["f2"].Values[0])
+		req.Equal("r2f3", aux["f3"].Values[0])
+
+		more, err = dc.Next(nil, aux)
+		req.NoError(err)
+		req.True(more)
+		req.Equal("r3f1", aux["f1"].Values[0])
+		req.Equal("r3f2", aux["f2"].Values[0])
+		req.Equal("r3f3", aux["f3"].Values[0])
+
+		more, err = dc.Next(nil, aux)
+		req.NoError(err)
+		req.False(more)
+	})
+}
+
+func testReader() io.Reader {
+	src := `{"f1": "r1f1", "f2": "r1f2", "f3": "r1f3"}
+{"f1": "r2f1", "f2": "r2f2", "f3": "r2f3"}
+{"f1": "r3f1", "f2": "r3f2", "f3": "r3f3"}`
+
+	return strings.NewReader(src)
+}