snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/pkg/envoy/json/bulk_compose_record_marshal.go

@@ -0,0 +1,135 @@
+package json
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"github.com/cortezaproject/corteza/server/compose/types"
+	"github.com/cortezaproject/corteza/server/pkg/envoy"
+	"github.com/cortezaproject/corteza/server/pkg/envoy/resource"
+)
+
+type (
+	bulkComposeRecordEncoder struct {
+		encoderConfig *EncoderConfig
+
+		res *resource.ComposeRecord
+
+		refNs string
+
+		refMod string
+		relMod *types.Module
+	}
+)
+
+func bulkComposeRecordEncoderFromResource(rec *resource.ComposeRecord, cfg *EncoderConfig) *bulkComposeRecordEncoder {
+	return &bulkComposeRecordEncoder{
+		encoderConfig: cfg,
+
+		res: rec,
+	}
+}
+
+func (n *bulkComposeRecordEncoder) Prepare(ctx context.Context, state *envoy.ResourceState) (err error) {
+	_, ok := state.Res.(*resource.ComposeRecord)
+	if !ok {
+		return encoderErrInvalidResource(types.RecordResourceType, state.Res.ResourceType())
+	}
+
+	return nil
+}
+
+func (n *bulkComposeRecordEncoder) Encode(ctx context.Context, w io.Writer, state *envoy.ResourceState) (err error) {
+	enc := json.NewEncoder(w)
+	all := len(n.encoderConfig.Fields) == 0
+
+	getMapValue := func(v resource.ComposeRecordRawValue, proc func(k any) *resource.Userstamp) any {
+		if v.IsMulti {
+			if proc == nil {
+				return v.Values
+			}
+
+			out := make([]interface{}, 0, len(v.Values))
+			for _, vv := range v.Values {
+				out = append(out, proc(vv))
+			}
+
+			return out
+		}
+
+		if proc == nil {
+			return v.Value
+		}
+
+		return proc(v.Value)
+	}
+
+	err = n.res.Walker(func(r *resource.ComposeRecordRaw) error {
+		m, err := makeMap(
+			"id", r.ID,
+		)
+
+		r.Ts, err = r.Ts.Model(n.encoderConfig.TimeLayout, n.encoderConfig.Timezone)
+		if err != nil {
+			return err
+		}
+
+		// Encode system values
+		sysValues := make(map[string]interface{})
+		sysValues["createdAt"] = r.Ts.CreatedAt
+		sysValues["updatedAt"] = r.Ts.UpdatedAt
+		sysValues["deletedAt"] = r.Ts.DeletedAt
+		sysValues["createdBy"] = n.res.UserFlakes.GetByStamp(r.Us.CreatedBy)
+		sysValues["updatedBy"] = n.res.UserFlakes.GetByStamp(r.Us.UpdatedBy)
+		sysValues["deletedBy"] = n.res.UserFlakes.GetByStamp(r.Us.DeletedBy)
+		sysValues["ownedBy"] = n.res.UserFlakes.GetByStamp(r.Us.OwnedBy)
+
+		for k, v := range sysValues {
+			// Skip fields we don't want
+			if !all && !n.encoderConfig.Fields[k] {
+				continue
+			}
+			m, err = addMap(m,
+				k, v,
+			)
+
+			if err != nil {
+				return err
+			}
+		}
+
+		// Encode record values
+		for k, v := range r.Values {
+			// Skip fields we don't want
+			if !all && !n.encoderConfig.Fields[k] {
+				continue
+			}
+
+			f := n.res.RelMod.Fields.FindByName(k)
+			if f == nil {
+				return fmt.Errorf("field %s not found", k)
+			}
+			if f.Kind == "User" {
+				m, err = addMap(m,
+					k, getMapValue(v, n.res.UserFlakes.GetByKey),
+				)
+			} else {
+				m, err = addMap(m,
+					k, getMapValue(v, nil),
+				)
+			}
+			if err != nil {
+				return err
+			}
+		}
+
+		return enc.Encode(m)
+	})
+
+	if err != nil {
+		return err
+	}
+	return nil
+}

server/pkg/envoy/json/decoder.go

@@ -0,0 +1,139 @@
+package json
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoy"
+	"github.com/cortezaproject/corteza/server/pkg/envoy/resource"
+	"github.com/gabriel-vasile/mimetype"
+)
+
+type (
+	// wrapper struct for json related methods
+	decoder struct{}
+
+	// json decoder wrapper for additional bits
+	reader struct {
+		header []string
+		count  uint64
+
+		cache      []map[string]string
+		cacheIndex int
+	}
+)
+
+// Decoder initializes and returns a fresh JSON decoder
+//
+// We'll only do jsonl for now -- for record importing.
+// We'll expand this to work with other resources later on.
+func Decoder() *decoder {
+	return &decoder{}
+}
+
+// CanDecodeFile determines if the file can be decoded by this decoder
+func (d *decoder) CanDecodeFile(f io.Reader) bool {
+	var buff bytes.Buffer
+	tr := io.TeeReader(f, &buff)
+
+	m, err := mimetype.DetectReader(tr)
+	if err != nil {
+		return false
+	}
+	ext := m.Extension()
+
+	return d.CanDecodeExt(ext)
+}
+
+func (d *decoder) CanDecodeMime(m string) bool {
+	return m == "application/json" || m == "application/jsonlines"
+}
+
+func (d *decoder) CanDecodeExt(ext string) bool {
+	pt := strings.Split(ext, ".")
+	ext = strings.TrimSpace(pt[len(pt)-1])
+	return ext == "jsonl" || ext == "json" || ext == "ndjson"
+}
+
+// Decode decodes the given io.Reader into a generic resource dataset
+func (d *decoder) Decode(ctx context.Context, r io.Reader, do *envoy.DecoderOpts) ([]resource.Interface, error) {
+	jr := &reader{
+		cacheIndex: 0,
+		cache:      make([]map[string]string, 0, 1000),
+	}
+
+	err := jr.prepare(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return []resource.Interface{resource.NewResourceDataset(do.Name, jr)}, nil
+}
+
+// The prepare step caches all of the rows into a cache array that will be used
+// to access already visited json rows.
+//
+// @todo implement FS cache file for larger files
+func (jr *reader) prepare(r io.Reader) (err error) {
+	jReader := json.NewDecoder(r)
+
+	// JSON can omit empty values, so we can't be 100% that all of the headers
+	// were read in the first go.
+	// We'll determine all of the headers below where we count the entries
+	hx := make(map[string]bool)
+	jr.header = make([]string, 0, 100)
+
+	for jReader.More() {
+		aux := make(map[string]string)
+		err = jReader.Decode(&aux)
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return
+		}
+
+		// Get all the header fields
+		for h := range aux {
+			if !hx[h] {
+				jr.header = append(jr.header, h)
+				hx[h] = true
+			}
+		}
+
+		// Entry count
+		jr.count++
+
+		// Cache entry
+		jr.cache = append(jr.cache, aux)
+	}
+
+	return nil
+}
+
+func (jr *reader) Reset() error {
+	jr.cacheIndex = 0
+	return nil
+}
+
+// Fields returns every available field in this dataset
+func (jr *reader) Fields() []string {
+	return jr.header
+}
+
+// Next returns the field: value mapping for the next row
+func (jr *reader) Next() (map[string]string, error) {
+	if jr.cacheIndex >= len(jr.cache) {
+		return nil, nil
+	}
+
+	mr := jr.cache[jr.cacheIndex]
+	jr.cacheIndex++
+	return mr, nil
+}
+
+func (cr *reader) Count() uint64 {
+	return cr.count
+}

server/pkg/envoy/json/decoder_test.go

@@ -0,0 +1,58 @@
+package json
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoy"
+	"github.com/cortezaproject/corteza/server/pkg/envoy/resource"
+	"github.com/stretchr/testify/require"
+)
+
+func parseDocument(ctx context.Context, name string) (*resource.ResourceDataset, error) {
+	path := fmt.Sprintf("testdata/%s.jsonl", name)
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	cd := Decoder()
+	ii, err := cd.Decode(ctx, f, &envoy.DecoderOpts{
+		Name: name,
+		Path: path,
+	})
+	if err != nil {
+		return nil, err
+	}
+	rd, _ := ii[0].(*resource.ResourceDataset)
+	return rd, nil
+}
+
+func TestDecoder(t *testing.T) {
+	ctx := context.Background()
+
+	t.Run("doc 1", func(t *testing.T) {
+		req := require.New(t)
+
+		ds, err := parseDocument(ctx, "records_1")
+		req.NoError(err)
+		req.NotNil(ds)
+		req.Equal(resource.MakeIdentifiers("records_1"), ds.Identifiers())
+		req.Subset([]string{"id", "c1", "c2", "c3"}, ds.P.Fields())
+		req.Equal(uint64(3), ds.P.Count())
+
+		for i := 0; i < 3; i++ {
+			n, err := ds.P.Next()
+			req.NoError(err)
+			req.NotNil(n)
+
+			req.Len(n, 4)
+		}
+		n, err := ds.P.Next()
+		req.Nil(n)
+		req.Nil(err)
+	})
+}

server/pkg/envoy/json/encoder.go

@@ -0,0 +1,163 @@
+package json
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoy"
+	"github.com/cortezaproject/corteza/server/pkg/envoy/resource"
+)
+
+type (
+	bulkRecordEncoder struct {
+		cfg *EncoderConfig
+
+		resState map[resource.Interface]*encoderState
+	}
+
+	encoderState struct {
+		res          resourceState
+		source       io.ReadWriter
+		resourceType string
+		Scope        string
+		identifier   string
+	}
+
+	// EncoderConfig allows us to configure the resource encoding process
+	EncoderConfig struct {
+		// Timezone defines what timezone should be used when encoding timestamps
+		//
+		// If not defined, UTC is used
+		Timezone string
+		// TimeLayout defines how to format the encoded timestamp
+		//
+		// If not defined, RFC3339 is used (this one - 2006-01-02T15:04:05Z07:00)
+		TimeLayout string
+		// Fields specifies what fields we wish to include in the export
+		Fields map[string]bool
+	}
+
+	// resourceState holds some intermedia values to help with encoding
+	resourceState interface {
+		Prepare(ctx context.Context, state *envoy.ResourceState) (err error)
+		Encode(ctx context.Context, w io.Writer, state *envoy.ResourceState) (err error)
+	}
+)
+
+var (
+	ErrUnknownResource        = fmt.Errorf("unknown resource")
+	ErrResourceStateUndefined = fmt.Errorf("undefined resource state")
+)
+
+func NewBulkRecordEncoder(cfg *EncoderConfig) envoy.PrepareEncodeStreamer {
+	if cfg == nil {
+		cfg = &EncoderConfig{}
+	}
+
+	return &bulkRecordEncoder{
+		cfg: cfg,
+
+		resState: make(map[resource.Interface]*encoderState),
+	}
+}
+
+// Prepare prepares the encoder for the given set of resources
+//
+// It initializes and prepares the resource state for each provided resource
+func (se *bulkRecordEncoder) Prepare(ctx context.Context, ee ...*envoy.ResourceState) (err error) {
+	f := func(rs resourceState, es *envoy.ResourceState) error {
+		err = rs.Prepare(ctx, es)
+		if err != nil {
+			return err
+		}
+
+		se.resState[es.Res] = &encoderState{
+			res:          rs,
+			source:       &bytes.Buffer{},
+			resourceType: es.Res.ResourceType(),
+			identifier:   es.Res.Identifiers().First(),
+		}
+		return nil
+	}
+
+	for _, e := range ee {
+		switch res := e.Res.(type) {
+		// @todo other resources; we'll only do records for now
+		case *resource.ComposeRecord:
+			err = f(bulkComposeRecordEncoderFromResource(res, se.cfg), e)
+
+		default:
+			err = ErrUnknownResource
+		}
+
+		if err != nil {
+			return se.WrapError("prepare", e.Res, err)
+		}
+	}
+
+	return nil
+}
+
+// Encode encodes the resources into a series of jsonl documents
+func (se *bulkRecordEncoder) Encode(ctx context.Context, p envoy.Provider) error {
+	var e *envoy.ResourceState
+	var err error
+
+	// Encode the resources into document structs
+	for {
+		e, err = p.NextInverted(ctx)
+
+		if err != nil {
+			return err
+		}
+		if e == nil {
+			break
+		}
+
+		state := se.resState[e.Res]
+		if state == nil {
+			err = ErrResourceStateUndefined
+		} else {
+			err = state.res.Encode(ctx, state.source, e)
+		}
+
+		if err != nil {
+			return se.WrapError("encode: build doc", e.Res, err)
+		}
+	}
+
+	for _, s := range se.resState {
+		s.res = nil
+	}
+
+	return nil
+}
+
+func (se *bulkRecordEncoder) Stream() []*envoy.Stream {
+	ss := make([]*envoy.Stream, 0, 20)
+
+	for _, s := range se.resState {
+		ss = append(ss, &envoy.Stream{
+			Resource:   s.resourceType,
+			Identifier: s.identifier,
+			Source:     s.source,
+		})
+	}
+
+	return ss
+}
+
+// WrapError wraps errors related to json encoding
+//
+// Always wrap your errors.
+func (se *bulkRecordEncoder) WrapError(act string, res resource.Interface, err error) error {
+	rt := strings.Join(strings.Split(strings.TrimSpace(strings.TrimRight(res.ResourceType(), ":")), ":"), " ")
+	return fmt.Errorf("json encoder %s %s %v: %s", act, rt, res.Identifiers().StringSlice(), err)
+}
+
+func encoderErrInvalidResource(exp, got string) error {
+	return fmt.Errorf("invalid resource type: expecting %s, got %s", exp, got)
+}

server/pkg/envoy/json/testdata/records_1.jsonl

@@ -0,0 +1,3 @@
+{"id": "1", "c1": "c1.v1", "c2": "c2.v1", "c3": "c3.v1"}
+{"id": "2", "c1": "c1.v2", "c2": "c2.v2", "c3": "c3.v2"}
+{"id": "3", "c1": "c1.v3", "c2": "c2.v3", "c3": "c3.v3"}

server/pkg/envoy/json/util_encode.go

@@ -0,0 +1,143 @@
+package json
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoy/resource"
+)
+
+type (
+	seqNode []interface{}
+	mapNode map[string]interface{}
+)
+
+// mapTimestamps helper encodes Timestamps into the mapping node
+func mapTimestamps(n mapNode, ts *resource.Timestamps) (mapNode, error) {
+	if ts == nil {
+		return n, nil
+	}
+
+	return addMap(n,
+		"createdAt", ts.CreatedAt,
+		"updatedAt", ts.UpdatedAt,
+		"deletedAt", ts.DeletedAt,
+		"archivedAt", ts.ArchivedAt,
+		"suspendedAt", ts.SuspendedAt,
+	)
+}
+
+// mapUserstamps helper encodes Userstamps into the mapping node
+func mapUserstamps(n mapNode, us *resource.Userstamps) (mapNode, error) {
+	if us == nil {
+		return n, nil
+	}
+
+	return addMap(n,
+		"createdBy", us.CreatedBy,
+		"updatedBy", us.UpdatedBy,
+		"deletedBy", us.DeletedBy,
+		"ownedBy", us.OwnedBy,
+		"runAs", us.RunAs,
+	)
+}
+
+// cleanMap helper removes any empty k:v nodes from the mapping node
+//
+// The value is empty when the tag is !!null OR when the value and the content are empty
+func cleanMap(n mapNode) mapNode {
+	m := make(mapNode)
+
+	for k, v := range n {
+		if k != "" && v != nil {
+			m[k] = v
+		}
+	}
+
+	return m
+}
+
+// makeMap creates a new mapping node based on the provided k, v items
+//
+// pp is a set of k, v items; where k's lie at i, and v's lie at i+1.
+// non-string values (required by YAML nodes) are processed further.
+// eg.: ["k1", "v1", "k2", "v2"]
+func makeMap(pp ...interface{}) (mapNode, error) {
+	return addMap(make(mapNode), pp...)
+}
+
+func isNil(i interface{}) bool {
+	if i == nil {
+		return true
+	}
+	switch reflect.TypeOf(i).Kind() {
+	case reflect.Ptr, reflect.Map, reflect.Array, reflect.Chan, reflect.Slice:
+		return reflect.ValueOf(i).IsNil()
+	}
+	return false
+}
+
+// addMap adds a new item to the provided mapping node
+//
+// pp is a set of k, v items; where k's lie at i, and v's lie at i+1.
+// non-string values (required by YAML nodes) are processed further.
+// eg.: ["k1", "v1", "k2", "v2"]
+func addMap(n mapNode, pp ...interface{}) (mapNode, error) {
+	if len(pp) == 0 {
+		return n, nil
+	}
+
+	if len(pp)%2 == 1 {
+		return nil, fmt.Errorf("uneven number of elements provided (%d): %v", len(pp), pp)
+	}
+
+	for i := 0; i < len(pp); i += 2 {
+		kRaw := pp[i]
+		vRaw := pp[i+1]
+
+		if isNil(vRaw) {
+			continue
+		}
+
+		k, ok := kRaw.(string)
+		if !ok {
+			return nil, fmt.Errorf("keys must be of type string: %v", kRaw)
+		}
+
+		switch v := vRaw.(type) {
+		case string:
+			if v == "" {
+				continue
+			}
+		case bool:
+			if !v {
+				continue
+			}
+		case int, uint,
+			int32, uint32,
+			int64, uint64:
+			if v == 0 {
+				continue
+			}
+		}
+
+		n[k] = vRaw
+	}
+	return n, nil
+}
+
+// makeSeq creates a new sequence node based on the provided items
+func makeSeq(vv ...interface{}) (seqNode, error) {
+	return addSeq(make(seqNode, 0, len(vv)), vv...)
+}
+
+// addSeq adds new items to the sequence node
+func addSeq(n seqNode, vv ...interface{}) (seqNode, error) {
+	for _, vRaw := range vv {
+		if vRaw != "" && vRaw != nil {
+			n = append(n, vRaw)
+		}
+	}
+
+	return n, nil
+}