snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/pkg/envoy/csv/bulk_compose_record_marshal.go

@@ -0,0 +1,162 @@
+package csv
+
+import (
+	"context"
+	"encoding/csv"
+	"fmt"
+	"io"
+
+	"github.com/cortezaproject/corteza/server/compose/types"
+	"github.com/cortezaproject/corteza/server/pkg/envoy"
+	"github.com/cortezaproject/corteza/server/pkg/envoy/resource"
+)
+
+type (
+	bulkComposeRecordEncoder struct {
+		encoderConfig *EncoderConfig
+
+		res *resource.ComposeRecord
+
+		refNs string
+
+		refMod string
+		relMod *types.Module
+	}
+)
+
+func bulkComposeRecordEncoderFromResource(rec *resource.ComposeRecord, cfg *EncoderConfig) *bulkComposeRecordEncoder {
+	return &bulkComposeRecordEncoder{
+		encoderConfig: cfg,
+
+		res: rec,
+	}
+}
+
+// Prepare prepares the composeRecord to be encoded
+//
+// Any validation, additional constraining should be performed here.
+func (n *bulkComposeRecordEncoder) Prepare(ctx context.Context, state *envoy.ResourceState) (err error) {
+	_, ok := state.Res.(*resource.ComposeRecord)
+	if !ok {
+		return encoderErrInvalidResource(types.RecordResourceType, state.Res.ResourceType())
+	}
+
+	return nil
+}
+
+// Encode encodes the composeRecord to the document
+//
+// Encode is allowed to do some data manipulation, but no resource constraints
+// should be changed.
+func (n *bulkComposeRecordEncoder) Encode(ctx context.Context, w io.Writer, state *envoy.ResourceState) (err error) {
+	enc := csv.NewWriter(w)
+	all := len(n.encoderConfig.Fields) == 0
+
+	// Generate header & cell index
+	hh := make([]string, 0, 100)
+	hh = append(hh, "id")
+
+	pickLast := func(v resource.ComposeRecordRawValue) string {
+		if v.IsMulti {
+			if len(v.Values) > 0 {
+				return v.Values[len(v.Values)-1]
+			}
+			return ""
+		}
+
+		return v.Value
+	}
+
+	fx := make(map[string]int)
+	// 1 sys fields
+	sysFields := []string{"ownedBy", "createdAt", "createdBy", "updatedAt", "updatedBy", "deletedAt", "deletedBy"}
+	for _, sf := range sysFields {
+		if all || n.encoderConfig.Fields[sf] {
+			fx[sf] = len(hh)
+			hh = append(hh, sf)
+		}
+	}
+
+	// 2 modlue fields
+	for _, f := range n.res.RelMod.Fields {
+		if all || n.encoderConfig.Fields[f.Name] {
+			fx[f.Name] = len(hh)
+			hh = append(hh, f.Name)
+		}
+	}
+	enc.Write(hh)
+
+	err = n.res.Walker(func(r *resource.ComposeRecordRaw) error {
+
+		row := make([]string, len(hh))
+
+		var err error
+		row[0] = r.ID
+
+		if r.Us != nil {
+			if r.Us.OwnedBy != nil && (all || n.encoderConfig.Fields["ownedBy"]) {
+				row[fx["ownedBy"]], err = n.res.UserFlakes.GetByStamp(r.Us.OwnedBy).Model()
+			}
+			if r.Us.CreatedBy != nil && (all || n.encoderConfig.Fields["createdBy"]) {
+				row[fx["createdBy"]], err = n.res.UserFlakes.GetByStamp(r.Us.CreatedBy).Model()
+			}
+			if r.Us.UpdatedBy != nil && (all || n.encoderConfig.Fields["updatedBy"]) {
+				row[fx["updatedBy"]], err = n.res.UserFlakes.GetByStamp(r.Us.UpdatedBy).Model()
+			}
+			if r.Us.DeletedBy != nil && (all || n.encoderConfig.Fields["deletedBy"]) {
+				row[fx["deletedBy"]], err = n.res.UserFlakes.GetByStamp(r.Us.DeletedBy).Model()
+			}
+		}
+
+		if r.Ts != nil {
+			r.Ts, err = r.Ts.Model(n.encoderConfig.TimeLayout, n.encoderConfig.Timezone)
+			if err != nil {
+				return err
+			}
+			if r.Ts.CreatedAt != nil && (all || n.encoderConfig.Fields["createdAt"]) {
+				row[fx["createdAt"]] = r.Ts.CreatedAt.S
+			}
+			if r.Ts.UpdatedAt != nil && (all || n.encoderConfig.Fields["updatedAt"]) {
+				row[fx["updatedAt"]] = r.Ts.UpdatedAt.S
+			}
+			if r.Ts.DeletedAt != nil && (all || n.encoderConfig.Fields["deletedAt"]) {
+				row[fx["deletedAt"]] = r.Ts.DeletedAt.S
+			}
+		}
+		if err != nil {
+			return err
+		}
+
+		for k, v := range r.Values {
+			// Skip fields we don't want
+			if !all && !n.encoderConfig.Fields[k] {
+				continue
+			}
+
+			cell, has := fx[k]
+			if !has {
+				return fmt.Errorf("unknown cell %s", k)
+			}
+			f := n.res.RelMod.Fields.FindByName(k)
+			if f == nil {
+				return fmt.Errorf("field %s not found", k)
+			}
+
+			if f.Kind == "User" {
+				row[cell], err = n.res.UserFlakes.GetByKey(pickLast(v)).Model()
+				if err != nil {
+					return err
+				}
+			} else {
+				row[cell] = pickLast(v)
+			}
+		}
+		return enc.Write(row)
+	})
+	if err != nil {
+		return err
+	}
+
+	enc.Flush()
+	return nil
+}

server/pkg/envoy/csv/decoder.go

@@ -0,0 +1,123 @@
+package csv
+
+import (
+	"context"
+	"encoding/csv"
+	"io"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoy"
+	"github.com/cortezaproject/corteza/server/pkg/envoy/resource"
+	"github.com/gabriel-vasile/mimetype"
+)
+
+type (
+	// wrapper struct for csv related methods
+	decoder struct{}
+
+	// csv decoder wrapper for additional bits
+	reader struct {
+		header []string
+		count  uint64
+
+		cache      []map[string]string
+		cacheIndex int
+	}
+)
+
+// Decoder initializes and returns a fresh CSV decoder
+func Decoder() *decoder {
+	return &decoder{}
+}
+
+// CanDecodeFile determines if the file can be determined by this decoder
+func (y *decoder) CanDecodeFile(f io.Reader) bool {
+	m, err := mimetype.DetectReader(f)
+	if err != nil {
+		return false
+	}
+
+	return y.CanDecodeExt(m.Extension())
+}
+
+func (y *decoder) CanDecodeMime(m string) bool {
+	return m == "text/csv"
+}
+
+func (y *decoder) CanDecodeExt(ext string) bool {
+	pt := strings.Split(ext, ".")
+	return strings.TrimSpace(pt[len(pt)-1]) == "csv"
+}
+
+// Decode decodes the given io.Reader into a generic resource dataset
+func (c *decoder) Decode(ctx context.Context, r io.Reader, do *envoy.DecoderOpts) ([]resource.Interface, error) {
+	cr := &reader{
+		cacheIndex: 0,
+		cache:      make([]map[string]string, 0, 1000),
+	}
+
+	err := cr.prepare(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return []resource.Interface{resource.NewResourceDataset(do.Name, cr)}, nil
+}
+
+// The prepare step caches all of the rows into a cache array that will be used
+// to access already visited CSV rows.
+//
+// @todo implement FS cache file for larger files
+func (cr *reader) prepare(r io.Reader) (err error) {
+	cReader := csv.NewReader(r)
+
+	// Header
+	cr.header, err = cReader.Read()
+	if err != nil {
+		return err
+	}
+
+	for {
+		aux := make(map[string]string)
+		rr, err := cReader.Read()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return err
+		}
+
+		// Entry count
+		cr.count++
+
+		// Cache entry
+		for i, h := range cr.header {
+			aux[h] = rr[i]
+		}
+		cr.cache = append(cr.cache, aux)
+	}
+}
+
+// Fields returns every available field in this dataset
+func (cr *reader) Fields() []string {
+	return cr.header
+}
+
+func (cr *reader) Reset() error {
+	cr.cacheIndex = 0
+	return nil
+}
+
+// Next returns the field: value mapping for the next row
+func (cr *reader) Next() (map[string]string, error) {
+	if cr.cacheIndex >= len(cr.cache) {
+		return nil, nil
+	}
+
+	mr := cr.cache[cr.cacheIndex]
+	cr.cacheIndex++
+	return mr, nil
+}
+
+func (cr *reader) Count() uint64 {
+	return cr.count
+}

server/pkg/envoy/csv/decoder_test.go

@@ -0,0 +1,58 @@
+package csv
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoy"
+	"github.com/cortezaproject/corteza/server/pkg/envoy/resource"
+	"github.com/stretchr/testify/require"
+)
+
+func parseDocument(ctx context.Context, name string) (*resource.ResourceDataset, error) {
+	path := fmt.Sprintf("testdata/%s.csv", name)
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	cd := Decoder()
+	ii, err := cd.Decode(ctx, f, &envoy.DecoderOpts{
+		Name: name,
+		Path: path,
+	})
+	if err != nil {
+		return nil, err
+	}
+	rd, _ := ii[0].(*resource.ResourceDataset)
+	return rd, nil
+}
+
+func TestDecoder(t *testing.T) {
+	ctx := context.Background()
+
+	t.Run("doc 1", func(t *testing.T) {
+		req := require.New(t)
+
+		ds, err := parseDocument(ctx, "records_1")
+		req.NoError(err)
+		req.NotNil(ds)
+		req.Equal(resource.MakeIdentifiers("records_1"), ds.Identifiers())
+		req.Equal([]string{"id", "c1", "c2", "c3"}, ds.P.Fields())
+		req.Equal(uint64(3), ds.P.Count())
+
+		for i := 0; i < 3; i++ {
+			n, err := ds.P.Next()
+			req.NoError(err)
+			req.NotNil(n)
+
+			req.Len(n, 4)
+		}
+		n, err := ds.P.Next()
+		req.Nil(n)
+		req.Nil(err)
+	})
+}

server/pkg/envoy/csv/encoder.go

@@ -0,0 +1,163 @@
+package csv
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoy"
+	"github.com/cortezaproject/corteza/server/pkg/envoy/resource"
+)
+
+type (
+	bulkRecordEncoder struct {
+		cfg *EncoderConfig
+
+		resState map[resource.Interface]*encoderState
+	}
+
+	encoderState struct {
+		res          resourceState
+		source       io.ReadWriter
+		resourceType string
+		Scope        string
+		identifier   string
+	}
+
+	// EncoderConfig allows us to configure the resource encoding process
+	EncoderConfig struct {
+		// Timezone defines what timezone should be used when encoding timestamps
+		//
+		// If not defined, UTC is used
+		Timezone string
+		// TimeLayout defines how to format the encoded timestamp
+		//
+		// If not defined, RFC3339 is used (this one - 2006-01-02T15:04:05Z07:00)
+		TimeLayout string
+		// Fields specifies what fields we wish to include in the export
+		Fields map[string]bool
+	}
+
+	// resourceState holds some intermedia values to help with encoding
+	resourceState interface {
+		Prepare(ctx context.Context, state *envoy.ResourceState) (err error)
+		Encode(ctx context.Context, w io.Writer, state *envoy.ResourceState) (err error)
+	}
+)
+
+var (
+	ErrUnknownResource        = fmt.Errorf("unknown resource")
+	ErrResourceStateUndefined = fmt.Errorf("undefined resource state")
+)
+
+func NewBulkRecordEncoder(cfg *EncoderConfig) envoy.PrepareEncodeStreamer {
+	if cfg == nil {
+		cfg = &EncoderConfig{}
+	}
+
+	return &bulkRecordEncoder{
+		cfg: cfg,
+
+		resState: make(map[resource.Interface]*encoderState),
+	}
+}
+
+// Prepare prepares the encoder for the given set of resources
+//
+// It initializes and prepares the resource state for each provided resource
+func (se *bulkRecordEncoder) Prepare(ctx context.Context, ee ...*envoy.ResourceState) (err error) {
+	f := func(rs resourceState, es *envoy.ResourceState) error {
+		err = rs.Prepare(ctx, es)
+		if err != nil {
+			return err
+		}
+
+		se.resState[es.Res] = &encoderState{
+			res:          rs,
+			source:       &bytes.Buffer{},
+			resourceType: es.Res.ResourceType(),
+			identifier:   es.Res.Identifiers().First(),
+		}
+		return nil
+	}
+
+	for _, e := range ee {
+		switch res := e.Res.(type) {
+		// @todo other resources; we'll only do records for now
+		case *resource.ComposeRecord:
+			err = f(bulkComposeRecordEncoderFromResource(res, se.cfg), e)
+
+		default:
+			err = ErrUnknownResource
+		}
+
+		if err != nil {
+			return se.WrapError("prepare", e.Res, err)
+		}
+	}
+
+	return nil
+}
+
+// Encode encodes the resources into a series of csv documents
+func (se *bulkRecordEncoder) Encode(ctx context.Context, p envoy.Provider) error {
+	var e *envoy.ResourceState
+	var err error
+
+	// Encode the resources into document structs
+	for {
+		e, err = p.NextInverted(ctx)
+
+		if err != nil {
+			return err
+		}
+		if e == nil {
+			break
+		}
+
+		state := se.resState[e.Res]
+		if state == nil {
+			err = ErrResourceStateUndefined
+		} else {
+			err = state.res.Encode(ctx, state.source, e)
+		}
+
+		if err != nil {
+			return se.WrapError("encode: build doc", e.Res, err)
+		}
+	}
+
+	for _, s := range se.resState {
+		s.res = nil
+	}
+
+	return nil
+}
+
+func (se *bulkRecordEncoder) Stream() []*envoy.Stream {
+	ss := make([]*envoy.Stream, 0, 20)
+
+	for _, s := range se.resState {
+		ss = append(ss, &envoy.Stream{
+			Resource:   s.resourceType,
+			Identifier: s.identifier,
+			Source:     s.source,
+		})
+	}
+
+	return ss
+}
+
+// WrapError wraps errors related to csv encoding
+//
+// Always wrap your errors.
+func (se *bulkRecordEncoder) WrapError(act string, res resource.Interface, err error) error {
+	rt := strings.Join(strings.Split(strings.TrimSpace(strings.TrimRight(res.ResourceType(), ":")), ":"), " ")
+	return fmt.Errorf("csv encoder %s %s %v: %s", act, rt, res.Identifiers().StringSlice(), err)
+}
+
+func encoderErrInvalidResource(exp, got string) error {
+	return fmt.Errorf("invalid resource type: expecting %s, got %s", exp, got)
+}

server/pkg/envoy/csv/testdata/records_1.csv

@@ -0,0 +1,4 @@
+id,c1,c2,c3
+1,c1.v1,c2.v1,c3.v1
+2,c1.v2,c2.v2,c3.v2
+3,c1.v3,c2.v3,c3.v3

server/pkg/envoy/csv/util_encode.go

@@ -0,0 +1,126 @@
+package csv
+
+import (
+	"fmt"
+
+	"github.com/cortezaproject/corteza/server/pkg/envoy/resource"
+)
+
+type (
+	seqNode []interface{}
+	mapNode map[string]interface{}
+)
+
+// mapTimestamps helper encodes Timestamps into the mapping node
+func mapTimestamps(n mapNode, ts *resource.Timestamps) (mapNode, error) {
+	if ts == nil {
+		return n, nil
+	}
+
+	return addMap(n,
+		"createdAt", ts.CreatedAt,
+		"updatedAt", ts.UpdatedAt,
+		"deletedAt", ts.DeletedAt,
+		"archivedAt", ts.ArchivedAt,
+		"suspendedAt", ts.SuspendedAt,
+	)
+}
+
+// mapUserstamps helper encodes Userstamps into the mapping node
+func mapUserstamps(n mapNode, us *resource.Userstamps) (mapNode, error) {
+	if us == nil {
+		return n, nil
+	}
+
+	return addMap(n,
+		"createdBy", us.CreatedBy,
+		"updatedBy", us.UpdatedBy,
+		"deletedBy", us.DeletedBy,
+		"ownedBy", us.OwnedBy,
+	)
+}
+
+// cleanMap helper removes any empty k:v nodes from the mapping node
+//
+// The value is empty when the tag is !!null OR when the value and the content are empty
+func cleanMap(n mapNode) mapNode {
+	m := make(mapNode)
+
+	for k, v := range n {
+		if k != "" && v != nil {
+			m[k] = v
+		}
+	}
+
+	return m
+}
+
+// makeMap creates a new mapping node based on the provided k, v items
+//
+// pp is a set of k, v items; where k's lie at i, and v's lie at i+1.
+// non-string values (required by YAML nodes) are processed further.
+// eg.: ["k1", "v1", "k2", "v2"]
+func makeMap(pp ...interface{}) (mapNode, error) {
+	return addMap(make(mapNode), pp...)
+}
+
+// addMap adds a new item to the provided mapping node
+//
+// pp is a set of k, v items; where k's lie at i, and v's lie at i+1.
+// non-string values (required by YAML nodes) are processed further.
+// eg.: ["k1", "v1", "k2", "v2"]
+func addMap(n mapNode, pp ...interface{}) (mapNode, error) {
+	if len(pp) == 0 {
+		return n, nil
+	}
+
+	if len(pp)%2 == 1 {
+		return nil, fmt.Errorf("uneven number of elements provided (%d): %v", len(pp), pp)
+	}
+
+	for i := 0; i < len(pp); i += 2 {
+		kRaw := pp[i]
+		vRaw := pp[i+1]
+
+		k, ok := kRaw.(string)
+		if !ok {
+			return nil, fmt.Errorf("keys must be of type string: %v", kRaw)
+		}
+
+		switch v := vRaw.(type) {
+		case string:
+			if v == "" {
+				continue
+			}
+		case bool:
+			if !v {
+				continue
+			}
+		case int, uint,
+			int32, uint32,
+			int64, uint64:
+			if v == 0 {
+				continue
+			}
+		}
+
+		n[k] = vRaw
+	}
+	return n, nil
+}
+
+// makeSeq creates a new sequence node based on the provided items
+func makeSeq(vv ...interface{}) (seqNode, error) {
+	return addSeq(make(seqNode, 0, len(vv)), vv...)
+}
+
+// addSeq adds new items to the sequence node
+func addSeq(n seqNode, vv ...interface{}) (seqNode, error) {
+	for _, vRaw := range vv {
+		if vRaw != "" && vRaw != nil {
+			n = append(n, vRaw)
+		}
+	}
+
+	return n, nil
+}