snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/store/adapters/rdbms/ddl/commands.go

@@ -0,0 +1,339 @@
+package ddl
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/cortezaproject/corteza/server/pkg/dal"
+	"github.com/doug-martin/goqu/v9/exp"
+	"github.com/jmoiron/sqlx"
+)
+
+type (
+	dialect interface {
+		QuoteIdent(i string) string
+	}
+
+	CreateTable struct {
+		Dialect               dialect
+		Table                 *Table
+		OmitIfNotExistsClause bool
+		SuffixClause          string
+	}
+
+	DropTable struct {
+		Dialect dialect
+		Table   string
+	}
+
+	CreateIndex struct {
+		Dialect               dialect
+		Index                 *Index
+		OmitIfNotExistsClause bool
+		OmitFieldLength       bool
+	}
+
+	DropIndex struct {
+		Dialect    dialect
+		Ident      string
+		TableIdent string
+	}
+
+	AddColumn struct {
+		Dialect dialect
+		Table   string
+		Column  *Column
+	}
+
+	DropColumn struct {
+		Dialect dialect
+		Table   string
+		Column  string
+	}
+
+	RenameColumn struct {
+		Dialect dialect
+		Table   string
+		Old     string
+		New     string
+	}
+
+	ReTypeColumn struct {
+		Dialect dialect
+		Table   string
+		Column  string
+		Type    *ColumnType
+	}
+
+	DropConstraint struct {
+		Dialect    dialect
+		Table      string
+		Constraint string
+	}
+)
+
+// Exec is a utility for executing series of commands
+//
+// # Parameters can be string, Stringer interface or goqu's exp.SQLExpression
+//
+// Any other type will result in panic
+func Exec(ctx context.Context, db sqlx.ExtContext, ss ...any) (err error) {
+	for _, s := range ss {
+		var (
+			sql  string
+			args []any
+		)
+
+		switch c := s.(type) {
+		case interface{ ToSQL() (string, []any, error) }:
+			sql, args, err = c.ToSQL()
+			if err != nil {
+				return
+			}
+		case fmt.Stringer:
+			sql = c.String()
+		case string:
+			sql = c
+		default:
+			panic(fmt.Sprintf("unexecutable input (%T)", s))
+		}
+
+		if _, err = db.ExecContext(ctx, sql, args...); err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func (t *CreateTable) String() string {
+	sql := "CREATE "
+	if t.Table.Temporary {
+		sql += "TEMPORARY "
+	}
+
+	sql += "TABLE "
+	if !t.OmitIfNotExistsClause {
+		sql += "IF NOT EXISTS "
+	}
+
+	sql += t.Dialect.QuoteIdent(t.Table.Ident) + " (\n"
+	sql += GenCreateTableBody(t.Dialect, t.Table)
+	sql += ")"
+	sql += t.SuffixClause
+
+	return sql
+}
+
+func (c *DropTable) ToSQL() (sql string, aa []interface{}, err error) {
+	return fmt.Sprintf(
+		`DROP TABLE %s`,
+		c.Dialect.QuoteIdent(c.Table),
+	), nil, nil
+}
+
+func GenCreateTableBody(d dialect, t *Table) string {
+	sql := ""
+
+	for c, col := range t.Columns {
+		if c == 0 {
+			sql += "  "
+		} else {
+			sql += ", "
+		}
+
+		sql += GenTableColumn(d, col)
+
+		sql += "\n"
+	}
+
+	// check if any of the indexes is a primary key
+	for _, pk := range t.Indexes {
+		if pk.Ident != PRIMARY_KEY {
+			continue
+		}
+
+		sql += "\n"
+		sql += ", " + GenPrimaryKey(d, pk) + "\n"
+		break
+	}
+
+	return sql
+}
+
+func GenTableColumn(d dialect, col *Column) string {
+	sql := d.QuoteIdent(col.Ident) + " " + col.Type.Name + " "
+
+	if col.Type.Null {
+		sql += "    NULL"
+	} else {
+		sql += "NOT NULL"
+	}
+
+	if col.Default > "" {
+		sql += " DEFAULT " + col.Default
+	}
+
+	return sql
+}
+
+func GenPrimaryKey(d dialect, pk *Index) string {
+	sql := "PRIMARY KEY ("
+	for f, field := range pk.Fields {
+		if f > 0 {
+			sql += ", "
+		}
+
+		sql += d.QuoteIdent(field.Column)
+	}
+	sql += ")"
+
+	return sql
+}
+
+func (t *CreateIndex) String() string {
+	sql := "CREATE "
+
+	if t.Index.Unique {
+		sql += "UNIQUE "
+	}
+
+	sql += "INDEX "
+
+	if !t.OmitIfNotExistsClause {
+		sql += "IF NOT EXISTS "
+	}
+
+	sql += t.Dialect.QuoteIdent(t.Index.Ident) + " ON " + t.Dialect.QuoteIdent(t.Index.TableIdent) + " ("
+
+	for f, field := range t.Index.Fields {
+		isExpr := len(field.Expression) > 0
+
+		if f > 0 {
+			sql += ", "
+		}
+
+		if isExpr {
+			sql += "("
+			sql += field.Expression
+		} else {
+			sql += t.Dialect.QuoteIdent(field.Column)
+		}
+
+		if field.Length > 0 && !t.OmitFieldLength {
+			sql += fmt.Sprintf("(%d)", field.Length)
+		}
+
+		if isExpr {
+			sql += ")"
+		}
+
+		switch field.Sort {
+		case dal.IndexFieldSortDesc:
+			sql += " DESC"
+		case dal.IndexFieldSortAsc:
+			sql += " ASC"
+		}
+
+		switch field.Nulls {
+		case dal.IndexFieldNullsLast:
+			sql += " NULLS LAST"
+		case dal.IndexFieldNullsFirst:
+			sql += " NULLS FIRST"
+		}
+
+	}
+	sql += ")"
+
+	if t.Index.Predicate != "" {
+		sql += " WHERE " + t.Index.Predicate
+	}
+
+	return sql
+}
+
+func (c *AddColumn) ToSQL() (sql string, aa []interface{}, err error) {
+	sql = fmt.Sprintf(
+		`ALTER TABLE %s ADD COLUMN %s %s`,
+		c.Dialect.QuoteIdent(c.Table),
+		c.Dialect.QuoteIdent(c.Column.Ident),
+		c.Column.Type.Name,
+	)
+
+	if !c.Column.Type.Null {
+		sql += " NOT NULL"
+	}
+
+	if len(c.Column.Default) > 0 {
+		// @todo right now we can (and need to) trust that default
+		//       values are unharmful!
+		sql += " DEFAULT " + c.Column.Default
+	}
+
+	return
+}
+
+func (c *DropColumn) ToSQL() (sql string, aa []interface{}, err error) {
+	return fmt.Sprintf(
+		`ALTER TABLE %s DROP COLUMN %s`,
+		c.Dialect.QuoteIdent(c.Table),
+		c.Dialect.QuoteIdent(c.Column),
+	), nil, nil
+}
+
+func (c *DropIndex) ToSQL() (sql string, aa []interface{}, err error) {
+	return fmt.Sprintf(
+		`DROP INDEX %s ON %s`,
+		c.Dialect.QuoteIdent(c.Ident),
+		c.Dialect.QuoteIdent(c.TableIdent),
+	), nil, nil
+}
+
+func (c *RenameColumn) ToSQL() (sql string, aa []interface{}, err error) {
+	return fmt.Sprintf(
+		`ALTER TABLE %s RENAME COLUMN %s TO %s`,
+		c.Dialect.QuoteIdent(c.Table),
+		c.Dialect.QuoteIdent(c.Old),
+		c.Dialect.QuoteIdent(c.New),
+	), nil, nil
+}
+
+func (c *ReTypeColumn) ToSQL() (sql string, aa []interface{}, err error) {
+	return fmt.Sprintf(
+		`ALTER TABLE %s MODIFY COLUMN %s %s`,
+		c.Dialect.QuoteIdent(c.Table),
+		c.Dialect.QuoteIdent(c.Column),
+		c.Type.Name,
+	), nil, nil
+}
+
+// GetBool is a utility function to simplify getting a boolean value from a query result.
+func GetBool(ctx context.Context, db sqlx.QueryerContext, query exp.SQLExpression) (bool, error) {
+	var (
+		exists         bool
+		sql, args, err = query.ToSQL()
+	)
+
+	if err != nil {
+		return false, fmt.Errorf("could not generate SQL: %v", err)
+	}
+
+	if err = sqlx.GetContext(ctx, db, &exists, sql, args...); err != nil {
+		return false, err
+	}
+
+	return exists, nil
+}
+
+// Structs is a utility function to simplify selecting data into slice of structs
+func Structs(ctx context.Context, db sqlx.QueryerContext, query exp.SQLExpression, t any) error {
+	var (
+		sql, args, err = query.ToSQL()
+	)
+
+	if err != nil {
+		return fmt.Errorf("could not generate SQL: %v", err)
+	}
+
+	return sqlx.SelectContext(ctx, db, t, sql, args...)
+}

server/store/adapters/rdbms/ddl/dal_model.go

@@ -0,0 +1,70 @@
+package ddl
+
+import (
+	"context"
+	"fmt"
+	"github.com/cortezaproject/corteza/server/pkg/dal"
+)
+
+// CreateModel creates table with columns and indexes that match Model definition
+func CreateModel(ctx context.Context, dd DataDefiner, m *dal.Model) (err error) {
+	var (
+		t *Table
+	)
+
+	if t, err = dd.ConvertModel(m); err != nil {
+		return err
+	}
+
+	if err = dd.TableCreate(ctx, t); err != nil {
+		return err
+	}
+
+	if err = EnsureIndexes(ctx, dd, m.Indexes...); err != nil {
+		return
+	}
+
+	return
+}
+
+func DeleteModel(ctx context.Context, dd DataDefiner, m *dal.Model) (err error) {
+	return fmt.Errorf("not implemented")
+}
+
+// UpdateModel alters existing table's columns and indexes to match Model definition
+func UpdateModel(ctx context.Context, dd DataDefiner, m *dal.Model) (err error) {
+	var (
+		t        *Table
+		col      *Column
+		colIdent string
+	)
+
+	if t, err = dd.TableLookup(ctx, m.Ident); err != nil {
+		return err
+	}
+
+	// @todo check model against table structure
+	for _, attr := range m.Attributes {
+		// iterate over attributes and check if they exist in the table
+		colIdent = attr.StoreIdent()
+		col = t.ColumnByIdent(colIdent)
+		if col != nil {
+			// @todo check if column type matches
+			// @todo check if column is nullable
+			continue
+		}
+
+		// @todo add column to table
+		return fmt.Errorf("column %q on table %q not found; adding columns is not yet supported", attr.Ident, m.Ident)
+	}
+
+	if err = EnsureIndexes(ctx, dd, m.Indexes...); err != nil {
+		return
+	}
+
+	return
+}
+
+func EnsureIndexes(ctx context.Context, dd DataDefiner, ii ...*dal.Index) (err error) {
+	return nil
+}

server/store/adapters/rdbms/ddl/data_definer.go

@@ -0,0 +1,289 @@
+package ddl
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"github.com/cortezaproject/corteza/server/pkg/dal"
+)
+
+type (
+	driverDialect interface {
+		QuoteIdent(string) string
+		// AttributeToColumn converts attribute to column
+		AttributeToColumn(*dal.Attribute) (*Column, error)
+		IndexFieldModifiers(*dal.Attribute, ...dal.IndexFieldModifier) (string, error)
+	}
+
+	// DataDefiner describes an interface for all DDL commands
+	DataDefiner interface {
+		ConvertModel(*dal.Model) (*Table, error)
+		ConvertAttribute(attr *dal.Attribute) (*Column, error)
+
+		// Tables(ctx context.Context) ([]*Table, error)
+		TableLookup(context.Context, string) (*Table, error)
+		TableCreate(context.Context, *Table) error
+		TableDrop(context.Context, string) error
+
+		ColumnAdd(context.Context, string, *Column) error
+		ColumnDrop(context.Context, string, string) error
+		ColumnRename(context.Context, string, string, string) error
+		ColumnReType(context.Context, string, string, *ColumnType) error
+
+		IndexLookup(context.Context, string, string) (*Index, error)
+		IndexCreate(context.Context, string, *Index) error
+		IndexDrop(context.Context, string, string) error
+	}
+
+	// Table describes structure of the SQL table
+	Table struct {
+		Ident   string
+		Columns []*Column
+		Indexes []*Index
+		Comment string
+
+		Temporary bool
+
+		// implementation variations
+		Meta map[string]interface{}
+	}
+
+	Column struct {
+		Ident string
+		Type  *ColumnType
+
+		Default string
+
+		// implementation variations
+		Meta map[string]interface{}
+
+		Comment string
+	}
+
+	ColumnType struct {
+		Name string
+
+		Null bool
+
+		// implementation variations
+		Meta map[string]interface{}
+	}
+
+	// Index describes structure of the SQL index
+	Index struct {
+		TableIdent string
+		Ident      string
+		Type       string
+		Fields     []*IndexField
+		Unique     bool
+		Predicate  string
+		Comment    string
+
+		// implementation variations
+		Meta map[string]interface{}
+	}
+
+	// IndexField describes a single field (column or expression) of the SQL index
+	IndexField struct {
+		// Expression or a single column
+		Column string
+
+		Length int
+
+		// Wrap part in parentheses
+		Expression string
+
+		// Ascending or descending
+		Sort  dal.IndexFieldSort
+		Nulls dal.IndexFieldNulls
+
+		Statistics *IndexFieldStatistics
+
+		// implementation variations
+		Meta map[string]interface{}
+	}
+
+	IndexFieldStatistics struct {
+		// Cardinality is an indicator that refers to the uniqueness
+		// of all values in a column. Low cardinality means a lot
+		// of duplicate values in that column. For example, a column
+		// that stores the gender values has low cardinality.
+		// In contrast, high cardinality means that there are many distinct values.
+		Cardinality int64
+	}
+)
+
+const (
+	PRIMARY_KEY = "PRIMARY"
+)
+
+func (t *Table) ColumnByIdent(i string) *Column {
+	for _, c := range t.Columns {
+		if c.Ident == i {
+			return c
+		}
+	}
+
+	return nil
+}
+
+// ConvertModel is generic model converter
+func ConvertModel(m *dal.Model, d driverDialect) (t *Table, err error) {
+	var (
+		col *Column
+		idx *Index
+
+		// keeps track of embedded attributes
+		// key is storeIdent
+		embeds = make(map[string]bool)
+	)
+
+	t = &Table{Ident: m.Ident}
+	for _, a := range m.Attributes {
+		if a.Type == nil {
+			continue
+		}
+
+		if embeds[a.StoreIdent()] {
+			continue
+		}
+
+		switch a.Store.(type) {
+		case *dal.CodecRecordValueSetJSON:
+			// add to embeds and make sure we do not add the column again!
+			embeds[a.StoreIdent()] = true
+
+			// throw away the attribute and create a new one
+			// to for JSON storage
+			a = &dal.Attribute{
+				Ident: a.StoreIdent(),
+				Type:  &dal.TypeJSON{Nullable: false},
+				Store: &dal.CodecPlain{},
+			}
+		}
+
+		if col, err = ConvertAttribute(a, d); err != nil {
+			return
+		}
+
+		t.Columns = append(t.Columns, col)
+	}
+
+	for _, i := range m.Indexes {
+		if idx, err = ConvertIndex(i, m.Attributes, m.Ident, d); err != nil {
+			return nil, fmt.Errorf("could not convert index %q: %w", i.Ident, err)
+		}
+
+		t.Indexes = append(t.Indexes, idx)
+	}
+
+	return
+}
+
+func ConvertAttribute(attr *dal.Attribute, d driverDialect) (col *Column, err error) {
+	col, err = d.AttributeToColumn(attr)
+	if err != nil {
+		return nil, fmt.Errorf("could not convert attribute %q to column: %w", attr.Ident, err)
+	}
+
+	return
+}
+
+// ConvertIndex converts dal.Index to ddl.Index
+func ConvertIndex(i *dal.Index, aa dal.AttributeSet, table string, d driverDialect) (idx *Index, err error) {
+	var (
+		a        *dal.Attribute
+		idxField *IndexField
+	)
+
+	idx = &Index{
+		TableIdent: table,
+		Ident:      i.Ident,
+		Type:       i.Type,
+		Unique:     i.Unique,
+		Predicate:  i.Predicate,
+	}
+
+	for _, f := range i.Fields {
+		// ensure attribute exists
+		if a = aa.FindByIdent(f.AttributeIdent); a == nil {
+			return nil, fmt.Errorf("referenced attribute %q does not exist", f.AttributeIdent)
+		}
+
+		idxField = &IndexField{
+			Sort:  f.Sort,
+			Nulls: f.Nulls,
+		}
+
+		if len(f.Modifiers) > 0 {
+			if idxField.Expression, err = d.IndexFieldModifiers(a, f.Modifiers...); err != nil {
+				return nil, fmt.Errorf("could not convert index field modifiers: %w", err)
+			}
+		} else {
+			idxField.Column = a.StoreIdent()
+		}
+
+		idx.Fields = append(idx.Fields, idxField)
+	}
+
+	return
+}
+
+func DefaultValueCurrentTimestamp(set bool) string {
+	if !set {
+		return ""
+	}
+
+	return "CURRENT_TIMESTAMP"
+}
+
+func DefaultBoolean(set, value bool) string {
+	switch {
+	case !set:
+		return ""
+	case value:
+		return "true"
+	default:
+		return "false"
+	}
+}
+
+func DefaultID(set bool, value uint64) string {
+	switch {
+	case !set:
+		return ""
+	default:
+		return strconv.FormatUint(value, 10)
+	}
+}
+
+func DefaultNumber(set bool, precision int, value float64) string {
+	switch {
+	case !set:
+		return ""
+	case precision >= 0:
+		return strconv.FormatFloat(value, 'f', precision, 64)
+	default:
+		return strconv.FormatFloat(value, 'f', -1, 64)
+	}
+}
+
+func DefaultJSON(set bool, value any) (_ string, err error) {
+	if str, is := value.(string); is {
+		return "'" + str + "'", nil
+	}
+
+	switch {
+	case !set:
+		return "", nil
+	default:
+		var aux []byte
+		if aux, err = json.Marshal(value); err != nil {
+			return "", fmt.Errorf("could not serialize default value for JSON field: %w", err)
+		}
+
+		return string(aux), nil
+	}
+}

server/store/adapters/rdbms/ddl/data_definer_test.go

@@ -0,0 +1,163 @@
+package ddl
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/cortezaproject/corteza/server/pkg/dal"
+	"github.com/stretchr/testify/require"
+)
+
+type (
+	mockDriver struct{}
+)
+
+func (mockDriver) QuoteIdent(i string) string { return `"` + i + `"` }
+
+func (mockDriver) AttributeToColumn(attr *dal.Attribute) (col *Column, err error) {
+	col = &Column{
+		Ident:   attr.StoreIdent(),
+		Comment: attr.Label,
+		Type: &ColumnType{
+			Null: attr.Type.IsNullable(),
+		},
+	}
+
+	switch t := attr.Type.(type) {
+	case *dal.TypeID:
+		_ = t
+		col.Type.Name = "INT"
+	case *dal.TypeText:
+		_ = t
+		col.Type.Name = "TEXT"
+	}
+
+	return col, nil
+}
+
+func (d mockDriver) IndexFieldModifiers(attr *dal.Attribute, mm ...dal.IndexFieldModifier) (string, error) {
+	return IndexFieldModifiers(attr, d.QuoteIdent, mm...)
+}
+
+// Copied over to avoid import cycle
+// @todo improve
+func IndexFieldModifiers(attr *dal.Attribute, quoteIdent func(i string) string, mm ...dal.IndexFieldModifier) (string, error) {
+	var (
+		modifier string
+		out      = quoteIdent(attr.StoreIdent())
+	)
+
+	for _, m := range mm {
+		switch m {
+		case dal.IndexFieldModifierLower:
+			modifier = "LOWER"
+		default:
+			return "", fmt.Errorf("unknown index field modifier: %s", m)
+		}
+
+		out = fmt.Sprintf("%s(%s)", modifier, out)
+	}
+
+	return out, nil
+}
+
+func TestModelToTable(t *testing.T) {
+
+	tests := []struct {
+		name string
+		m    *dal.Model
+		d    driverDialect
+
+		createTableSQL string
+		createIndexSQL string
+	}{
+		{
+			name: "simple",
+			d:    mockDriver{},
+			m: &dal.Model{
+				Ident: "simple",
+				Attributes: dal.AttributeSet{
+					&dal.Attribute{
+						Ident: "null_ID",
+						Type: &dal.TypeID{
+							Nullable: true,
+						},
+					},
+					&dal.Attribute{
+						Ident: "some_txt",
+						Type:  &dal.TypeText{},
+					},
+				},
+
+				Indexes: []*dal.Index{
+					{
+						Ident: PRIMARY_KEY,
+						Fields: []*dal.IndexField{
+							{
+								AttributeIdent: "null_ID",
+							},
+						},
+					},
+					{
+						Ident: "first_idx",
+						Fields: []*dal.IndexField{
+							{
+								AttributeIdent: "null_ID",
+								Modifiers:      []dal.IndexFieldModifier{dal.IndexFieldModifierLower},
+								Sort:           dal.IndexFieldSortDesc,
+							},
+						},
+					},
+					{
+						Ident: "second_idx",
+						Fields: []*dal.IndexField{
+							{
+								AttributeIdent: "null_ID",
+							},
+							{
+								AttributeIdent: "some_txt",
+								Modifiers:      []dal.IndexFieldModifier{dal.IndexFieldModifierLower},
+								Sort:           dal.IndexFieldSortDesc,
+								Nulls:          dal.IndexFieldNullsLast,
+							},
+						},
+					},
+				},
+			},
+
+			createTableSQL: `
+CREATE TABLE IF NOT EXISTS "simple" (
+  "null_ID" INT     NULL
+, "some_txt" TEXT NOT NULL
+
+, PRIMARY KEY ("null_ID")
+)`,
+
+			createIndexSQL: `
+CREATE INDEX IF NOT EXISTS "first_idx" ON "simple" ((LOWER("null_ID")) DESC)
+CREATE INDEX IF NOT EXISTS "second_idx" ON "simple" ("null_ID", (LOWER("some_txt")) DESC NULLS LAST)
+`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var req = require.New(t)
+
+			tbl, err := ConvertModel(tt.m, tt.d)
+			req.NoError(err)
+
+			req.Equal(
+				strings.TrimSpace(tt.createTableSQL),
+				strings.TrimSpace((&CreateTable{Table: tbl, Dialect: tt.d}).String()),
+			)
+
+			idxSQL := "\n"
+			for i := 1; i < len(tbl.Indexes); i++ {
+				idxSQL += (&CreateIndex{Index: tbl.Indexes[i], Dialect: tt.d}).String() + "\n"
+			}
+
+			req.Equal(tt.createIndexSQL, idxSQL)
+		})
+	}
+}

server/store/adapters/rdbms/ddl/goqu.go

@@ -0,0 +1,23 @@
+package ddl
+
+import "github.com/doug-martin/goqu/v9/exp"
+
+// goqu's little helpers
+
+type (
+	// literalExpression is a wrapper around goqu's LiteralExpression
+	sqlExpression struct {
+		exp.LiteralExpression
+	}
+)
+
+func SQLExpression(e exp.LiteralExpression) exp.SQLExpression {
+	return &sqlExpression{e}
+}
+
+func (e *sqlExpression) ToSQL() (string, []any, error) { return e.Literal(), e.Args(), nil }
+func (e *sqlExpression) IsPrepared() bool              { return e.IsPrepared() }
+
+var (
+	_ exp.SQLExpression = &sqlExpression{}
+)

server/store/adapters/rdbms/ddl/types.go

@@ -0,0 +1,18 @@
+package ddl
+
+import "strings"
+
+func ParseColumnTypes(c *Column) (original, name string, meta []string) {
+	original = strings.ToLower(c.Type.Name)
+
+	pp := strings.Split(original, "(")
+	name = pp[0]
+	if len(pp) > 1 {
+		meta = strings.Split(strings.TrimRight(pp[1], ")"), ",")
+		for i, m := range meta {
+			meta[i] = strings.TrimSpace(m)
+		}
+	}
+
+	return
+}