snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/pkg/ql/ast_nodes.go

@@ -0,0 +1,400 @@
+package ql
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/cortezaproject/corteza/server/pkg/expr"
+	"github.com/spf13/cast"
+)
+
+type (
+	ASTNode struct {
+		pMeta *parserMeta
+
+		// Meta holds additional node data
+		// while processing the AST
+		Meta map[string]any `json:"-"`
+
+		Ref  string     `json:"ref,omitempty"`
+		Args ASTNodeSet `json:"args,omitempty"`
+
+		Symbol string      `json:"symbol,omitempty"`
+		Value  *typedValue `json:"value,omitempty"`
+
+		Raw string `json:"raw,omitempty"`
+	}
+	ASTNodeSet []*ASTNode
+
+	typedValue struct {
+		V expr.TypedValue
+	}
+
+	parserMeta struct {
+		opDef *opDef
+	}
+)
+
+func (n *ASTNode) String() string {
+	if n == nil {
+		return "<nil>"
+	}
+
+	// Leaf edge-cases
+	switch {
+	case n.Symbol != "":
+		return n.Symbol
+	case n.Value != nil:
+		return "\"" + cast.ToString(n.Value.V.Get()) + "\""
+	}
+
+	// Process arguments for the op.
+	args := make([]string, len(n.Args))
+	for i, a := range n.Args {
+		s := a.String()
+		args[i] = s
+	}
+
+	// Default handlers
+	return fmt.Sprintf("%s(%s)", n.Ref, strings.Join(args, ", "))
+}
+
+func MakeValueOf(t string, v interface{}) *typedValue {
+	return &typedValue{
+		V: expr.Must(qlTypeRegistry(t).Cast(v)),
+	}
+}
+
+func WrapValue(v expr.TypedValue) *typedValue {
+	return &typedValue{
+		V: v,
+	}
+}
+
+func (t *typedValue) UnmarshalJSON(in []byte) (err error) {
+	var (
+		aux = struct {
+			Type  string      `json:"@type"`
+			Value interface{} `json:"@value"`
+		}{}
+	)
+
+	if len(in) == 0 {
+		return nil
+	}
+
+	if err = json.Unmarshal(in, &aux); err != nil {
+		return
+	}
+
+	if aux.Type == "" {
+		return fmt.Errorf("invalid value definition: missing @type definition")
+	}
+
+	t.V, err = qlTypeRegistry(aux.Type).Cast(aux.Value)
+	return
+}
+
+func (t *typedValue) MarshalJSON() ([]byte, error) {
+	var (
+		aux = struct {
+			Type  string      `json:"@type"`
+			Value interface{} `json:"@value"`
+		}{}
+	)
+
+	if t.V == nil {
+		return json.Marshal(aux)
+	}
+
+	aux.Type = t.V.Type()
+	aux.Value = t.V.Get()
+
+	switch aux.Type {
+	case "ID", "Record", "User":
+		v := aux.Value.(uint64)
+		aux.Value = strconv.FormatUint(v, 10)
+	}
+
+	return json.Marshal(aux)
+}
+
+func (n *ASTNode) CollectSymbols() (out []string) {
+	out = make([]string, 0, 8)
+
+	n.Traverse(func(a *ASTNode) (bool, *ASTNode, error) {
+		if a.Symbol != "" {
+			out = append(out, a.Symbol)
+		}
+
+		return true, a, nil
+	})
+
+	return
+}
+
+func (n *ASTNode) CollectUniqueSymbols() []string {
+	seen := make(map[string]struct{}, 8)
+	n.Traverse(func(a *ASTNode) (bool, *ASTNode, error) {
+		if a.Symbol != "" {
+			seen[a.Symbol] = struct{}{}
+		}
+		return true, a, nil
+	})
+
+	out := make([]string, 0, len(seen))
+	for s := range seen {
+		out = append(out, s)
+	}
+	return out
+}
+
+// Traverse traverses the AST down to leaf nodes.
+//
+// If fnc. returns false, the traversal of the current branch ends.
+func (n *ASTNode) Traverse(f func(*ASTNode) (bool, *ASTNode, error)) (err error) {
+	var ok bool
+	var r *ASTNode
+	if n == nil {
+		return nil
+	}
+
+	ok, r, err = f(n)
+	if err != nil {
+		return err
+	}
+	*n = *r
+	if !ok {
+		return
+	}
+
+	for _, a := range n.Args {
+		if err = a.Traverse(f); err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func (n ASTNode) Clone() *ASTNode {
+	aa := n.Args
+
+	if n.Args != nil {
+		n.Args = make(ASTNodeSet, len(aa))
+		for i, a := range aa {
+			n.Args[i] = a.Clone()
+		}
+	}
+
+	if n.Value != nil {
+		n.Value = &typedValue{
+			V: n.Value.V,
+		}
+	}
+
+	return &n
+}
+
+func (n lNull) ToAST() (out *ASTNode) {
+	return &ASTNode{
+		Ref: "null",
+	}
+}
+
+func (n lBoolean) ToAST() (out *ASTNode) {
+	return &ASTNode{
+		Value: MakeValueOf("Boolean", n.value),
+	}
+}
+
+func (n lString) ToAST() (out *ASTNode) {
+	return &ASTNode{
+		Value: MakeValueOf("String", n.value),
+	}
+}
+
+// @todo differentiate between floats and others
+func (n lNumber) ToAST() (out *ASTNode) {
+	if isFloaty(n.value) {
+		return &ASTNode{
+			Value: MakeValueOf("Float", n.value),
+		}
+	} else {
+		return &ASTNode{
+			Value: MakeValueOf("Integer", n.value),
+		}
+	}
+}
+
+func (n operator) ToAST() (out *ASTNode) {
+	op := getOp(n.kind)
+
+	return &ASTNode{
+		Ref:   op.name,
+		Args:  make(ASTNodeSet, 0, 2),
+		pMeta: &parserMeta{opDef: op},
+	}
+}
+
+func (n Ident) ToAST() (out *ASTNode) {
+	return &ASTNode{
+		Symbol: n.Value,
+	}
+}
+
+func (n keyword) ToAST() (out *ASTNode) {
+	return &ASTNode{
+		Ref: n.keyword,
+	}
+}
+
+func (n interval) ToAST() (out *ASTNode) {
+	return &ASTNode{
+		Ref: "interval",
+		Args: ASTNodeSet{
+			// @todo consider introducing a new concept on the ASTNode to cover
+			//       system defined symbols/keywords. This is a temporary solutions
+			//       to fix reporting interval definitions; it should eventually be reworked
+			{Value: MakeValueOf("String", n.unit)},
+			{Value: MakeValueOf("Number", n.value)},
+		},
+	}
+}
+
+func (n function) ToAST() (out *ASTNode) {
+	auxA := n.arguments.ToAST()
+
+	return &ASTNode{
+		Ref:  n.name,
+		Args: auxA.Args,
+	}
+}
+
+func (nn parserNodeSet) ToAST() (out *ASTNode) {
+	auxArgs := make(ASTNodeSet, 0, len(nn))
+
+	for _, n := range nn {
+		auxArgs = append(auxArgs, n.ToAST())
+	}
+
+	return &ASTNode{
+		Ref:  "group",
+		Args: auxArgs,
+	}
+}
+
+func (nn parserNodes) ToAST() (out *ASTNode) {
+	// Prep
+	auxArgs := make(ASTNodeSet, 0, len(nn))
+
+	// Convert the entire level to AST nodes
+	for _, n := range nn {
+		auxArgs = append(auxArgs, n.ToAST())
+	}
+
+	// In the current level, have the operators consume operands
+	// based on their defined weight
+	//
+	// - find the highest prio. op and have it consume what it wants
+	// - repeat until all ops are satisfied
+	// -- post optimizations?
+	for {
+		var bestOp *opDef
+		bestOpIx := -1
+
+		// We're done when it is reduced to 1
+		if len(auxArgs) <= 1 {
+			break
+		}
+
+		for _i, _a := range auxArgs {
+			i := _i
+			a := _a
+
+			// use this as a delimiter for now
+			if a.pMeta == nil || a.pMeta.opDef == nil {
+				continue
+			}
+
+			if bestOp == nil {
+				bestOp = a.pMeta.opDef
+				bestOpIx = i
+				continue
+			}
+
+			if a.pMeta.opDef.weight < bestOp.weight {
+				bestOp = a.pMeta.opDef
+				bestOpIx = i
+				continue
+			}
+		}
+
+		// Have the op consume what it needs.
+		arg := auxArgs[bestOpIx]
+		if !isUnary(arg.Ref) {
+			skip := 2
+			arg.Args = append(arg.Args, auxArgs[bestOpIx-1], auxArgs[bestOpIx+1])
+			if bestOpIx > -1 && len(auxArgs) > bestOpIx+2 {
+				if !isOperator(auxArgs[bestOpIx+2].Ref) {
+					skip = 3
+					arg.Args = append(arg.Args, auxArgs[bestOpIx+2])
+				}
+			}
+
+			// this is not needed anymore so we can remove it
+			arg.pMeta = nil
+
+			// Remove the consumed bits and replace it with the new bit
+			aux := auxArgs[0 : bestOpIx-1]
+			aux = append(aux, arg)
+			// +X for right side, +1 because the left index is inclusive
+			aux = append(aux, auxArgs[bestOpIx+skip:]...)
+			auxArgs = aux
+		} else {
+			arg.Args = append(arg.Args, auxArgs[bestOpIx+1])
+			// this is not needed anymore so we can remove it
+			arg.pMeta = nil
+
+			// Remove the consumed bits and replace it with the new bit
+			aux := auxArgs[0:bestOpIx]
+			aux = append(aux, arg)
+			// +1 because the left index is inclusive
+			aux = append(aux, auxArgs[bestOpIx+2:]...)
+			auxArgs = aux
+		}
+	}
+
+	if len(auxArgs) > 1 {
+		return &ASTNode{
+			Ref:  "group",
+			Args: auxArgs,
+		}
+	}
+
+	return auxArgs[0]
+}
+
+// A simplified type registry for the types that QL needs to understand
+func qlTypeRegistry(ref string) expr.Type {
+	switch ref {
+	case "ID", "Record", "User":
+		return &expr.ID{}
+	case "Boolean", "Bool":
+		return &expr.Boolean{}
+	case "Integer":
+		return &expr.Integer{}
+	case "UnsignedInteger":
+		return &expr.UnsignedInteger{}
+	case "Float", "Number":
+		return &expr.Float{}
+	case "String", "Select":
+		return &expr.String{}
+	case "DateTime":
+		return &expr.DateTime{}
+	}
+
+	return nil
+}

server/pkg/ql/ast_nodes_test.go

@@ -0,0 +1,32 @@
+package ql
+
+import (
+	"testing"
+)
+
+// Ensure the parser can parse strings into Statement ASTs.
+func Test_Validators(t *testing.T) {
+	var tests = []struct {
+		tree parserNode
+	}{
+		{
+			tree: parserNodes{
+				Ident{Value: "foo"},
+				operator{kind: "="},
+			},
+		},
+		{
+			tree: parserNodes{
+				operator{kind: "="},
+				Ident{Value: "foo"},
+			},
+		},
+	}
+
+	for i, test := range tests {
+		if err := test.tree.Validate(); err == nil {
+			t.Fatalf("expecting error, got nil:\n"+
+				"      test case: %d. %q", i, test.tree.String())
+		}
+	}
+}

server/pkg/ql/lexer.go

@@ -0,0 +1,112 @@
+package ql
+
+import (
+	"bufio"
+	"io"
+)
+
+// Lexer represents a lexical scanner.
+type (
+	RuneReader interface {
+		read() rune
+		unread()
+	}
+
+	TokenConsumers interface {
+		Test(ch rune) bool
+		Consume(s RuneReader) Token
+	}
+
+	Token struct {
+		code    tokenCode
+		literal string
+		line    uint
+		char    uint
+	}
+
+	Lexer struct {
+		r         *bufio.Reader
+		consumers []TokenConsumers
+		line      uint
+		char      uint
+	}
+)
+
+// eof represents a marker rune for the end of the reader.
+var eof = rune(0)
+
+// NewLexer returns a new instance of Lexer.
+func NewLexer(r io.Reader) *Lexer {
+	return &Lexer{
+		r: bufio.NewReader(r),
+		consumers: []TokenConsumers{
+			&TokenConsumerGeneric{token: WS, whitelist: CHAR_WHITELIST_WHITESPACE},
+			// @todo ensure operator order (eg != is valid, =! is not)
+			&TokenConsumerGeneric{token: OPERATOR, whitelist: CHAR_WHITELIST_OPERATORS},
+			&TokenConsumerGeneric{token: COMMA, whitelist: ",", maxLength: 1},
+			&TokenConsumerGeneric{token: DOT, whitelist: ".", maxLength: 1},
+			&TokenConsumerGeneric{token: PARENTHESIS_OPEN, whitelist: "(", maxLength: 1},
+			&TokenConsumerGeneric{token: PARENTHESIS_CLOSE, whitelist: ")", maxLength: 1},
+			&TokenConsumerString{},
+			&TokenConsumerNumber{},
+			&TokenConsumerIdent{},
+		},
+	}
+}
+
+// Scan returns the next token and literal value.
+func (s *Lexer) Scan() Token {
+	var ch = s.peek()
+
+	if ch == '\n' {
+		s.line++
+		s.char = 0
+	}
+
+	if eof == ch {
+		return Token{code: EOF, line: s.line, char: s.char}
+	}
+
+	for _, c := range s.consumers {
+		if c.Test(ch) {
+			t := c.Consume(s)
+			t.line = s.line
+			t.char = s.char
+			return t
+		}
+	}
+
+	return Token{code: ILLEGAL, literal: string(ch), line: s.line, char: s.char}
+}
+
+// read reads the next rune from the buffered reader.
+// Returns the rune(0) if an error occurs (or io.EOF is returned).
+func (s *Lexer) read() rune {
+	ch, _, err := s.r.ReadRune()
+	s.char++
+	if err != nil {
+		return eof
+	}
+	return ch
+}
+
+func (s *Lexer) peek() rune {
+	bb, err := s.r.Peek(1)
+	if err != nil || len(bb) == 0 {
+		return eof
+	}
+	return rune(bb[0])
+}
+
+// unread places the previously read rune back on the reader.
+func (s *Lexer) unread() { _ = s.r.UnreadRune() }
+
+func (t Token) Is(cc ...tokenCode) bool {
+	for _, c := range cc {
+		if t.code == c {
+			return true
+		}
+	}
+
+	return false
+}

server/pkg/ql/lexer_test.go

@@ -0,0 +1,105 @@
+package ql
+
+import (
+	"strings"
+	"testing"
+)
+
+// Ensure the scanner can scan tokens correctly.
+func TestScanner_ScanSimple(t *testing.T) {
+	var tests = []struct {
+		s   string
+		tok tokenCode
+		lit string
+	}{
+		// Special tokens (EOF, ILLEGAL, WS)
+		{s: ``, tok: EOF},
+		{s: `#`, tok: ILLEGAL, lit: `#`},
+		{s: ` `, tok: WS, lit: " "},
+		{s: "\t", tok: WS, lit: "\t"},
+		{s: "\n", tok: WS, lit: "\n"},
+
+		// Operators
+		{s: `*`, tok: OPERATOR, lit: "*"},
+		{s: `!=`, tok: OPERATOR, lit: "!="},
+		{s: `<`, tok: OPERATOR, lit: "<"},
+		{s: `>`, tok: OPERATOR, lit: ">"},
+		{s: `>=`, tok: OPERATOR, lit: ">="},
+		{s: `<>`, tok: OPERATOR, lit: "<>"},
+		{s: `+`, tok: OPERATOR, lit: "+"},
+		{s: `'fooo'`, tok: LSTRING, lit: "fooo"},
+		{s: `'escaped \' quote'`, tok: LSTRING, lit: "escaped ' quote"},
+		{s: `'double \\ escape'`, tok: LSTRING, lit: "double \\ escape"},
+		{s: `12345`, tok: LNUMBER, lit: "12345"},
+
+		// Identifiers
+		{s: `foo`, tok: IDENT, lit: `foo`},
+		{s: `Zx12_3U_-`, tok: IDENT, lit: `Zx12_3U_`},
+
+		// Parenthesis
+		{s: `(`, tok: PARENTHESIS_OPEN, lit: `(`},
+		{s: `)`, tok: PARENTHESIS_CLOSE, lit: `)`},
+
+		// Literals
+		{s: `true`, tok: LBOOL, lit: `TRUE`},
+	}
+
+	for i, test := range tests {
+		s := NewLexer(strings.NewReader(test.s))
+		tok := s.Scan()
+		if test.tok != tok.code {
+			t.Errorf("%d. %q token mismatch: exp=%d got=%d <%q>", i, test.s, test.tok, tok.code, tok.literal)
+		} else if test.lit != tok.literal {
+			t.Errorf("%d. %q literal mismatch: exp=%q got=%q", i, test.s, test.lit, tok.literal)
+		}
+	}
+}
+
+func TestScanner_ScanComplex(t *testing.T) {
+	var tests = []struct {
+		s      string
+		tokens []tokenCode
+	}{
+		// Special tokens (EOF, ILLEGAL, WS)
+		{`func(arg1, arg2)`,
+			[]tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, WS, IDENT, PARENTHESIS_CLOSE}},
+		{`arg1 * arg2`,
+			[]tokenCode{IDENT, WS, OPERATOR, WS, IDENT}},
+		{`date_format(created_at,'%Y')`,
+			[]tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, COMMA, LSTRING, PARENTHESIS_CLOSE}},
+		{`foo LIKE 'abc%'`,
+			[]tokenCode{IDENT, WS, OPERATOR, WS, LSTRING}},
+		{`foo NOT LIKE 'abc%'`,
+			[]tokenCode{IDENT, WS, OPERATOR, WS, OPERATOR, WS, LSTRING}},
+		{`foo DESC`,
+			[]tokenCode{IDENT, WS, KEYWORD}},
+		{`year(now())-1`,
+			[]tokenCode{IDENT, PARENTHESIS_OPEN, IDENT, PARENTHESIS_OPEN, PARENTHESIS_CLOSE, PARENTHESIS_CLOSE, OPERATOR, LNUMBER}},
+	}
+
+	for _, test := range tests {
+		var tokens []Token
+		s := NewLexer(strings.NewReader(test.s))
+		for {
+			tok := s.Scan()
+			if tok.Is(EOF) {
+				break
+			}
+
+			tokens = append(tokens, tok)
+		}
+
+		if len(tokens) != len(test.tokens) {
+			t.Errorf("Collected tokens do not match (%v)", tokens)
+		}
+
+		for i := 0; i < len(tokens); i++ {
+			if tokens[i].code != test.tokens[i] {
+				t.Errorf("Input:     %s", test.s)
+				t.Errorf("Expected:  %v", test.tokens)
+				t.Errorf("Collected: %v", tokens)
+				break
+			}
+		}
+	}
+}

server/pkg/ql/parser.go

@@ -0,0 +1,309 @@
+package ql
+
+import (
+	"fmt"
+	"strings"
+)
+
+type (
+	// Parser represents a parser.
+	Parser struct {
+		lexer  *Lexer
+		tokbuf []Token
+
+		OnIdent    IdentHandler
+		OnFunction FunctionHandler
+
+		// parenthesis level control
+		level uint
+	}
+
+	IdentHandler    func(ident Ident) (Ident, error)
+	FunctionHandler func(ident function) (parserNode, error)
+)
+
+// NewParser returns a new instance of Parser.
+func NewParser() *Parser {
+	p := &Parser{
+		OnIdent:    func(ident Ident) (Ident, error) { return ident, nil },
+		OnFunction: func(ident function) (parserNode, error) { return ident, nil },
+	}
+
+	return p
+}
+
+// Removes oldest token in the buffer, adds new one and returns 2nd oldest
+func (p *Parser) nextToken() Token {
+	var t Token
+	for {
+		t = p.lexer.Scan()
+		if !t.Is(WS) {
+			p.tokbuf = append(p.tokbuf[1:], t)
+			return p.tokbuf[0]
+		}
+	}
+}
+
+func (p *Parser) peekToken(s int) Token {
+	return p.tokbuf[s]
+}
+
+func (p *Parser) initLexer(s string) {
+	p.lexer = NewLexer(strings.NewReader(s))
+	p.tokbuf = make([]Token, 3)
+
+	for c := 1; c < cap(p.tokbuf); c++ {
+		// Fill the buffer
+		p.nextToken()
+	}
+}
+
+// Parse parses the given expression and returns the generated AST
+func (p *Parser) Parse(s string) (*ASTNode, error) {
+	p.initLexer(s)
+
+	if set, err := p.parse(p.nextToken()); err != nil {
+		return nil, err
+	} else if len(set) == 1 {
+		err := set[0].Validate()
+		if err != nil {
+			return nil, err
+		}
+		return set[0].ToAST(), nil
+	} else {
+		err := set.Validate()
+		if err != nil {
+			return nil, err
+		}
+		return set.ToAST(), nil
+	}
+
+}
+
+// Peek ahead if there is an alias ident (<IDENT:AS>)
+//
+// @todo remove this; not used anywhere after report builder gets removed
+func (p *Parser) peekIfAlias() bool {
+	var f, s = p.peekToken(1), p.peekToken(2)
+	return f.Is(IDENT) && strings.ToUpper(f.literal) == "AS" && s.Is(IDENT)
+}
+
+func (p *Parser) parse(t Token) (list parserNodes, err error) {
+	goto checkToken
+
+next:
+	if p.peekToken(1).Is(COMMA) || p.peekIfAlias() {
+		// Peek ahead and exit on comma
+		return
+	}
+	if p.peekToken(1).Is(PARENTHESIS_CLOSE) {
+		// Peek ahead and exit on closed parenthesis
+		if p.level == 0 {
+			return nil, fmt.Errorf("closing unopened parenthesis in expression")
+		}
+		p.level--
+		return
+	}
+
+	t = p.nextToken()
+
+checkToken:
+	switch t.code {
+	case EOF:
+		break
+	case WS:
+		// Ignore ws... next token...
+		goto next
+	case ILLEGAL:
+		return nil, fmt.Errorf("found an illegal token (%+v)", t)
+	case IDENT:
+		var ident parserNode
+		if ident, err = p.parseIdent(t); err != nil {
+			return nil, err
+		} else {
+			list = append(list, ident)
+			goto next
+		}
+	case LNULL:
+		list = append(list, lNull{})
+		goto next
+	case LBOOL:
+		list = append(list, lBoolean{value: evalBool(t.literal)})
+		goto next
+	case OPERATOR:
+		if len(list) > 0 {
+			// Merge with previous operator node
+			if prevOp, ok := list[len(list)-1].(operator); ok {
+				list[len(list)-1] = operator{kind: prevOp.kind + " " + t.literal}
+				goto next
+			}
+		}
+
+		list = append(list, operator{kind: t.literal})
+		goto next
+	case KEYWORD:
+		if keyword, err := p.parseKeyword(t); err != nil {
+			return nil, err
+		} else {
+			list = append(list, keyword)
+		}
+		goto next
+	case LNUMBER:
+		list = append(list, lNumber{value: t.literal})
+		goto next
+	case LSTRING:
+		list = append(list, lString{value: t.literal})
+		goto next
+	case PARENTHESIS_OPEN:
+		depth := p.level
+		p.level++
+		if sub, err := p.parse(p.nextToken()); err != nil {
+			return nil, err
+		} else {
+			list = append(list, sub)
+		}
+
+		// Allow parent level to continue parsing.
+		// Example: ((A) AND (B))
+		// +1 since PARENTHESIS_CLOSE decrease level
+		if (p.level + 1) != depth {
+			p.nextToken()
+			goto next
+		}
+	default:
+		return nil, fmt.Errorf("unexpected token while parsing expression (%v)", t)
+	}
+
+	return list, nil
+}
+
+func (p *Parser) parseIdent(t Token) (list parserNode, err error) {
+	if p.peekToken(1).Is(PARENTHESIS_OPEN) {
+		// Handle function calls: <IDENT><PARENTHESIS_OPEN>...
+		f := function{name: t.literal}
+		if f.arguments, err = p.parseSet(); err != nil {
+			return nil, err
+		} else {
+			return p.OnFunction(f)
+		}
+	}
+
+	i := Ident{Value: t.literal}
+
+	if p.peekToken(1).Is(DOT) {
+		p.nextToken()
+		i.Value += "."
+		l2 := p.nextToken()
+		if l2.Is(IDENT) {
+			i.Value += l2.literal
+		}
+	}
+
+	return p.OnIdent(i)
+}
+
+func (p *Parser) parseSet() (list parserNodeSet, err error) {
+	var expr parserNodes
+	var parenthesisOpened = false
+
+next:
+	t := p.nextToken()
+
+	if p.peekIfAlias() {
+		return
+	}
+
+	switch t.code {
+	case WS:
+		goto next
+	case PARENTHESIS_OPEN:
+		p.level++
+		parenthesisOpened = true
+		goto next
+	case LNUMBER:
+		list = append(list, lNumber{value: t.literal})
+		goto next
+	case LSTRING:
+		list = append(list, lString{value: t.literal})
+		goto next
+	case OPERATOR:
+		list = append(list, operator{kind: t.literal})
+		goto next
+	case KEYWORD:
+		if keyword, err := p.parseKeyword(t); err != nil {
+			return nil, err
+		} else {
+			list = append(list, keyword)
+		}
+		goto next
+	case IDENT:
+		if p.peekToken(1).Is(OPERATOR) {
+			// Looks like we have an expression ahead of us
+
+			if parenthesisOpened {
+				// Expression will find closing parenthesis and dec. the level
+				// so, lets bump up the number
+				p.level++
+			}
+
+			if expr, err = p.parse(t); err != nil {
+				return
+			}
+
+			list = append(list, expr)
+			goto next
+		}
+
+		var ident parserNode
+		if ident, err = p.parseIdent(t); err != nil {
+			return nil, err
+		} else {
+			list = append(list, ident)
+
+			goto next
+		}
+	case COMMA:
+		goto next
+	case EOF:
+		return
+	case PARENTHESIS_CLOSE:
+		// Peek ahead and exit on closed parenthesis
+		if p.level == 0 {
+			return nil, fmt.Errorf("closing unopened parenthesis in set")
+		}
+		p.level--
+		return
+	default:
+		return nil, fmt.Errorf("unexpected token while parsing set (%v)", t)
+	}
+}
+
+func (p *Parser) parseKeyword(t Token) (list parserNode, err error) {
+	switch strings.ToUpper(t.literal) {
+	case "INTERVAL":
+		i := interval{value: p.nextToken().literal}
+		u := p.nextToken()
+
+		if u.code != IDENT {
+			return nil, fmt.Errorf("expecting identifier, got %v", t)
+		} else {
+			switch strings.ToUpper(u.literal) {
+			case "MICROSECOND", "SECOND", "MINUTE", "HOUR",
+				"DAY", "WEEK", "MONTH", "QUARTER", "YEAR",
+				"SECOND_MICROSECOND", "MINUTE_MICROSECOND", "MINUTE_SECOND", "HOUR_MICROSECOND", "HOUR_SECOND",
+				"HOUR_MINUTE", "DAY_MICROSECOND", "DAY_SECOND", "DAY_MINUTE", "DAY_HOUR", "YEAR_MONTH":
+				// All good
+				break
+			default:
+				return nil, fmt.Errorf("expecting interval unit, got %v", u.literal)
+			}
+		}
+
+		i.unit = u.literal
+
+		return i, nil
+
+	default:
+		return keyword{keyword: t.literal}, nil
+	}
+}

server/pkg/ql/parser_nodes.go

@@ -0,0 +1,221 @@
+package ql
+
+import (
+	"fmt"
+	"strings"
+)
+
+// SelectStatement represents a SQL SELECT statement.
+type (
+	parserNode interface {
+		fmt.Stringer
+
+		Validate() error
+		ToAST() *ASTNode
+	}
+
+	parserNodeSet []parserNode // Stream of comma delimited nodes
+	parserNodes   []parserNode // Stream of space delimited nodes
+
+	lNull    struct{}
+	lBoolean struct{ value bool }
+
+	lString struct {
+		value string
+		args  []interface{}
+	}
+
+	lNumber struct {
+		value string
+	}
+
+	operator struct {
+		kind string
+	}
+
+	Ident struct {
+		Value string
+		args  []interface{}
+	}
+
+	keyword struct {
+		keyword string
+	}
+
+	interval struct {
+		value string
+		unit  string
+	}
+
+	function struct {
+		name      string
+		arguments parserNodeSet
+	}
+
+	opDef struct {
+		name   string
+		weight int
+	}
+)
+
+var (
+	ops = map[string]opDef{
+		// generic comparison
+		`=`:      {name: `eq`, weight: 40},
+		`==`:     {name: `eq`, weight: 40},
+		`===`:    {name: `eq`, weight: 40},
+		`!=`:     {name: `ne`, weight: 40},
+		`!==`:    {name: `ne`, weight: 40},
+		`<>`:     {name: `ne`, weight: 40},
+		`<`:      {name: `lt`, weight: 30},
+		`<=`:     {name: `le`, weight: 30},
+		`>`:      {name: `gt`, weight: 30},
+		`>=`:     {name: `ge`, weight: 30},
+		`IS`:     {name: `is`, weight: 40},
+		`IS NOT`: {name: `nis`, weight: 40},
+		`IN`:     {name: `in`, weight: 40},
+		`NOT IN`: {name: `nin`, weight: 40},
+
+		// conjunction
+		`AND`: {name: `and`, weight: 50},
+		`&&`:  {name: `and`, weight: 50},
+		`OR`:  {name: `or`, weight: 60},
+		`||`:  {name: `or`, weight: 60},
+		`XOR`: {name: `xor`, weight: 60},
+
+		// math
+		`+`: {name: `add`, weight: 20},
+		`-`: {name: `sub`, weight: 20},
+		`*`: {name: `mult`, weight: 10},
+		`/`: {name: `div`, weight: 10},
+
+		// modifiers
+		`!`: {name: `not`, weight: 0},
+
+		// str comp.
+		`LIKE`:     {name: `like`, weight: 40},
+		`NOT LIKE`: {name: `nlike`, weight: 40},
+
+		// range comp.
+		`BETWEEN`:     {name: `between`, weight: 40},
+		`NOT BETWEEN`: {name: `nbetween`, weight: 40},
+	}
+)
+
+func isUnary(s string) bool {
+	return s == "!" || s == "not"
+}
+
+func isOperator(s string) bool {
+	return s == "and" || s == "or" || s == "xor"
+}
+
+func getOp(op string) *opDef {
+	o, ok := ops[strings.ToUpper(op)]
+	if !ok {
+		return nil
+	}
+	return &o
+}
+
+func (n lNull) Validate() (err error) { return }
+func (n lNull) String() string        { return "NULL" }
+
+func (n lBoolean) Validate() (err error) { return }
+func (n lBoolean) String() string {
+	if n.value {
+		return "TRUE"
+	} else {
+		return "FALSE"
+	}
+}
+
+func (n lString) Validate() (err error) { return }
+func (n lString) String() string        { return fmt.Sprintf("%q", n.value) }
+
+func (n lNumber) Validate() (err error) { return }
+func (n lNumber) String() string        { return n.value }
+
+func (n operator) Validate() (err error) {
+	if getOp(n.kind) == nil {
+		return fmt.Errorf("unknown operator '%s'", n.kind)
+	}
+	return
+}
+func (n operator) String() string { return n.kind }
+
+func (n keyword) Validate() (err error) { return }
+func (n keyword) String() string        { return n.keyword }
+
+func (n interval) Validate() (err error) { return }
+func (n interval) String() string        { return fmt.Sprintf("INTERVAL %s %s", n.value, n.unit) }
+
+func (n function) Validate() (err error) { return }
+func (n function) String() string        { return fmt.Sprintf("%s(%s)", n.name, n.arguments) }
+
+func (n Ident) Validate() (err error) { return }
+func (n Ident) String() string        { return n.Value }
+
+func (nn parserNodes) Validate() (err error) {
+	if err = validate(nn); err != nil {
+		return
+	}
+
+	l := len(nn)
+	if l == 0 {
+		return fmt.Errorf("empty set")
+	}
+
+	if op, ok := nn[0].(operator); ok && !isUnary(op.kind) {
+		return fmt.Errorf("malformed expression, unexpected operator '%s' at first node", op)
+	}
+
+	if l > 1 {
+		if op, ok := nn[l-1].(operator); ok {
+			return fmt.Errorf("malformed expression, unexpected operator '%s' at last node", op)
+		}
+	}
+
+	return
+}
+
+func (nn parserNodes) String() (out string) {
+	for i, n := range nn {
+		if i > 0 {
+			out = out + "  "
+		}
+
+		out = out + n.String()
+	}
+
+	return
+}
+
+func (nn parserNodeSet) Validate() (err error) {
+	return validate(nn)
+}
+
+func (nn parserNodeSet) String() (out string) {
+	for i, n := range nn {
+		if i > 0 {
+			out = out + ", "
+		}
+		out = out + n.String()
+	}
+
+	return
+}
+
+func validate(nn []parserNode) (err error) {
+	if len(nn) == 0 {
+		return fmt.Errorf("empty node set")
+	}
+
+	for _, n := range nn {
+		if err = n.Validate(); err != nil {
+			return
+		}
+	}
+
+	return
+}

server/pkg/ql/token_codes.go

@@ -0,0 +1,29 @@
+package ql
+
+type (
+	tokenCode int
+)
+
+const (
+	CHAR_WHITELIST_WHITESPACE = " \n\t"
+	CHAR_WHITELIST_OPERATORS  = "!+-/*=<>&|"
+	CHAR_WHITELIST_QUOTES     = "'"
+)
+
+const (
+	// Special tokens
+	ILLEGAL tokenCode = iota
+	EOF
+	WS // 2
+	IDENT
+	LNULL
+	LBOOL // 4
+	LNUMBER
+	LSTRING
+	COMMA    // ,
+	DOT      // .
+	OPERATOR // + - / *
+	PARENTHESIS_OPEN
+	PARENTHESIS_CLOSE
+	KEYWORD
+)

server/pkg/ql/token_consumers.go

@@ -0,0 +1,159 @@
+package ql
+
+import (
+	"bytes"
+	"strings"
+)
+
+type (
+	TokenConsumerWS       struct{}
+	TokenConsumerIdent    struct{}
+	TokenConsumerOperator struct{}
+	TokenConsumerComma    struct{}
+	TokenConsumerString   struct{}
+	TokenConsumerNumber   struct{}
+	TokenConsumerGeneric  struct {
+		token     tokenCode
+		whitelist string
+		maxLength int
+	}
+)
+
+func in(ch rune, wl string) bool {
+	var w rune
+	for _, w = range wl {
+		if ch == w {
+			return true
+		}
+	}
+	return false
+}
+
+func (g TokenConsumerGeneric) Test(ch rune) bool {
+	return in(ch, g.whitelist)
+}
+
+func (g TokenConsumerGeneric) Consume(s RuneReader) Token {
+	// Create a buffer and read the current character into it.
+	var buf bytes.Buffer
+	buf.WriteRune(s.read())
+
+	// Read every subsequent whitespace character into the buffer.
+	// Non-whitespace characters and EOF will cause the loop to exit.
+	for {
+		if g.maxLength > 0 && buf.Len() >= g.maxLength {
+			// Length control
+			break
+		}
+
+		if ch := s.read(); ch == eof {
+			break
+		} else if !g.Test(ch) {
+			s.unread()
+			break
+		} else {
+			buf.WriteRune(ch)
+		}
+	}
+
+	return Token{code: g.token, literal: buf.String()}
+}
+
+func (i TokenConsumerIdent) Test(ch rune) bool {
+	return isLetter(ch)
+}
+
+// Consumes the current rune and all contiguous ident runes.
+func (TokenConsumerIdent) Consume(s RuneReader) Token {
+	// Create a buffer and read the current character into it.
+	var buf bytes.Buffer
+	buf.WriteRune(s.read())
+
+	// Read every subsequent ident character into the buffer.
+	// Non-ident characters and EOF will cause the loop to exit.
+	for {
+		if ch := s.read(); ch == eof {
+			break
+		} else if !isLetter(ch) && !isDigit(ch) && ch != '_' {
+			s.unread()
+			break
+		} else {
+			_, _ = buf.WriteRune(ch)
+		}
+	}
+
+	lit := strings.ToUpper(buf.String())
+
+	switch lit {
+	case "NULL":
+		return Token{code: LNULL}
+	case "TRUE", "FALSE":
+		return Token{code: LBOOL, literal: lit}
+	case "IS", "LIKE", "NOT", "AND", "OR", "XOR", "IN", "BETWEEN":
+		return Token{code: OPERATOR, literal: lit}
+	case "DESC", "ASC", "INTERVAL":
+		return Token{code: KEYWORD, literal: lit}
+	}
+
+	// Otherwise return as a regular identifier.
+	return Token{code: IDENT, literal: buf.String()}
+}
+
+func (str TokenConsumerString) Test(ch rune) bool {
+	return in(ch, CHAR_WHITELIST_QUOTES)
+}
+
+// Consumes entire string (skipping quotes)
+func (str TokenConsumerString) Consume(s RuneReader) Token {
+	var buf bytes.Buffer
+	var escaping = false
+	var ch = s.read() // skip quite
+
+	for {
+		if ch = s.read(); ch == eof {
+			break
+		} else if !escaping && str.Test(ch) { // test for quote
+			return Token{code: LSTRING, literal: buf.String()}
+		} else {
+			escaping = !escaping && ch == '\\'
+			if !escaping {
+				// Add char to buffer if not escaping
+				_, _ = buf.WriteRune(ch)
+			}
+		}
+	}
+
+	// This string did not end properly (with an enclosing quote).
+	return Token{code: ILLEGAL, literal: buf.String() + string(ch)}
+}
+
+func (str TokenConsumerNumber) Test(ch rune) bool {
+	return isDigit(ch)
+}
+
+// Consumes entire number (very naive and simplified)
+func (str TokenConsumerNumber) Consume(s RuneReader) Token {
+	// Create a buffer and read the current character into it.
+	var buf bytes.Buffer
+	buf.WriteRune(s.read())
+
+	for {
+		if ch := s.read(); ch == eof {
+			break
+		} else if !isDigit(ch) {
+			s.unread()
+			break
+		} else {
+			_, _ = buf.WriteRune(ch)
+		}
+	}
+
+	// Otherwise return as a regular identifier.
+	return Token{code: LNUMBER, literal: buf.String()}
+}
+
+// isLetter returns true if the rune is a letter.
+func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') }
+
+// isDigit returns true if the rune is a digit.
+func isDigit(ch rune) bool { return ch >= '0' && ch <= '9' }

server/pkg/ql/util.go

@@ -0,0 +1,56 @@
+package ql
+
+import (
+	"regexp"
+	"strings"
+)
+
+var (
+	truthy = regexp.MustCompile(`^(t(rue)?|y(es)?|1)$`)
+)
+
+// Check what boolean value the given string conforms to
+func evalBool(v string) bool {
+	return truthy.MatchString(strings.ToLower(v))
+}
+
+// Check if the given string is a float
+func isFloaty(v string) bool {
+	return strings.Contains(v, ".")
+}
+
+func MergeAnd(a, b *ASTNode) *ASTNode {
+	return merger(a, b, "and")
+}
+
+func MergeOr(a, b *ASTNode) *ASTNode {
+	return merger(a, b, "or")
+}
+
+func merger(a, b *ASTNode, ref string) *ASTNode {
+	// 1. merge the two
+	aa := make(ASTNodeSet, 0, 2)
+
+	// It needs to be under a group, so we get an `(a) and/or (b)`
+	if a != nil {
+		aa = append(aa, &ASTNode{Ref: "group", Args: ASTNodeSet{a}})
+	}
+	if b != nil {
+		aa = append(aa, &ASTNode{Ref: "group", Args: ASTNodeSet{b}})
+	}
+
+	// 2. flatten
+	// @todo do some more in-depth processing?
+	if len(aa) == 1 {
+		// this [0][0] will always hold if we get to this point
+		return aa[0].Args[0]
+	}
+	if len(aa) == 0 {
+		return nil
+	}
+
+	return &ASTNode{
+		Ref:  ref,
+		Args: aa,
+	}
+}