snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)
Some checks failed
SonarQube / SonarQube (push) Failing after 5m28s
E2e test on main branch push / server and client test (map[name:admin port:8080]) (push) Failing after 3m33s
Server and lib tests / Library test (js) (push) Failing after 1m36s
Server and lib tests / Library test (vue) (push) Successful in 2m24s
Server and lib tests / Web client test (admin) (push) Successful in 2m52s
Server and lib tests / Web client test (compose) (push) Successful in 3m23s
Server and lib tests / Web client test (discovery) (push) Successful in 2m49s
Server and lib tests / Web client test (one) (push) Successful in 2m52s
Server and lib tests / Web client test (privacy) (push) Successful in 2m44s
Server and lib tests / Web client test (reporter) (push) Successful in 2m56s
Server and lib tests / Web client test (workflow) (push) Successful in 3m0s
Server and lib tests / Server tests (push) Failing after 4m55s
E2e test on main branch push / notify-failure (push) Failing after 9s
Server and lib tests / Send matrix testing notification (push) Successful in 2s
Mark stale issues and pull requests / stale (push) Successful in 40s

This commit is contained in:
upstream-mirror
2026-09-18 01:22:23 +05:00
commit 7e385fc126
8391 changed files with 1766236 additions and 0 deletions

View File

@@ -0,0 +1,41 @@
package ctx
import (
"context"
"github.com/cortezaproject/corteza/server/pkg/apigw/profiler"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
)
type (
scopeCtxKey struct{}
profilerCtxKey struct{}
)
func ScopeToContext(ctx context.Context, s *types.Scp) context.Context {
return context.WithValue(ctx, scopeCtxKey{}, s)
}
func ScopeFromContext(ctx context.Context) (ss *types.Scp) {
s := ctx.Value(scopeCtxKey{})
if s == nil {
return &types.Scp{}
}
return s.(*types.Scp)
}
func ProfilerToContext(ctx context.Context, h interface{}) context.Context {
return context.WithValue(ctx, profilerCtxKey{}, h)
}
func ProfilerFromContext(ctx context.Context) (h *profiler.Hit) {
hh := ctx.Value(profilerCtxKey{})
if hh == nil {
return nil
}
return hh.(*profiler.Hit)
}

View File

@@ -0,0 +1,26 @@
package filter
import (
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
)
const (
PreFilterWeight = iota
ProcesserWeight
PostFilterWeight
)
func FilterWeight(w int, t types.FilterKind) int {
mul := PreFilterWeight
switch t {
case types.PreFilter:
mul = PreFilterWeight
case types.Processer:
mul = ProcesserWeight
case types.PostFilter:
mul = PostFilterWeight
}
return mul*100 + w
}

View File

@@ -0,0 +1,309 @@
package filter
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/url"
"reflect"
"strings"
atypes "github.com/cortezaproject/corteza/server/automation/types"
agctx "github.com/cortezaproject/corteza/server/pkg/apigw/ctx"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
errors "github.com/cortezaproject/corteza/server/pkg/errors"
"github.com/cortezaproject/corteza/server/pkg/expr"
)
type (
typesRegistry interface {
Type(ref string) expr.Type
}
redirection struct {
types.FilterMeta
location *url.URL
status int
cfg types.Config
params struct {
HTTPStatus int `json:"status,string"`
Location string `json:"location"`
}
}
response struct {
types.FilterMeta
reg typesRegistry
cfg types.Config
params struct {
Header http.Header `json:"header"`
Exp *atypes.Expr `json:"input"`
Evaluable expr.Evaluable
}
}
defaultJsonResponse struct {
types.FilterMeta
cfg types.Config
}
)
func NewRedirection(cfg types.Config) (e *redirection) {
e = &redirection{}
e.Name = "redirection"
e.Label = "Redirection"
e.Kind = types.PostFilter
e.cfg = cfg
e.Args = []*types.FilterMetaArg{
{
Type: "status",
Label: "status",
Options: map[string]interface{}{},
},
{
Type: "text",
Label: "location",
Options: map[string]interface{}{},
},
}
return
}
func (h redirection) New(cfg types.Config) types.Handler {
return NewRedirection(cfg)
}
func (h redirection) Enabled() bool {
return true
}
func (h redirection) String() string {
return fmt.Sprintf("apigw filter %s (%s)", h.Name, h.Label)
}
func (h redirection) Meta() types.FilterMeta {
return h.FilterMeta
}
func (h redirection) Weight() int {
return h.Wgt
}
func (h *redirection) Merge(params []byte, cfg types.Config) (types.Handler, error) {
err := json.NewDecoder(bytes.NewBuffer(params)).Decode(&h.params)
if err != nil {
return h, err
}
loc, err := url.ParseRequestURI(h.params.Location)
if err != nil {
return nil, fmt.Errorf("could not validate parameters, invalid URL: %s", err)
}
if !checkStatus("redirect", h.params.HTTPStatus) {
return nil, fmt.Errorf("could not validate parameters, wrong status %d", h.params.HTTPStatus)
}
h.location = loc
h.status = h.params.HTTPStatus
h.cfg = cfg
return h, err
}
func (h redirection) Handler() types.HandlerFunc {
return func(rw http.ResponseWriter, r *http.Request) error {
http.Redirect(rw, r, h.location.String(), h.status)
return nil
}
}
func NewDefaultJsonResponse(cfg types.Config) (e *defaultJsonResponse) {
e = &defaultJsonResponse{}
e.Name = "defaultJsonResponse"
e.Label = "Default JSON response"
e.Kind = types.PostFilter
e.cfg = cfg
return
}
func (j defaultJsonResponse) New(cfg types.Config) types.Handler {
return NewDefaultJsonResponse(cfg)
}
func (j defaultJsonResponse) Enabled() bool {
return true
}
func (j defaultJsonResponse) String() string {
return fmt.Sprintf("apigw filter %s (%s)", j.Name, j.Label)
}
func (j defaultJsonResponse) Meta() types.FilterMeta {
return j.FilterMeta
}
func (j *defaultJsonResponse) Merge(params []byte, cfg types.Config) (h types.Handler, err error) {
j.cfg = cfg
return j, err
}
func (j defaultJsonResponse) Handler() types.HandlerFunc {
return func(rw http.ResponseWriter, r *http.Request) error {
rw.Header().Set("Content-Type", "application/json")
rw.WriteHeader(http.StatusAccepted)
if _, err := rw.Write([]byte(`{}`)); err != nil {
return errors.Internal("could not write to body: %v", err)
}
return nil
}
}
func checkStatus(typ string, status int) bool {
switch typ {
case "redirect":
return status >= 300 && status <= 399
default:
return true
}
}
func NewResponse(cfg types.Config, reg typesRegistry) (e *response) {
e = &response{}
e.Name = "response"
e.Label = "Response"
e.Kind = types.PostFilter
e.Args = []*types.FilterMetaArg{
{
Type: "input",
Label: "input",
Options: map[string]interface{}{},
},
{
Type: "header",
Label: "header",
Options: map[string]interface{}{},
},
}
e.reg = reg
e.cfg = cfg
return
}
func (j response) New(cfg types.Config) types.Handler {
return NewResponse(cfg, j.reg)
}
func (j response) Enabled() bool {
return true
}
func (j response) String() string {
return fmt.Sprintf("apigw filter %s (%s)", j.Name, j.Label)
}
func (j response) Meta() types.FilterMeta {
return j.FilterMeta
}
func (j *response) Merge(params []byte, cfg types.Config) (h types.Handler, err error) {
var (
parser = expr.NewParser()
)
err = json.NewDecoder(bytes.NewBuffer(params)).Decode(&j.params)
if err != nil {
return j, err
}
j.params.Evaluable, err = parser.Parse(j.params.Exp.Expr)
if err != nil {
return j, fmt.Errorf("could not evaluate expression: %s", err)
}
j.params.Exp.SetEval(j.params.Evaluable)
j.cfg = cfg
return j, err
}
func (j response) Handler() types.HandlerFunc {
return func(rw http.ResponseWriter, r *http.Request) (err error) {
var (
ctx = r.Context()
scope = agctx.ScopeFromContext(ctx)
hasJsonHeader = false
evald interface{}
)
in, err := expr.NewVars(scope.Dict())
if err != nil {
return errors.Internal("could not validate request data: %v", err)
}
// set type to the registered expression from
// any of the already registered types
j.params.Exp.SetType(func(name string) (e expr.Type, err error) {
if name == "" {
name = "Any"
}
if typ := j.reg.Type(name); typ != nil {
return typ, nil
} else {
return nil, errors.Internal("unknown or unregistered type %s", name)
}
})
evald, err = j.params.Exp.Eval(ctx, in)
if err != nil {
return
}
for h, v := range j.params.Header {
for _, vv := range v {
rw.Header().Add(h, vv)
if strings.ToLower(h) == "content-type" {
hasJsonHeader = vv == "application/json"
}
}
}
if hasJsonHeader && reflect.ValueOf(evald).Kind() != reflect.String {
err = (json.NewEncoder(rw)).Encode(expr.UntypedValue(evald))
return
}
fmt.Fprintf(rw, "%v", evald)
return
}
}

View File

@@ -0,0 +1,207 @@
package filter
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
agctx "github.com/cortezaproject/corteza/server/pkg/apigw/ctx"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
"github.com/cortezaproject/corteza/server/pkg/expr"
"github.com/stretchr/testify/require"
)
type (
mockHandlerRegistry struct{}
)
func Test_redirectionMerge(t *testing.T) {
var (
tcc = []tf{
{
name: "url validation",
expr: `{"status":"301", "location": "invalid url"}`,
err: `could not validate parameters, invalid URL: parse "invalid url": invalid URI for request`,
},
{
name: "invalid redirection status",
expr: `{"status":"400", "location": "http://redire.ct/to"}`,
err: "could not validate parameters, wrong status 400",
},
}
)
for _, tc := range tcc {
t.Run(tc.name, testMerge(NewRedirection(types.Config{}), tc))
}
}
func Test_redirection(t *testing.T) {
type (
tf struct {
name string
expr string
err string
loc string
code int
}
)
var (
tcc = []tf{
{
name: "simple redirection",
expr: `{"status":"302", "location": "http://redire.ct/to"}`,
loc: "http://redire.ct/to",
code: 302,
},
{
name: "permanent redirection",
expr: `{"status":"301", "location": "http://redire.ct/to"}`,
loc: "http://redire.ct/to",
code: 301,
},
}
)
for _, tc := range tcc {
t.Run(tc.name, func(t *testing.T) {
var (
req = require.New(t)
r = httptest.NewRequest(http.MethodGet, "/foo", http.NoBody)
rc = httptest.NewRecorder()
cfg = types.Config{}
)
h := getHandler(NewRedirection(cfg))
h, err := h.Merge([]byte(tc.expr), cfg)
req.NoError(err)
hn := h.Handler()
err = hn(rc, r)
if tc.err != "" {
req.EqualError(err, tc.err)
return
}
req.NoError(err)
req.Equal(tc.loc, rc.Header().Get("Location"))
req.Equal(tc.code, rc.Code)
})
}
}
func Test_jsonResponse(t *testing.T) {
type (
tf struct {
name string
expr string
err string
exp string
scope interface{}
}
aux struct {
Name string `json:"name"`
Surname string `json:"surname"`
}
)
var (
tcc = []tf{
{
name: "String response as JSON",
expr: `{"header":{"content-type":["application/json"]},"input":{"expr": "records", "type": "String"}}`,
scope: expr.Must(expr.Any{}.Cast("foobar")),
exp: `foobar`,
},
{
name: "Array response as JSON",
expr: `{"header":{"content-type":["application/json"]},"input":{"expr": "records", "type": "Array"}}`,
scope: expr.Must(expr.Any{}.Cast([]float64{3.14, 42.690})),
exp: `[3.14,42.69]`,
},
{
name: "Array response as text",
expr: `{"input":{"expr": "records", "type": "Array"}}`,
scope: expr.Must(expr.Any{}.Cast([]float64{3.14, 42.690})),
exp: `[3.14 42.69]`,
},
{
name: "Any response as JSON",
expr: `{"header":{"content-type":["application/json"]},"input": {"expr": "records", "type": "Any"}}`,
scope: expr.Must(expr.Any{}.Cast(map[string]string{"foo": "bar", "baz": "bzz"})),
exp: `{"baz":"bzz","foo":"bar"}`,
},
{
name: "Any response as text",
expr: `{"input": {"expr": "records", "type": "Any"}}`,
scope: expr.Must(expr.Any{}.Cast(map[string]string{"foo": "bar", "baz": "bzz"})),
exp: `map[baz:bzz foo:bar]`,
},
{
name: "struct array response as JSON",
expr: `{"input":{"expr": "toJSON(records)", "type": "String"}}`,
scope: []aux{{"First", "Last"}, {"Foo", "bar"}},
exp: `[{"name":"First","surname":"Last"},{"name":"Foo","surname":"bar"}]`,
},
{
name: "struct array response as text",
expr: `{"input":{"expr": "records", "type": "String"}}`,
scope: []aux{{"First", "Last"}, {"Foo", "bar"}},
exp: `[{First Last} {Foo bar}]`,
},
{
name: "string csv response as text",
expr: `{"header":{"content-type":["application/octet-stream"],"Content-Disposition":["attachment; filename=foo.txt"],"Content-Transfer-Encoding":["binary"]},"input":{"expr": "records", "type": "String"}}`,
scope: "\"header 1\",\"header 2\"\nvalue 1,value 2\nvalue 3, value 4",
exp: "\"header 1\",\"header 2\"\nvalue 1,value 2\nvalue 3, value 4",
},
}
)
for _, tc := range tcc {
t.Run(tc.name, func(t *testing.T) {
var (
req = require.New(t)
r = httptest.NewRequest(http.MethodGet, "/foo", http.NoBody)
rc = httptest.NewRecorder()
scope = &types.Scp{"records": tc.scope}
cfg = types.Config{}
)
r = r.WithContext(agctx.ScopeToContext(context.Background(), scope))
h := getHandler(NewResponse(cfg, &mockHandlerRegistry{}))
h, err := h.Merge([]byte(tc.expr), cfg)
req.NoError(err)
hn := h.Handler()
err = hn(rc, r)
if tc.err != "" {
req.EqualError(err, tc.err)
return
}
req.NoError(err)
req.Equal(tc.exp, strings.TrimSuffix(rc.Body.String(), "\n"))
})
}
}
// hackity hack
func getHandler(h types.Handler) types.Handler {
return h
}
func (r *mockHandlerRegistry) Type(ref string) expr.Type {
return expr.Any{}
}

View File

@@ -0,0 +1,298 @@
package filter
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"time"
agctx "github.com/cortezaproject/corteza/server/pkg/apigw/ctx"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
pe "github.com/cortezaproject/corteza/server/pkg/errors"
"github.com/cortezaproject/corteza/server/pkg/expr"
)
type (
header struct {
types.FilterMeta
eval expr.Evaluable
cfg types.Config
params struct {
Expr string `json:"expr"`
}
}
queryParam struct {
types.FilterMeta
eval expr.Evaluable
cfg types.Config
params struct {
Expr string `json:"expr"`
}
}
origin struct {
types.FilterMeta
eval expr.Evaluable
cfg types.Config
params struct {
Expr string `json:"expr"`
}
}
profiler struct {
cfg types.Config
types.FilterMeta
}
)
func NewHeader(cfg types.Config) (v *header) {
v = &header{}
v.Name = "header"
v.Label = "Header"
v.Kind = types.PreFilter
v.Args = []*types.FilterMetaArg{
{
Type: "expr",
Label: "expr",
Options: map[string]interface{}{},
},
}
v.cfg = cfg
return
}
func (h header) New(cfg types.Config) types.Handler {
return NewHeader(cfg)
}
func (h header) Enabled() bool {
return true
}
func (h header) String() string {
return fmt.Sprintf("apigw filter %s (%s)", h.Name, h.Label)
}
func (h header) Meta() types.FilterMeta {
return h.FilterMeta
}
func (h *header) Merge(params []byte, cfg types.Config) (types.Handler, error) {
err := json.NewDecoder(bytes.NewBuffer(params)).Decode(&h.params)
if err != nil {
return nil, err
}
parser := expr.NewParser()
h.cfg = cfg
h.eval, err = parser.Parse(h.params.Expr)
if err != nil {
return nil, fmt.Errorf("could not validate origin parameters: %s", err)
}
return h, err
}
func (h header) Handler() types.HandlerFunc {
return func(rw http.ResponseWriter, r *http.Request) error {
var (
ctx = r.Context()
)
vv := map[string]interface{}{}
headers := r.Header
for k, v := range headers {
vv[k] = v[0]
}
// get the request data and put it into vars
out, err := expr.NewVars(vv)
if err != nil {
return pe.Internal("could not validate headers: %v", err)
}
err = out.Set("headers", vv)
if err != nil {
return pe.Internal("could not set headers: %v", err)
}
b, err := h.eval.Test(ctx, out)
if err != nil {
return pe.InvalidData("could not validate headers: %v", err)
}
if !b {
return pe.InvalidData("could not validate headers: %v", fmt.Errorf("validation failed"))
}
return nil
}
}
func NewQueryParam(cfg types.Config) (v *queryParam) {
v = &queryParam{}
v.Name = "queryParam"
v.Label = "Query parameters"
v.Kind = types.PreFilter
v.Args = []*types.FilterMetaArg{
{
Type: "expr",
Label: "expr",
Options: map[string]interface{}{},
},
}
v.cfg = cfg
return
}
func (qp queryParam) New(cfg types.Config) types.Handler {
return NewQueryParam(cfg)
}
func (qp queryParam) Enabled() bool {
return true
}
func (qp queryParam) String() string {
return fmt.Sprintf("apigw filter %s (%s)", qp.Name, qp.Label)
}
func (qp queryParam) Meta() types.FilterMeta {
return qp.FilterMeta
}
func (qp *queryParam) Merge(params []byte, cfg types.Config) (types.Handler, error) {
err := json.NewDecoder(bytes.NewBuffer(params)).Decode(&qp.params)
if err != nil {
return nil, err
}
parser := expr.NewParser()
qp.cfg = cfg
qp.eval, err = parser.Parse(qp.params.Expr)
if err != nil {
return nil, fmt.Errorf("could not validate query parameters: %s", err)
}
return qp, err
}
func (qp *queryParam) Handler() types.HandlerFunc {
return func(rw http.ResponseWriter, r *http.Request) error {
var (
ctx = r.Context()
)
vv := map[string]interface{}{}
vals := r.URL.Query()
for k, v := range vals {
vv[k] = v[0]
}
// get the request data and put it into vars
out, err := expr.NewVars(vv)
if err != nil {
return pe.Internal("could not validate query parameters: %v", err)
}
err = out.Set("params", vv)
if err != nil {
return pe.Internal("could not set params: %v", err)
}
b, err := qp.eval.Test(ctx, out)
if err != nil {
return pe.InvalidData("could not validate query parameters: %v", err)
}
if !b {
return pe.InvalidData("could not validate query parameters: %v", fmt.Errorf("validation failed"))
}
return nil
}
}
func NewProfiler(cfg types.Config) (pp *profiler) {
pp = &profiler{}
pp.cfg = cfg
pp.Name = "profiler"
pp.Label = "Profiler"
pp.Kind = types.PreFilter
return
}
func (pr *profiler) New(cfg types.Config) types.Handler {
return NewProfiler(cfg)
}
func (pr *profiler) Enabled() bool {
return pr.cfg.Profiler.Enabled
}
func (pr *profiler) String() string {
return fmt.Sprintf("apigw filter %s (%s)", pr.Name, pr.Label)
}
func (pr *profiler) Meta() types.FilterMeta {
return pr.FilterMeta
}
func (pr *profiler) Merge(params []byte, cfg types.Config) (types.Handler, error) {
pr.cfg = cfg
return pr, nil
}
func (pr *profiler) Handler() types.HandlerFunc {
return func(rw http.ResponseWriter, r *http.Request) (err error) {
var (
ctx = r.Context()
scope = agctx.ScopeFromContext(ctx)
)
if pr.cfg.Profiler.Global {
// profiler global overrides any profiling prefilter
// the hit is registered on lower level
return
}
var (
n = time.Now()
hit = agctx.ProfilerFromContext(r.Context())
)
if hit == nil {
return
}
hit.Ts = &n
hit.R = scope.Request()
r = r.WithContext(agctx.ProfilerToContext(r.Context(), hit))
return
}
}

View File

@@ -0,0 +1,254 @@
package filter
import (
"net/http"
"net/http/httptest"
"testing"
agctx "github.com/cortezaproject/corteza/server/pkg/apigw/ctx"
prf "github.com/cortezaproject/corteza/server/pkg/apigw/profiler"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
h "github.com/cortezaproject/corteza/server/pkg/http"
"github.com/stretchr/testify/require"
)
type (
tf struct {
name string
expr string
err string
url string
o string
headers http.Header
}
)
func Test_headerMerge(t *testing.T) {
var (
tcc = []tf{
{
name: "non matching key",
expr: `{"expr":"Foo1 == bar\""}`,
headers: map[string][]string{"Foo": {"bar"}},
err: "could not validate origin parameters: parsing error: Foo1 == bar\"\t:1:12 - 1:13 unexpected String while scanning operator",
},
}
)
for _, tc := range tcc {
t.Run(tc.name, testMerge(NewHeader(types.Config{}), tc))
}
}
func Test_headerHandle(t *testing.T) {
var (
tcc = []tf{
{
name: "matching simple",
expr: `{"expr":"foo == \"bar\""}`,
headers: map[string][]string{"foo": {"bar"}},
},
{
name: "matching case",
expr: `{"expr":"Foo == \"bar\""}`,
headers: map[string][]string{"Foo": {"bar"}},
},
{
name: "non matching value",
expr: `{"expr":"Foo == \"bar1\""}`,
headers: map[string][]string{"Foo": {"bar"}},
err: `could not validate headers: validation failed`,
},
{
name: "non matching key",
expr: `{"expr":"Foo1 == \"bar\""}`,
headers: map[string][]string{"Foo": {"bar"}},
err: `could not validate headers: failed to select 'Foo1' on *expr.Vars: no such key 'Foo1'`,
},
{
name: "regex matching key",
expr: `{"expr":"match(Foo, \"^b\\\\wr\\\\s.*$\")"}`,
headers: map[string][]string{"Foo": {"bar "}},
},
{
name: "matching header with hyphen",
expr: `{"expr":"headers[\"Content-type\"] == \"application/json\""}`,
headers: map[string][]string{"Content-type": {"application/json"}},
},
}
)
for _, tc := range tcc {
r := httptest.NewRequest(http.MethodGet, "/foo", http.NoBody)
r.Header = tc.headers
t.Run(tc.name, testHandle(NewHeader(types.Config{}), r, tc))
}
}
func Test_queryParamMerge(t *testing.T) {
var (
tcc = []tf{
{
name: "matching simple query parameter",
expr: `{"expr":"foo == \"bar\""}`,
url: "https://examp.le?foo=bar",
},
{
name: "matching simple query parameter - invalid expression key",
expr: `{"expr1":"foo == \"bar\""}`,
url: "https://examp.le?foo=bar",
err: "could not validate query parameters: parsing error: - 1:1 unexpected EOF while scanning extensions",
},
{
name: "matching simple query parameter - missing value",
expr: `{"expr":"foo == \"bar\""}`,
url: "https://examp.le?foo=bar1",
},
{
name: "matching simple query parameter - missing value",
expr: `{"expr":"foo == \"bar-baz\""}`,
url: "https://examp.le?foo=bar-baz",
},
}
)
for _, tc := range tcc {
t.Run(tc.name, testMerge(NewQueryParam(types.Config{}), tc))
}
}
func Test_queryParamHandle(t *testing.T) {
var (
tcc = []tf{
{
name: "matching simple query parameter",
expr: `{"expr":"foo == \"bar\""}`,
url: "https://examp.le?foo=bar",
},
{
name: "matching simple query parameter - missing value",
expr: `{"expr":"foo == \"bar\""}`,
url: "https://examp.le?foo=bar1",
err: `could not validate query parameters: validation failed`,
},
{
name: "matching query parameter",
expr: `{"expr":"foo == \"bar-baz\""}`,
url: "https://examp.le?foo=bar-baz",
},
{
name: "matching query parameter",
expr: `{"expr":"params[\"foo-bar\"] == \"bar-baz\""}`,
url: "https://examp.le?foo-bar=bar-baz",
},
}
)
for _, tc := range tcc {
r := httptest.NewRequest(http.MethodGet, tc.url, http.NoBody)
t.Run(tc.name, testHandle(NewQueryParam(types.Config{}), r, tc))
}
}
func Test_profilerHandle_profilerGlobal(t *testing.T) {
type (
tfp struct {
name string
cfg types.Config
r *http.Request
exp *h.Request
}
)
var (
rr = httptest.NewRequest(http.MethodGet, "/foo", http.NoBody)
tcc = []tfp{
{
name: "skip profiler hit on profiler global = true",
cfg: types.Config{Profiler: struct {
Enabled bool
Global bool
}{Global: true}},
r: rr,
exp: nil,
},
{
name: "add profiler hit on profiler global = false",
cfg: types.Config{Profiler: struct {
Enabled bool
Global bool
}{Global: false}},
r: rr,
exp: createRequest(rr),
},
}
)
for _, tc := range tcc {
var (
req = require.New(t)
ph = NewProfiler(tc.cfg)
hfn = ph.Handler()
hr = createRequest(tc.r)
hit = &prf.Hit{}
)
req.Nil(hit.R)
tc.r = tc.r.WithContext(agctx.ScopeToContext(tc.r.Context(), &types.Scp{"request": hr}))
tc.r = tc.r.WithContext(agctx.ProfilerToContext(tc.r.Context(), hit))
err := hfn(httptest.NewRecorder(), tc.r)
req.NoError(err)
scoped := agctx.ProfilerFromContext(tc.r.Context())
req.Equal(tc.exp, scoped.R)
}
}
func createRequest(r *http.Request) (hr *h.Request) {
hr, _ = h.NewRequest(r)
return
}
func testMerge(h types.Handler, tc tf) func(t *testing.T) {
return func(t *testing.T) {
var (
req = require.New(t)
)
_, err := h.Merge([]byte(tc.expr), types.Config{})
if tc.err != "" {
req.EqualError(err, tc.err)
} else {
req.NoError(err)
}
}
}
func testHandle(h types.Handler, r *http.Request, tc tf) func(t *testing.T) {
return func(t *testing.T) {
var (
req = require.New(t)
)
h, err := h.Merge([]byte(tc.expr), types.Config{})
req.NoError(err)
hfn := h.Handler()
err = hfn(httptest.NewRecorder(), r)
if tc.err != "" {
req.EqualError(err, tc.err)
} else {
req.NoError(err)
}
}
}

View File

@@ -0,0 +1,303 @@
package filter
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"strconv"
"github.com/cortezaproject/corteza/server/automation/automation"
atypes "github.com/cortezaproject/corteza/server/automation/types"
agctx "github.com/cortezaproject/corteza/server/pkg/apigw/ctx"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
"github.com/cortezaproject/corteza/server/pkg/auth"
pe "github.com/cortezaproject/corteza/server/pkg/errors"
"github.com/cortezaproject/corteza/server/pkg/expr"
"github.com/cortezaproject/corteza/server/pkg/jsenv"
"go.uber.org/zap"
)
type (
workflow struct {
types.FilterMeta
d WfExecer
cfg types.Config
params struct {
Workflow uint64 `json:"workflow,string"`
}
}
WfExecer interface {
Load(ctx context.Context) error
Exec(ctx context.Context, workflowID uint64, p atypes.WorkflowExecParams) (*expr.Vars, uint64, atypes.Stacktrace, error)
Search(ctx context.Context, filter atypes.WorkflowFilter) (atypes.WorkflowSet, atypes.WorkflowFilter, error)
}
processerPayload struct {
types.FilterMeta
vm jsenv.Vm
fn *jsenv.Fn
log *zap.Logger
cfg types.Config
params struct {
Func string `json:"jsfunc"`
Encode bool `json:"encode"`
}
}
)
func NewWorkflow(cfg types.Config, wf WfExecer) (p *workflow) {
p = &workflow{}
p.d = wf
p.cfg = cfg
p.Name = "workflow"
p.Label = "Workflow processer"
p.Kind = types.Processer
p.Args = []*types.FilterMetaArg{
{
Type: "workflow",
Label: "workflow",
Options: map[string]interface{}{},
},
}
return
}
func (h workflow) New(cfg types.Config) types.Handler {
return NewWorkflow(cfg, h.d)
}
func (h workflow) Enabled() bool {
return true
}
func (h workflow) String() string {
return fmt.Sprintf("apigw filter %s (%s)", h.Name, h.Label)
}
func (h workflow) Meta() types.FilterMeta {
return h.FilterMeta
}
func (h *workflow) Merge(params []byte, cfg types.Config) (types.Handler, error) {
var (
t = struct {
Workflow string `json:"workflow"`
}{}
ctx = auth.SetIdentityToContext(context.Background(), auth.ServiceUser())
)
h.cfg = cfg
err := json.NewDecoder(bytes.NewBuffer(params)).Decode(&t)
if err != nil {
return h, err
}
i, err := strconv.Atoi(t.Workflow)
if err == nil {
h.params.Workflow = uint64(i)
return h, h.d.Load(ctx)
}
if wf, _, err := h.d.Search(ctx, atypes.WorkflowFilter{Query: t.Workflow}); err != nil {
return h, err
} else {
h.params.Workflow = wf[0].ID
}
// preload workflow cache
return h, h.d.Load(ctx)
}
func (h workflow) Handler() types.HandlerFunc {
return func(rw http.ResponseWriter, r *http.Request) error {
var (
err error
ctx = r.Context()
scope = agctx.ScopeFromContext(ctx)
)
// cleanup scope for wf
scp := filterScope(scope, "opts")
in, err := expr.NewVars(scp.Dict())
if err != nil {
return pe.Internal("could not validate request data: %v", err)
}
wp := atypes.WorkflowExecParams{
Trace: true,
// todo depending on settings per-route
Async: false,
// todo depending on settings per-route
Wait: true,
Input: in,
}
out, _, _, err := h.d.Exec(ctx, h.params.Workflow, wp)
if err != nil {
return pe.Internal("could not exec workflow: %v", err)
}
// merge out with scope
merged, err := in.Merge(out)
if err != nil {
return pe.Internal("could not receive workflow results: %v", err)
}
mm, err := expr.CastToVars(merged)
if err != nil {
return pe.Internal("could not receive workflow results: %v", err)
}
for k, v := range mm {
scope.Set(k, v)
}
scope = filterScope(scope, "eventType", "resourceType", "invoker")
// update scope for next items in pipeline
_ = r.WithContext(agctx.ScopeToContext(ctx, scope))
return nil
}
}
func NewPayload(cfg types.Config, l *zap.Logger) (p *processerPayload) {
p = &processerPayload{}
p.vm = jsenv.New(jsenv.NewTransformer(jsenv.LoaderJS, jsenv.TargetES2016))
p.log = l
p.cfg = cfg
p.Name = "payload"
p.Label = "Payload processer"
p.Kind = types.Processer
p.Args = []*types.FilterMetaArg{
{
Type: "jsfunc",
Label: "jsfunc",
Options: map[string]interface{}{},
},
}
// register a request body reader
p.vm.Register("readRequestBody", automation.ReadRequestBody)
return
}
func (h processerPayload) New(cfg types.Config) types.Handler {
return NewPayload(cfg, h.log)
}
func (h processerPayload) Enabled() bool {
return true
}
func (h processerPayload) String() string {
return fmt.Sprintf("apigw filter %s (%s)", h.Name, h.Label)
}
func (h processerPayload) Meta() types.FilterMeta {
return h.FilterMeta
}
func (h *processerPayload) Merge(params []byte, cfg types.Config) (types.Handler, error) {
h.cfg = cfg
err := json.NewDecoder(bytes.NewBuffer(params)).Decode(&h.params)
if err != nil {
return nil, err
}
if h.params.Func == "" {
return nil, fmt.Errorf("could not register function, body empty")
}
if h.fn, err = h.vm.RegisterFunction(h.params.Func); err != nil {
return nil, fmt.Errorf("could not register function, invalid body: %s", err)
}
return h, err
}
func (h processerPayload) Handler() types.HandlerFunc {
return func(rw http.ResponseWriter, r *http.Request) (err error) {
var (
ctx = r.Context()
scope = agctx.ScopeFromContext(ctx)
)
// cleanup scope for js
scp := filterScope(scope, "opts")
// check fn type
if err != nil {
return pe.InvalidData("could not register function: %v", err)
}
// need to find a consistent approach to the workflow jsenv function
// wf: input expr Var and the resulting variable in the jsenv is `input`
// apigw: input types.Scope and the resulting variable in the jsenv is `input['some_var']`
out, err := h.fn.Exec(h.vm.New(scp))
if err != nil {
return pe.Internal("could not exec payload function: %v", err)
}
// add to scope, so next steps can get the structure
scope.Set("payload", out)
// check if string
switch out.(type) {
case string, []byte:
// handling the newline, to keep the consistency with the json encoder
// which automatically appends the newline
_, err = rw.Write([]byte(fmt.Sprintf("%s\n", out)))
default:
err = json.NewEncoder(rw).Encode(out)
}
if err != nil {
return pe.Internal("could not write to response body: %v", err)
}
return
}
}
func (h processerPayload) VM() jsenv.Vm {
return h.vm
}
func filterScope(scope *types.Scp, kk ...string) (s *types.Scp) {
s = scope.Filter(func(k string, v interface{}) bool {
for _, v := range kk {
if k == v {
return false
}
}
return true
})
return
}

View File

@@ -0,0 +1,244 @@
package filter
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"strings"
"testing"
atypes "github.com/cortezaproject/corteza/server/automation/types"
agctx "github.com/cortezaproject/corteza/server/pkg/apigw/ctx"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
"github.com/cortezaproject/corteza/server/pkg/expr"
h "github.com/cortezaproject/corteza/server/pkg/http"
"github.com/cortezaproject/corteza/server/pkg/options"
"github.com/cortezaproject/corteza/server/pkg/wfexec"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
type (
wfServicer struct {
load func(ctx context.Context) error
exec func(ctx context.Context, workflowID uint64, p atypes.WorkflowExecParams) (*expr.Vars, uint64, atypes.Stacktrace, error)
search func(ctx context.Context, filter atypes.WorkflowFilter) (atypes.WorkflowSet, atypes.WorkflowFilter, error)
}
)
func Test_processerWorkflow(t *testing.T) {
type (
tf struct {
name string
err string
params string
wfs wfServicer
exp []string
}
)
var (
tcc = []tf{
{
name: "workflow processer",
exp: []string{"opts", "request"},
params: `{"workflow":"1"}`,
err: `could not exec workflow: mocked error`,
wfs: wfServicer{
load: func(ctx context.Context) error {
return nil
},
exec: func(ctx context.Context, workflowID uint64, p atypes.WorkflowExecParams) (*expr.Vars, uint64, atypes.Stacktrace, error) {
return must(expr.NewVars(map[string]interface{}{"foo": "bar"})), 0, make([]*wfexec.Frame, 0), fmt.Errorf("mocked error")
},
},
},
{
name: "workflow processer",
exp: []string{"foo", "request", "opts"},
params: `{"workflow":"1"}`,
wfs: wfServicer{
load: func(ctx context.Context) error {
return nil
},
exec: func(ctx context.Context, workflowID uint64, p atypes.WorkflowExecParams) (*expr.Vars, uint64, atypes.Stacktrace, error) {
return must(expr.NewVars(map[string]interface{}{"foo": "bar"})), 0, make([]*wfexec.Frame, 0), nil
},
},
},
}
)
for _, tc := range tcc {
t.Run(tc.name, func(t *testing.T) {
var (
req = require.New(t)
rc = httptest.NewRecorder()
rq, _ = http.NewRequest("POST", "/foo", http.NoBody)
ar, err = h.NewRequest(rq)
pp = NewWorkflow(types.Config{}, tc.wfs)
)
_, err = pp.Merge([]byte(tc.params), types.Config{})
req.NoError(err)
scope := &types.Scp{
"opts": options.Apigw(),
"request": ar,
}
rq = rq.WithContext(agctx.ScopeToContext(context.Background(), scope))
hn := pp.Handler()
err = hn(rc, rq)
if tc.err != "" {
req.EqualError(err, tc.err)
} else {
req.NoError(err)
}
req.ElementsMatch(tc.exp, agctx.ScopeFromContext(rq.Context()).Keys())
})
}
}
func Test_processerPayload(t *testing.T) {
type (
tf struct {
name string
err string
errv string
params string
exp string
rq *http.Request
}
)
var (
tcc = []tf{
{
name: "payload processer parse request body",
rq: &http.Request{
Method: "POST",
Body: ioutil.NopCloser(strings.NewReader(`[1,2,3]`)),
},
exp: "2\n",
params: prepareFuncPayload(t, `
const b = JSON.parse(readRequestBody(input.Get('request')));
return b[1];
`),
},
{
name: "payload processer js map request body",
rq: &http.Request{
Method: "POST",
Body: ioutil.NopCloser(strings.NewReader(`[{"name":"johnny", "surname":"mnemonic"},{"name":"johnny", "surname":"knoxville"}]`)),
},
exp: "{\"count\":2,\"results\":[{\"fullname\":\"Johnny Mnemonic\"},{\"fullname\":\"Johnny Knoxville\"}]}\n",
params: prepareFuncPayload(t, `
const readOnce = JSON.parse(readRequestBody(input.Get('request')));
const readTwice = JSON.parse(readRequestBody(input.Get('request')));
return {
"results":
readTwice.map(function({ name, surname }) {
return {
"fullname": name[0].toUpperCase() + name.substring(1) + " " + surname[0].toUpperCase() + surname.substring(1)
}
}),
"count": readTwice.length
};
`),
},
{
name: "payload processer empty function",
rq: &http.Request{
Method: "POST",
Body: ioutil.NopCloser(strings.NewReader(`[{"name":"johnny", "surname":"mnemonic"},{"name":"johnny", "surname":"knoxville"}]`)),
},
params: prepareFuncPayload(t, ``),
errv: `could not register function, body empty`,
},
{
name: "payload processer invalid body",
rq: &http.Request{
Method: "POST",
Body: ioutil.NopCloser(strings.NewReader(`[{"name":"johnny", "surname":"mnemonic"},{"name":"johnny", "surname":"knoxville"}]`)),
},
params: prepareFuncPayload(t, `.foo`),
errv: `could not register function, invalid body: could not transform payload: Unexpected "."`,
},
}
)
for _, tc := range tcc {
t.Run(tc.name, func(t *testing.T) {
var (
req = require.New(t)
rc = httptest.NewRecorder()
ar, err = h.NewRequest(tc.rq)
cfg = types.Config{}
)
pp := NewPayload(cfg, zap.NewNop())
_, err = pp.Merge([]byte(tc.params), cfg)
if tc.errv != "" {
req.EqualError(err, tc.errv)
return
} else {
req.NoError(err)
}
scope := &types.Scp{
"opts": options.Apigw(),
"request": ar,
}
tc.rq = tc.rq.WithContext(agctx.ScopeToContext(context.Background(), scope))
hn := pp.Handler()
err = hn(rc, tc.rq)
if tc.err != "" {
req.EqualError(err, tc.err)
} else {
req.NoError(err)
req.Equal(tc.exp, rc.Body.String())
}
})
}
}
func prepareFuncPayload(t *testing.T, s string) string {
aux, err := json.Marshal(map[string]string{"jsfunc": s})
if err != nil {
t.Error(err)
}
return string(aux)
}
func (f wfServicer) Load(ctx context.Context) error {
return f.load(ctx)
}
func (f wfServicer) Exec(ctx context.Context, workflowID uint64, p atypes.WorkflowExecParams) (*expr.Vars, uint64, atypes.Stacktrace, error) {
return f.exec(ctx, workflowID, p)
}
func (f wfServicer) Search(ctx context.Context, filter atypes.WorkflowFilter) (atypes.WorkflowSet, atypes.WorkflowFilter, error) {
return f.search(ctx, filter)
}
func must(v *expr.Vars, err error) *expr.Vars {
if err != nil {
return nil
}
return v
}

View File

@@ -0,0 +1,211 @@
package proxy
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httputil"
"net/url"
"time"
actx "github.com/cortezaproject/corteza/server/pkg/apigw/ctx"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
pe "github.com/cortezaproject/corteza/server/pkg/errors"
"go.uber.org/zap"
)
var (
hopHeaders = []string{
"Connection",
"Keep-Alive",
"Proxy-Authenticate",
"Proxy-Authorization",
"Te",
"Trailers",
"Transfer-Encoding",
"Upgrade",
}
)
type (
proxy struct {
types.FilterMeta
a ProxyAuthServicer
s types.SecureStorager
c *http.Client
log *zap.Logger
cfg types.Config
params struct {
Location string `json:"location"`
Auth ProxyAuthParams `json:"auth"`
}
}
)
func New(cfg types.Config, l *zap.Logger, c *http.Client, s types.SecureStorager) (p *proxy) {
p = &proxy{}
p.c = c
p.s = s
p.log = l
p.cfg = cfg
p.Name = "proxy"
p.Label = "Proxy processer"
p.Kind = types.Processer
p.Args = []*types.FilterMetaArg{
{
Type: "text",
Label: "location",
Options: map[string]interface{}{},
},
}
return
}
func (h proxy) New(cfg types.Config) types.Handler {
return New(cfg, h.log, h.c, h.s)
}
func (h proxy) Enabled() bool {
return true
}
func (h proxy) String() string {
return fmt.Sprintf("apigw filter %s (%s)", h.Name, h.Label)
}
func (h proxy) Meta() types.FilterMeta {
return h.FilterMeta
}
func (h *proxy) Merge(params []byte, cfg types.Config) (types.Handler, error) {
h.cfg = cfg
err := json.NewDecoder(bytes.NewBuffer(params)).Decode(&h.params)
if err != nil {
return nil, err
}
// get the auth mechanism
h.a, err = NewProxyAuthServicer(h.c, h.params.Auth, h.s)
if err != nil {
return nil, fmt.Errorf("could not load auth servicer for proxying: %s", err)
}
return h, err
}
func (h proxy) Handler() types.HandlerFunc {
return func(rw http.ResponseWriter, r *http.Request) (err error) {
var (
ctx = r.Context()
scope = actx.ScopeFromContext(ctx)
)
ctx, cancel := context.WithTimeout(ctx, scope.Opts().ProxyOutboundTimeout)
defer cancel()
log := h.log.With(zap.String("ref", h.Name))
outreq := r.Clone(ctx)
l, err := url.ParseRequestURI(h.params.Location)
if err != nil {
return pe.InvalidData("could not parse destination location for proxying: (%v)", err)
}
outreq.URL = l
outreq.RequestURI = ""
outreq.Method = r.Method
outreq.Host = l.Hostname()
// use authservicer, set any additional headers
err = h.a.Do(outreq)
if err != nil {
return pe.External("could not authenticate to external auth: (%v)", err)
}
// merge the old query params to the new request
// do not overwrite old ones
// do it after the authServicer, since we also may add them there
mergeQueryParams(r, outreq)
if scope.Opts().ProxyEnableDebugLog {
o, _ := httputil.DumpRequestOut(outreq, false)
log.Debug("proxy outbound request", zap.Any("request", string(o)))
}
// temporary metrics before the proper functionality
startTime := time.Now()
// todo - disable / enable follow redirects, already
// added to options
resp, err := h.c.Do(outreq)
if err != nil {
return pe.Internal("could not proxy request: (%v)", err)
}
if scope.Opts().ProxyEnableDebugLog {
o, _ := httputil.DumpResponse(resp, false)
log.Debug("proxy outbound response", zap.Any("request", string(o)), zap.Duration("duration", time.Since(startTime)))
}
b, err := io.ReadAll(resp.Body)
if err != nil {
return pe.Internal("could not read body on proxy request: (%v)", err)
}
mergeHeaders(resp.Header, rw.Header())
// add to writer
rw.Write(b)
return nil
}
}
func mergeHeaders(orig, dest http.Header) {
OUTER:
for name, values := range orig {
// skip headers that need to be omitted
// when proxying
for _, v := range hopHeaders {
if v == name {
continue OUTER
}
}
dest[name] = values
}
}
func mergeQueryParams(orig, dest *http.Request) {
origValues := dest.URL.Query()
for k, qp := range orig.URL.Query() {
// skip existing
if dest.URL.Query().Get(k) != "" {
continue
}
for _, v := range qp {
origValues.Add(k, v)
}
}
dest.URL.RawQuery = origValues.Encode()
}

View File

@@ -0,0 +1,337 @@
package proxy
import (
"fmt"
"net/http"
"net/url"
"strings"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
"github.com/cortezaproject/corteza/server/pkg/http/auth"
)
const (
proxyAuthTypeOauth2 proxyAuthType = "oauth2"
proxyAuthTypeHeader proxyAuthType = "header"
proxyAuthTypeQuery proxyAuthType = "query"
proxyAuthTypeBasic proxyAuthType = "basic"
proxyAuthTypeJWT proxyAuthType = "jwt"
proxyAuthTypeNoop proxyAuthType = "noop"
)
type (
ProxyAuthServicer interface {
Do(*http.Request) error
}
proxyAuthServicerNoop struct{}
proxyAuthServicerHeader struct {
params map[string]interface{}
}
proxyAuthServicerQuery struct {
params map[string]interface{}
}
proxyAuthServicerBasic struct {
servicer auth.ServicerBasic
user string
pass string
}
proxyAuthServicerOauth2 struct {
c *http.Client
servicer auth.ServicerOauth2
client string
secret string
scope []string
tokenUrl string
Args []*proxyAuthMetaArg `json:"params"`
}
proxyAuthServicerJWT struct {
JWT string
}
proxyAuthType string
ProxyAuthParams struct {
Type proxyAuthType `json:"type"`
Params map[string]interface{} `json:"params"`
}
proxyAuthMetaArg struct {
Label string `json:"label"`
Type string `json:"type"`
Example string `json:"example,omitempty"`
Options map[string]interface{} `json:"options,omitempty"`
}
ProxyAuthDefinition struct {
Type proxyAuthType `json:"type"`
Params []*proxyAuthMetaArg `json:"params"`
}
)
func NewProxyAuthServicer(c *http.Client, p ProxyAuthParams, s types.SecureStorager) (ProxyAuthServicer, error) {
switch p.Type {
case proxyAuthTypeHeader:
return newProxyAuthHeader(p)
case proxyAuthTypeQuery:
return newProxyAuthQuery(p)
case proxyAuthTypeBasic:
return newProxyAuthBasic(p)
case proxyAuthTypeOauth2:
return newProxyAuthOauth2(p, c, s)
case proxyAuthTypeJWT:
return newProxyAuthJWT(p)
default:
return proxyAuthServicerNoop{}, nil
}
}
func newProxyAuthHeader(p ProxyAuthParams) (s proxyAuthServicerHeader, err error) {
s = proxyAuthServicerHeader{
params: p.Params,
}
return
}
func newProxyAuthQuery(p ProxyAuthParams) (s proxyAuthServicerQuery, err error) {
s = proxyAuthServicerQuery{
params: p.Params,
}
return
}
func newProxyAuthBasic(p ProxyAuthParams) (s proxyAuthServicerBasic, err error) {
var (
ok bool
user, pass string
)
if user, ok = p.Params["username"].(string); !ok {
err = fmt.Errorf("invalid param username")
return
}
if pass, ok = p.Params["password"].(string); !ok {
err = fmt.Errorf("invalid param password")
return
}
servicer, _ := auth.NewBasic(auth.BasicParams{
User: user,
Pass: pass,
})
s = proxyAuthServicerBasic{user: user, pass: pass, servicer: servicer}
return
}
func newProxyAuthOauth2(p ProxyAuthParams, c *http.Client, s types.SecureStorager) (ss proxyAuthServicerOauth2, err error) {
var (
u *url.URL
ok bool
client, secret, tokenUrl string
scope []string = []string{}
)
if client, ok = p.Params["client"].(string); !ok {
err = fmt.Errorf("invalid param client")
return
}
if secret, ok = p.Params["secret"].(string); !ok {
err = fmt.Errorf("invalid param secret")
return
}
if scopes, ok := p.Params["scope"].(string); ok {
scope = strings.Fields(scopes)
}
if u, err = url.Parse(p.Params["token_url"].(string)); err != nil {
err = fmt.Errorf("invalid param token url: %s", err)
return
}
servicer, err := auth.NewOauth2(auth.Oauth2Params{
Client: client,
Secret: secret,
Scope: scope,
TokenUrl: u,
}, c, s)
ss = proxyAuthServicerOauth2{
c: c,
client: client,
servicer: servicer,
secret: secret,
scope: scope,
tokenUrl: tokenUrl,
}
return
}
func newProxyAuthJWT(p ProxyAuthParams) (ss proxyAuthServicerJWT, err error) {
var (
ok bool
jwt string
)
if jwt, ok = p.Params["jwt"].(string); !ok {
err = fmt.Errorf("invalid param jwt")
return
}
ss = proxyAuthServicerJWT{
JWT: jwt,
}
return
}
func (s proxyAuthServicerHeader) Do(r *http.Request) error {
for k, v := range s.params {
r.Header.Add(k, v.(string))
}
return nil
}
func (s proxyAuthServicerQuery) Do(r *http.Request) error {
if len(s.params) == 0 {
return nil
}
q := r.URL.Query()
for k, v := range s.params {
q.Set(k, v.(string))
}
r.URL.RawQuery = q.Encode()
return nil
}
func (s proxyAuthServicerBasic) Do(r *http.Request) error {
r.Header.Set("Authorization", "Basic "+s.servicer.Do(r.Context()))
return nil
}
func (s proxyAuthServicerOauth2) Do(r *http.Request) error {
t, err := s.servicer.Do(r.Context())
if err != nil {
return err
}
r.Header.Set("Authorization", "Bearer "+t.AccessToken)
return nil
}
func (s proxyAuthServicerJWT) Do(r *http.Request) (err error) {
// set up the request and get the jwt
r.Header.Set("Authorization", "Bearer "+s.JWT)
return
}
func (s proxyAuthServicerNoop) Do(r *http.Request) error {
return nil
}
func (s proxyAuthServicerOauth2) Def() *ProxyAuthDefinition {
return &ProxyAuthDefinition{
Type: proxyAuthTypeOauth2,
Params: []*proxyAuthMetaArg{
{
Label: "client",
Type: "text",
Example: "",
},
{
Label: "secret",
Type: "text",
Example: "",
},
{
Label: "scope",
Type: "text",
Example: "Keep the scopes separated with spaces ie.: 'scope1 scope2'",
},
{
Label: "token_url",
Type: "text",
Example: "",
},
},
}
}
func (s proxyAuthServicerHeader) Def() *ProxyAuthDefinition {
return &ProxyAuthDefinition{
Type: proxyAuthTypeHeader,
Params: []*proxyAuthMetaArg{
{
Label: "header",
Type: "text",
Example: "",
},
{
Label: "key",
Type: "text",
Example: "",
},
},
}
}
func (s proxyAuthServicerBasic) Def() *ProxyAuthDefinition {
return &ProxyAuthDefinition{
Type: proxyAuthTypeBasic,
Params: []*proxyAuthMetaArg{
{
Label: "username",
Type: "text",
Example: "",
},
{
Label: "password",
Type: "text",
Example: "",
},
},
}
}
func (s proxyAuthServicerJWT) Def() *ProxyAuthDefinition {
return &ProxyAuthDefinition{
Type: proxyAuthTypeJWT,
Params: []*proxyAuthMetaArg{
{
Label: "jwt",
Type: "text",
Example: "",
},
},
}
}
func ProxyAuthDef() []*ProxyAuthDefinition {
return []*ProxyAuthDefinition{
proxyAuthServicerOauth2{}.Def(),
proxyAuthServicerHeader{}.Def(),
proxyAuthServicerBasic{}.Def(),
proxyAuthServicerJWT{}.Def(),
}
}

View File

@@ -0,0 +1,124 @@
package proxy
import (
"net/http"
"testing"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
"github.com/stretchr/testify/require"
)
func Test_authDo(t *testing.T) {
type (
tf struct {
name string
err string
errv string
params ProxyAuthParams
exp http.Header
}
)
var (
tcc = []tf{
{
name: "auth header match headers",
params: ProxyAuthParams{
Type: proxyAuthTypeHeader,
Params: map[string]interface{}{
"Client-Id": "123455",
"Client_credentials": "pass1234",
},
},
exp: http.Header{
"Client-Id": []string{"123455"},
"Client_credentials": []string{"pass1234"},
},
},
{
name: "auth header match canonicalized headers",
params: ProxyAuthParams{
Type: proxyAuthTypeHeader,
Params: map[string]interface{}{
"camelCaseHeader": "123455",
},
},
exp: http.Header{
"Camelcaseheader": []string{"123455"},
},
},
{
name: "auth basic match headers",
params: ProxyAuthParams{
Type: proxyAuthTypeBasic,
Params: map[string]interface{}{
"username": "user",
"password": "pass1234",
},
},
exp: http.Header{"Authorization": []string{"Basic dXNlcjpwYXNzMTIzNA=="}},
},
{
name: "auth basic match headers fail user validation",
params: ProxyAuthParams{
Type: proxyAuthTypeBasic,
Params: map[string]interface{}{"password": "pass1234"},
},
exp: http.Header{},
errv: "invalid param username",
},
{
name: "auth basic match headers fail pass validation",
params: ProxyAuthParams{
Type: proxyAuthTypeBasic,
Params: map[string]interface{}{"username": "user"},
},
exp: http.Header{},
errv: "invalid param password",
},
{
name: "noop default fallback",
params: ProxyAuthParams{},
exp: http.Header{},
},
{
name: "auth JWT token",
params: ProxyAuthParams{
Type: proxyAuthTypeJWT,
Params: map[string]interface{}{"jwt": "1234"},
},
exp: http.Header{
"Authorization": []string{"Bearer 1234"},
},
},
}
)
for _, tc := range tcc {
t.Run(tc.name, func(t *testing.T) {
var (
req = require.New(t)
c = http.DefaultClient
)
c.Transport = types.MockRoundTripper(func(r *http.Request) (rs *http.Response, err error) { return })
rq, _ := http.NewRequest("POST", "/foo", http.NoBody)
authServicer, err := NewProxyAuthServicer(c, tc.params, struct{}{})
if tc.errv != "" {
req.EqualError(err, tc.errv)
return
}
err = authServicer.Do(rq)
if tc.err != "" {
req.EqualError(err, tc.err)
} else {
req.Equal(tc.exp, rq.Header)
}
})
}
}

View File

@@ -0,0 +1,201 @@
package proxy
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
agctx "github.com/cortezaproject/corteza/server/pkg/apigw/ctx"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
"github.com/cortezaproject/corteza/server/pkg/options"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
func Test_proxy(t *testing.T) {
type (
exp struct {
Status int
Header http.Header
Body *bytes.Buffer
}
tf struct {
name string
err string
params string
exp exp
rq *http.Request
fn func(*require.Assertions) types.MockRoundTripper
}
)
var (
tcc = []tf{
{
name: "proxy processer with auth headers",
fn: func(req *require.Assertions) types.MockRoundTripper {
return func(r *http.Request) (rs *http.Response, err error) {
rs = &http.Response{
StatusCode: http.StatusOK,
Body: io.NopCloser(strings.NewReader("default response")),
}
return
}
},
params: `{"location": "/foo", "auth": {"type": "header", "params": {"access-token": "123", "client": "456"}}}`,
exp: exp{
Status: http.StatusOK,
Header: http.Header{"Content-Type": []string{"text/plain; charset=utf-8"}},
Body: bytes.NewBufferString("default response"),
},
},
{
name: "proxy processer with auth query params",
fn: func(req *require.Assertions) types.MockRoundTripper {
return func(r *http.Request) (rs *http.Response, err error) {
rs = &http.Response{}
req.Equal("access-param=123%2B456", r.URL.RawQuery)
return
}
},
params: `{"location": "/foo", "auth": {"type": "query", "params": {"access-param": "123+456"}}}`,
exp: exp{
Status: http.StatusOK,
Header: http.Header{"Content-Type": []string{"text/plain; charset=utf-8"}},
Body: bytes.NewBuffer(nil),
},
},
{
name: "proxy processer with auth headers unauthorized",
fn: func(req *require.Assertions) types.MockRoundTripper {
return func(r *http.Request) (rs *http.Response, err error) {
rs = &http.Response{
StatusCode: http.StatusUnauthorized,
Body: io.NopCloser(strings.NewReader("unauthorized response")),
}
return
}
},
params: `{"location": "/foo", "auth": {"type": "header", "params": {"access-token": "123", "client": "456"}}}`,
exp: exp{
Status: http.StatusUnauthorized,
Header: http.Header{"Content-Type": []string{"text/plain; charset=utf-8"}},
Body: bytes.NewBufferString("unauthorized response"),
},
},
{
name: "proxy processer params parse error",
params: `{"location": "invalid url", "auth": {"type": "header", "params": {}}}`,
err: `could not parse destination location for proxying: (parse "invalid url": invalid URI for request)`,
},
{
name: "proxy processer params request error",
fn: func(req *require.Assertions) types.MockRoundTripper {
return func(r *http.Request) (rs *http.Response, err error) {
err = fmt.Errorf("error on client.Do")
return
}
},
params: `{"location": "https://example.com", "auth": {"type": "header", "params": {}}}`,
err: `could not proxy request: (Post "https://example.com": error on client.Do)`,
},
{
name: "proxy processer hop headers removed",
fn: func(req *require.Assertions) types.MockRoundTripper {
return func(r *http.Request) (rs *http.Response, err error) {
rs = &http.Response{
Header: http.Header{
"Proxy-Authenticate": []string{`Basic realm="Access to the internal site"`},
"Content-Type": []string{"application/json; charset=utf-8"},
},
StatusCode: http.StatusOK,
Body: io.NopCloser(strings.NewReader("default response")),
}
return
}
},
params: `{"location": "https://example.com", "auth": {"type": "header", "params": {}}}`,
exp: exp{
Status: http.StatusUnauthorized,
Header: http.Header{"Content-Type": []string{"application/json; charset=utf-8"}},
Body: bytes.NewBufferString("default response"),
},
},
{
name: "proxy processer query parameters merged",
fn: func(req *require.Assertions) types.MockRoundTripper {
return func(r *http.Request) (rs *http.Response, err error) {
rs = &http.Response{}
req.Equal("access-param=123%2B456&addedCustomQueryParam=true", r.URL.RawQuery)
return
}
},
params: `{"location": "https://example.com", "auth": {"type": "query", "params": {"access-param": "123+456"}}}`,
exp: exp{
Status: http.StatusUnauthorized,
Header: http.Header{"Content-Type": []string{"text/plain; charset=utf-8"}},
Body: bytes.NewBuffer(nil),
},
rq: &http.Request{
Header: http.Header{},
URL: &url.URL{Path: "/foo", RawQuery: "addedCustomQueryParam=true"},
Body: http.NoBody,
Method: "POST",
},
},
}
)
for _, tc := range tcc {
t.Run(tc.name, func(t *testing.T) {
var (
rc = httptest.NewRecorder()
req = require.New(t)
c = http.DefaultClient
rq = tc.rq
cfg = types.Config{}
)
if tc.fn != nil {
c.Transport = types.MockRoundTripper(tc.fn(req))
}
if rq == nil {
rq = httptest.NewRequest("POST", "/foo", strings.NewReader(`custom request body`))
}
proxy := New(cfg, zap.NewNop(), c, struct{}{})
_, err := proxy.Merge([]byte(tc.params), cfg)
req.NoError(err)
scope := &types.Scp{
"opts": options.Apigw(),
}
ctx := agctx.ScopeToContext(context.Background(), scope)
rq = rq.WithContext(ctx)
hn := proxy.Handler()
err = hn(rc, rq)
if tc.err != "" {
req.EqualError(err, tc.err)
} else {
req.NoError(err)
req.Equal(tc.exp.Header, rc.Header())
req.Equal(tc.exp.Body, rc.Body)
}
})
}
}

View File

@@ -0,0 +1,43 @@
package apigw
import (
"net/http"
"github.com/cortezaproject/corteza/server/pkg/apigw/profiler"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
h "github.com/cortezaproject/corteza/server/pkg/http"
"go.uber.org/zap"
)
func helperDefaultResponse(cfg types.Config, pr *profiler.Profiler, log *zap.Logger) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
addToProfiler(cfg, pr, log, r, http.StatusNotFound)
http.Error(w, "", http.StatusNotFound)
}
}
func helperMethodNotAllowed(cfg types.Config, pr *profiler.Profiler, log *zap.Logger) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
addToProfiler(cfg, pr, log, r, http.StatusMethodNotAllowed)
http.Error(w, "", http.StatusMethodNotAllowed)
}
}
func addToProfiler(cfg types.Config, pr *profiler.Profiler, log *zap.Logger, r *http.Request, status int) {
if !(cfg.Profiler.Enabled && cfg.Profiler.Global) {
return
}
// add to profiler
ar, err := h.NewRequest(r)
if err != nil {
log.Warn("could not create request wrapper, not adding to profiler")
return
}
h := pr.Hit(ar)
h.Status = status
pr.Push(h)
}

View File

@@ -0,0 +1,38 @@
package chain
import (
"net/http"
"github.com/go-chi/chi/v5"
)
type (
middleware []func(http.Handler) http.Handler
ChainHandler interface {
Handler() http.Handler
Chain(middleware)
}
chiHandlerChainDefault struct {
chain middleware
}
)
func NewDefault() *chiHandlerChainDefault {
return &chiHandlerChainDefault{}
}
func (cc *chiHandlerChainDefault) Chain(m middleware) {
cc.chain = m
}
func (cc *chiHandlerChainDefault) Handler() http.Handler {
return chiHandler(cc.chain)
}
func chiHandler(cc middleware) http.Handler {
return chi.
Chain(cc...).
HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {})
}

View File

@@ -0,0 +1,29 @@
package chain
import (
"net/http"
"github.com/cortezaproject/corteza/server/pkg/apigw/profiler"
)
type (
chiHandlerChainProfiler struct {
chain middleware
}
)
func NewProfiler() *chiHandlerChainProfiler {
return &chiHandlerChainProfiler{}
}
func (cc *chiHandlerChainProfiler) Chain(m middleware) {
cc.chain = m
}
func (cc *chiHandlerChainProfiler) Handler() http.Handler {
// wrap handlers around the chain so we get some profiling info
cc.chain = append([]func(http.Handler) http.Handler{profiler.StartHandler}, cc.chain...)
cc.chain = append(cc.chain, profiler.FinishHandler)
return chiHandler(cc.chain)
}

View File

@@ -0,0 +1,157 @@
package pipeline
import (
"context"
"net/http"
"sort"
"time"
actx "github.com/cortezaproject/corteza/server/pkg/apigw/ctx"
"github.com/cortezaproject/corteza/server/pkg/apigw/pipeline/chain"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
"github.com/cortezaproject/corteza/server/pkg/auth"
"go.uber.org/zap"
)
type (
Worker struct {
Weight int
Async bool
Name string
Type types.FilterKind
Handler func(rw http.ResponseWriter, r *http.Request) error
}
workerSet []*Worker
Pl struct {
workers workerSet
ch chain.ChainHandler
err types.ErrorHandlerFunc
log *zap.Logger
async bool
}
)
func NewPipeline(log *zap.Logger, ch chain.ChainHandler) *Pl {
var (
defaultErrorHandler = types.NewDefaultErrorHandler(log)
)
return &Pl{
ch: ch,
log: log,
err: defaultErrorHandler.Handler(),
}
}
func (pp *Pl) Async(a bool) {
pp.async = a
}
func (pp *Pl) Error() types.ErrorHandlerFunc {
return pp.err
}
// ErrorHandler adds error handler
func (pp *Pl) ErrorHandler(ff types.ErrorHandlerFunc) {
pp.err = ff
}
// Add filter
func (pp *Pl) Add(w *Worker) {
pp.workers = append(pp.workers, w)
sort.Sort(pp.workers)
}
// Handler is the main operating entry point for requests
// that handles filter groups
func (pp *Pl) Handler() http.Handler {
// use the chi implementation of chains
pp.ch.Chain(pp.makeMiddleware(pp.workers...))
return pp.ch.Handler()
}
// makeMiddleware creates a list of handlers from workers
// it is used in chaining
func (pp *Pl) makeMiddleware(hh ...*Worker) (middleware []func(http.Handler) http.Handler) {
for _, wrker := range hh {
middleware = append(middleware, pp.makeHandler(*wrker))
}
return middleware
}
// makeHandler creates a handler from worker, it also
// wraps the error handling
func (pp *Pl) makeHandler(hh Worker) func(next http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
var (
err error
start = time.Now()
log = pp.log.With(zap.String("filter", hh.Name))
)
log.Debug("started processing", zap.Bool("async", hh.Async))
fn := func() (err error) {
err = hh.Handler(rw, r)
log.Debug("finished processing", zap.Duration("duration", time.Since(start)))
return
}
if hh.Async {
go func() {
var (
newCtx = context.Background()
ident = auth.GetIdentityFromContext(r.Context())
scope = actx.ScopeFromContext(r.Context())
)
newCtx = actx.ScopeToContext(context.Background(), scope)
newCtx = auth.SetIdentityToContext(newCtx, ident)
r = r.WithContext(newCtx)
// only log error, do not call error handler,
// since we do not reply back the response (it was already sent)
if err = fn(); err != nil {
log.Error(err.Error())
}
}()
} else {
err = fn()
}
if err != nil {
pp.err(rw, r, err)
return
} else {
next.ServeHTTP(rw, r)
}
})
}
}
func (a workerSet) Len() int { return len(a) }
func (a workerSet) Less(i, j int) bool {
return a[i].Weight < a[j].Weight
}
func (a workerSet) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a workerSet) Filter(f func(*Worker) (bool, error)) (out workerSet, err error) {
var ok bool
out = workerSet{}
for i := range a {
if ok, err = f(a[i]); err != nil {
return
} else if ok {
out = append(out, a[i])
}
}
return
}

View File

@@ -0,0 +1,126 @@
package pipeline
import (
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/cortezaproject/corteza/server/pkg/apigw/pipeline/chain"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
var (
mockEmptyHandler = func(rw http.ResponseWriter, r *http.Request) (err error) { return }
)
func NewPl() *Pl {
return NewPipeline(zap.NewNop(), chain.NewDefault())
}
func Test_pipelineAdd(t *testing.T) {
var (
req = require.New(t)
)
p := NewPl()
p.Add(&Worker{
Handler: mockEmptyHandler,
Weight: 0,
Name: "mockWorker",
})
req.Len(p.workers, 1)
}
func Test_pipelineHandleMultiple(t *testing.T) {
var (
req = require.New(t)
rr = httptest.NewRecorder()
p = NewPl()
first = types.MockHandler{
Handler_: func(rw http.ResponseWriter, r *http.Request) error {
rw.Write([]byte(`first`))
return nil
},
}
second = types.MockHandler{
Handler_: func(rw http.ResponseWriter, r *http.Request) error {
rw.Write([]byte(`second`))
return nil
},
}
)
p.Add(&Worker{
Handler: first.Handler(),
Weight: 5,
Name: "mockHandler",
})
p.Add(&Worker{
Handler: second.Handler(),
Weight: 0,
Name: "mockHandler",
})
p.Handler().ServeHTTP(rr, &http.Request{})
req.Equal(`secondfirst`, rr.Body.String())
}
func Test_pipelineExecErr(t *testing.T) {
type (
tf struct {
name string
mh types.MockHandler
w *Worker
wgt int
exp string
}
)
var (
tcc = []tf{
{
name: "matching simple",
mh: types.MockHandler{
Handler_: func(rw http.ResponseWriter, r *http.Request) error {
return fmt.Errorf("triggered")
}},
w: &Worker{},
exp: `{"error":{"message":"triggered"}}` + "\n",
},
{
name: "matching simple",
mh: types.MockHandler{
Handler_: func(rw http.ResponseWriter, r *http.Request) error {
rw.Write([]byte(`foobar`))
return nil
}},
w: &Worker{},
exp: `foobar`,
},
}
)
for _, tc := range tcc {
var (
p = NewPl()
rr = httptest.NewRecorder()
req = require.New(t)
)
tc.w.Handler = tc.mh.Handler()
p.Add(tc.w)
p.Handler().ServeHTTP(rr, &http.Request{})
req.Equal(tc.exp, rr.Body.String())
}
}

View File

@@ -0,0 +1,16 @@
package profiler
import "net/http"
func StartHandler(next http.Handler) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
// add some info to context
next.ServeHTTP(rw, r)
})
}
func FinishHandler(next http.Handler) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
next.ServeHTTP(rw, r)
})
}

View File

@@ -0,0 +1,57 @@
package profiler
import (
"encoding/base64"
"fmt"
"time"
h "github.com/cortezaproject/corteza/server/pkg/http"
)
type (
Hit struct {
ID string
Status int
Route uint64
R *h.Request
Ts *time.Time
Tf *time.Time
D *time.Duration
}
Hits map[string][]*Hit
)
func (h *Hit) generateID() {
h.ID = base64.URLEncoding.EncodeToString([]byte(fmt.Sprintf("%s_%d", h.R.URL.Path, h.Ts.UnixNano())))
}
func (s Hits) Filter(fn func(k string, v *Hit) bool) Hits {
ss := Hits{}
for k, v := range s {
for _, vv := range v {
if !fn(k, vv) {
continue
}
ss[k] = append(ss[k], vv)
}
}
return ss
}
func (h Hits) Len() int {
return h.Len()
}
func (h Hits) Less(i, j int) bool {
return false
}
func (h Hits) Swap(i, j int) {
return
}

View File

@@ -0,0 +1,90 @@
package profiler
import (
"net/http"
"sync"
"time"
h "github.com/cortezaproject/corteza/server/pkg/http"
)
type (
Profiler struct {
mux sync.RWMutex
l Hits
}
)
func New() *Profiler {
return &Profiler{l: make(Hits)}
}
func (p *Profiler) Hit(r *h.Request) (h *Hit) {
var (
n = time.Now()
)
h = &Hit{"", http.StatusOK, 0, r, &n, nil, nil}
h.generateID()
return
}
func (p *Profiler) Push(h *Hit) (id string) {
p.mux.Lock()
defer p.mux.Unlock()
if h.Tf == nil {
n := time.Now()
d := n.Sub(*h.Ts)
h.Tf = &n
h.D = &d
}
h.generateID()
id = p.id(h.R)
p.l[id] = append(p.l[id], h)
return
}
func (p *Profiler) Hits(s Sort) Hits {
p.mux.RLock()
defer p.mux.RUnlock()
ll := p.l.Filter(func(k string, v *Hit) bool {
var b bool = true
if s.Path != "" && v.R.URL.Path != s.Path {
b = false
}
if s.Hit != "" && v.ID != s.Hit {
b = false
}
return b
})
return ll
}
func (p *Profiler) Purge(f *PurgeFilter) {
p.mux.Lock()
defer p.mux.Unlock()
if f.RouteID == 0 {
p.l = make(Hits, 0)
return
}
p.l = p.l.Filter(func(k string, v *Hit) bool {
return v.Route != f.RouteID
})
}
func (p *Profiler) id(r *h.Request) string {
return r.URL.Path
}

View File

@@ -0,0 +1,163 @@
package profiler
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
h "github.com/cortezaproject/corteza/server/pkg/http"
"github.com/stretchr/testify/require"
)
const (
day = time.Hour * 24
)
func Test_ApigwProfiler_newHit(t *testing.T) {
var (
p = New()
req = require.New(t)
rr, err = h.NewRequest(httptest.NewRequest("POST", "/foo", strings.NewReader(`foo`)))
hit = p.Hit(rr)
)
req.NoError(err)
req.Equal(hit.Status, http.StatusOK)
req.NotNil(hit.Ts)
req.NotNil(hit.ID)
}
func Test_ApigwProfiler_push(t *testing.T) {
var (
p = New()
req = require.New(t)
rr, err = h.NewRequest(httptest.NewRequest("POST", "/foo", strings.NewReader(`foo`)))
hit = p.Hit(rr)
id = p.Push(hit)
)
_, found := p.l[id]
req.NoError(err)
req.NotEmpty(id)
req.Len(p.l, 1)
req.True(found)
}
func Test_ApigwProfiler_filterPath(t *testing.T) {
var (
pp = New()
req = require.New(t)
now = time.Date(2022, time.March, 1, 1, 1, 1, 0, time.UTC)
then = now.Add(-1 * day)
later = now.Add(day)
hr, _ = h.NewRequest(httptest.NewRequest("POST", "/foo", strings.NewReader(`foo`)))
hr2, _ = h.NewRequest(httptest.NewRequest("GET", "/bar", strings.NewReader(`foo`)))
hr3, _ = h.NewRequest(httptest.NewRequest("GET", "/baz", strings.NewReader(`foo`)))
)
pp.Push(&Hit{R: hr, Ts: &then})
pp.Push(&Hit{R: hr, Ts: &then})
pp.Push(&Hit{R: hr2, Ts: &now})
pp.Push(&Hit{R: hr3, Ts: &later})
list := pp.Hits(Sort{
Path: hr3.RequestURI,
})
_, found := list[hr3.RequestURI]
req.True(found)
req.Len(list[hr3.RequestURI], 1)
}
func Test_ApigwProfiler_filterHit(t *testing.T) {
var (
pp = New()
req = require.New(t)
now = time.Date(2022, time.March, 1, 1, 1, 1, 0, time.UTC)
then = now.Add(-1 * day)
later = now.Add(day)
hr, _ = h.NewRequest(httptest.NewRequest("POST", "/foo", strings.NewReader(`foo`)))
hr2, _ = h.NewRequest(httptest.NewRequest("GET", "/bar", strings.NewReader(`foo`)))
hr3, _ = h.NewRequest(httptest.NewRequest("GET", "/baz", strings.NewReader(`foo`)))
)
pp.Push(&Hit{R: hr, Ts: &then})
pp.Push(&Hit{R: hr, Ts: &now})
pp.Push(&Hit{R: hr2, Ts: &now})
pp.Push(&Hit{R: hr2, Ts: &later})
pp.Push(&Hit{R: hr2, Ts: &later})
h := pp.Hit(hr3)
h.Ts = &later
id := pp.Push(h)
list := pp.Hits(Sort{
Hit: h.ID,
})
_, found := list[id]
req.True(found)
req.Len(list[id], 1)
}
func Test_ApigwProfiler_purgeAll(t *testing.T) {
var (
pp = New()
req = require.New(t)
now = time.Date(2022, time.March, 1, 1, 1, 1, 0, time.UTC)
then = now.Add(-1 * day)
later = now.Add(day)
hr, _ = h.NewRequest(httptest.NewRequest("POST", "/foo", strings.NewReader(`foo`)))
hr2, _ = h.NewRequest(httptest.NewRequest("GET", "/bar", strings.NewReader(`foo`)))
)
pp.Push(&Hit{R: hr, Ts: &then})
pp.Push(&Hit{R: hr, Ts: &now})
pp.Push(&Hit{R: hr2, Ts: &now})
pp.Push(&Hit{R: hr2, Ts: &later})
pp.Push(&Hit{R: hr2, Ts: &later})
req.Len(pp.l, 2)
pp.Purge(&PurgeFilter{})
req.Len(pp.l, 0)
}
func Test_ApigwProfiler_purgeRoute(t *testing.T) {
var (
pp = New()
req = require.New(t)
now = time.Date(2022, time.March, 1, 1, 1, 1, 0, time.UTC)
then = now.Add(-1 * day)
later = now.Add(day)
hr, _ = h.NewRequest(httptest.NewRequest("POST", "/foo", strings.NewReader(`foo`)))
hr2, _ = h.NewRequest(httptest.NewRequest("GET", "/bar", strings.NewReader(`foo`)))
)
pp.Push(&Hit{Route: 1, R: hr, Ts: &then})
pp.Push(&Hit{Route: 1, R: hr, Ts: &now})
pp.Push(&Hit{Route: 2, R: hr2, Ts: &now})
pp.Push(&Hit{Route: 2, R: hr2, Ts: &later})
pp.Push(&Hit{Route: 2, R: hr2, Ts: &later})
req.Len(pp.l, 2)
pp.Purge(&PurgeFilter{RouteID: 2})
req.Len(pp.l, 1)
}

View File

@@ -0,0 +1,7 @@
package profiler
type (
PurgeFilter struct {
RouteID uint64
}
)

View File

@@ -0,0 +1,8 @@
package profiler
type (
Sort struct {
Hit string
Path string
}
)

View File

@@ -0,0 +1,83 @@
package registry
import (
"fmt"
"net/http"
"github.com/cortezaproject/corteza/server/automation/service"
"github.com/cortezaproject/corteza/server/pkg/apigw/filter"
"github.com/cortezaproject/corteza/server/pkg/apigw/filter/proxy"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
)
type (
Registry struct {
cfg types.Config
h map[string]types.Handler
}
secureStorageTodo struct{}
)
func NewRegistry(cfg types.Config) *Registry {
return &Registry{
h: map[string]types.Handler{},
cfg: cfg,
}
}
func (r *Registry) Add(n string, h types.Handler) {
r.h[n] = h
}
func (r *Registry) Merge(h types.Handler, b []byte, cfg types.Config) (hh types.Handler, err error) {
hh, err = h.Merge(b, cfg)
return
}
func (r *Registry) Get(identifier string) (types.Handler, error) {
var (
ok bool
f types.Handler
)
if f, ok = r.h[identifier]; !ok {
return nil, fmt.Errorf("could not get element from registry: %s", identifier)
}
return f.New(r.cfg), nil
}
func (r *Registry) All() (list types.FilterMetaList) {
for _, handler := range r.h {
if !handler.Enabled() {
continue
}
meta := handler.Meta()
list = append(list, &meta)
}
return
}
func (r *Registry) Preload() {
// prefilters
r.Add("queryParam", filter.NewQueryParam(r.cfg))
r.Add("header", filter.NewHeader(r.cfg))
r.Add("profiler", filter.NewProfiler(r.cfg))
// processers
r.Add("workflow", filter.NewWorkflow(r.cfg, NewWorkflow()))
r.Add("proxy", proxy.New(r.cfg, service.DefaultLogger, http.DefaultClient, secureStorageTodo{}))
r.Add("payload", filter.NewPayload(r.cfg, service.DefaultLogger))
// postfilters
r.Add("redirection", filter.NewRedirection(r.cfg))
r.Add("response", filter.NewResponse(r.cfg, service.Registry()))
r.Add("defaultJsonResponse", filter.NewDefaultJsonResponse(r.cfg))
}
func NewWorkflow() (wf filter.WfExecer) {
return service.DefaultWorkflow
}

View File

@@ -0,0 +1,101 @@
package registry
import (
"testing"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
"github.com/stretchr/testify/require"
)
func Test_registryAddGet(t *testing.T) {
var (
req = require.New(t)
r = NewRegistry(types.Config{})
)
r.Add("mockHandler", types.MockHandler{})
h, err := r.Get("mockHandler")
req.NoError(err)
req.Len(r.h, 1)
req.IsType(types.MockHandler{}, h)
}
func Test_registryAddGetErr(t *testing.T) {
var (
req = require.New(t)
r = NewRegistry(types.Config{})
)
r.Add("mockHandler", types.MockHandler{})
h, err := r.Get("foo")
req.EqualError(err, "could not get element from registry: foo")
req.Len(r.h, 1)
req.Nil(h)
}
func Test_registryMerge(t *testing.T) {
type (
tf struct {
name string
err string
params string
exp string
}
)
var (
tcc = []tf{
{
name: "set params",
params: `{"foo":"bar"}`,
exp: "bar",
},
{
name: "set invalid params",
params: `{"foo1":"bar"}`,
exp: "",
},
{
name: "set invalid params err",
params: `{"foo1":"bar"`,
exp: "",
err: "unexpected EOF",
},
}
)
for _, tc := range tcc {
var (
req = require.New(t)
r = NewRegistry(types.Config{})
)
m, err := r.Merge(types.MockHandler{}, []byte(tc.params), types.Config{})
if tc.err != "" {
req.EqualError(err, tc.err)
} else {
req.Equal(m.(types.MockHandler).Foo, tc.exp)
req.NoError(err)
}
}
}
func Test_registryAll(t *testing.T) {
var (
req = require.New(t)
r = NewRegistry(types.Config{})
)
r.Add("mockHandler", types.MockHandler{})
list := r.All()
req.Len(list, 1)
req.NotEmpty(list[0].Name)
}

94
server/pkg/apigw/route.go Normal file
View File

@@ -0,0 +1,94 @@
package apigw
import (
"fmt"
"net/http"
"time"
actx "github.com/cortezaproject/corteza/server/pkg/apigw/ctx"
"github.com/cortezaproject/corteza/server/pkg/apigw/profiler"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
"github.com/cortezaproject/corteza/server/pkg/auth"
h "github.com/cortezaproject/corteza/server/pkg/http"
"go.uber.org/zap"
)
type (
route struct {
ID uint64
endpoint string
method string
meta routeMeta
cfg types.Config
log *zap.Logger
pr *profiler.Profiler
handler http.Handler
errHandler types.ErrorHandlerFunc
}
routeMeta struct {
debug bool
async bool
}
)
func (r route) ServeHTTP(w http.ResponseWriter, req *http.Request) {
var (
ctx = auth.SetIdentityToContext(req.Context(), auth.ServiceUser())
start = time.Now()
scope = types.Scp{}
hit = &profiler.Hit{}
)
r.log.Debug("started serving route")
ar, err := h.NewRequest(req)
if err != nil {
r.log.Error("could not get initial request", zap.Error(err))
}
scope.Set("opts", r.cfg)
scope.Set("request", ar)
// use profiler, override any profiling prefilter
if r.cfg.Profiler.Enabled && r.cfg.Profiler.Global {
// add request to profiler
hit = r.pr.Hit(ar)
hit.Route = r.ID
}
req = req.WithContext(actx.ScopeToContext(ctx, &scope))
req = req.WithContext(actx.ProfilerToContext(req.Context(), hit))
r.handler.ServeHTTP(w, req)
r.log.Debug("finished serving route",
zap.Duration("duration", time.Since(start)),
)
if !r.cfg.Profiler.Enabled {
return
}
if r.cfg.Profiler.Global {
r.pr.Push(hit)
return
}
if hit = actx.ProfilerFromContext(req.Context()); hit != nil && hit.R != nil {
// updated hit from a possible prefilter
// we need to push route ID even if the profiler is disabled
hit.Route = r.ID
hit.Status = http.StatusOK
r.pr.Push(hit)
r.log.Debug("pushing request to profiler")
}
}
func (r route) String() string {
return fmt.Sprintf("%s %s", r.method, r.endpoint)
}

View File

@@ -0,0 +1,107 @@
package apigw
import (
"errors"
"net/http"
"net/http/httptest"
"testing"
"github.com/cortezaproject/corteza/server/pkg/apigw/pipeline"
"github.com/cortezaproject/corteza/server/pkg/apigw/pipeline/chain"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
func Test_pl(t *testing.T) {
type (
tf struct {
name string
method string
endpoint string
expError string
expStatus int
handler *types.MockHandler
errHandler *types.MockErrorHandler
}
)
var (
tcc = []tf{
{
name: "successful handler",
handler: &types.MockHandler{
Handler_: func(rw http.ResponseWriter, r *http.Request) error {
rw.WriteHeader(http.StatusTemporaryRedirect)
return nil
},
},
method: "POST",
expStatus: http.StatusTemporaryRedirect,
expError: "",
},
{
name: "unsuccessful handle custom error response",
handler: &types.MockHandler{
Handler_: func(rw http.ResponseWriter, r *http.Request) error {
rw.WriteHeader(http.StatusTemporaryRedirect)
return errors.New("test error")
},
},
errHandler: &types.MockErrorHandler{
Handler_: func(rw http.ResponseWriter, r *http.Request, err error) {
rw.Write([]byte("custom error response: " + err.Error()))
},
},
method: "POST",
expStatus: http.StatusTemporaryRedirect,
expError: "custom error response: test error",
},
{
name: "unsuccessful handle default error response",
handler: &types.MockHandler{
Handler_: func(rw http.ResponseWriter, r *http.Request) error {
rw.WriteHeader(http.StatusTemporaryRedirect)
return errors.New("test error")
},
},
method: "POST",
expStatus: http.StatusTemporaryRedirect,
expError: "{\"error\":{\"message\":\"test error\"}}\n",
},
}
)
for _, tc := range tcc {
t.Run(tc.name, func(t *testing.T) {
var (
req = require.New(t)
rr = httptest.NewRecorder()
pipe = pipeline.NewPipeline(zap.NewNop(), chain.NewDefault())
)
r := httptest.NewRequest("POST", "/foo", http.NoBody)
pipe.Add(&pipeline.Worker{
Handler: tc.handler.Handler(),
})
if tc.errHandler != nil {
pipe.ErrorHandler(tc.errHandler.Handler())
}
route := &route{
method: tc.method,
endpoint: tc.endpoint,
log: zap.NewNop(),
handler: pipe.Handler(),
errHandler: pipe.Error(),
}
route.ServeHTTP(rr, r)
req.Equal(tc.expStatus, rr.Result().StatusCode)
req.Equal(tc.expError, rr.Body.String())
})
}
}

466
server/pkg/apigw/service.go Normal file
View File

@@ -0,0 +1,466 @@
package apigw
import (
"context"
"encoding/json"
"fmt"
"math"
"net/http"
"github.com/cortezaproject/corteza/server/pkg/apigw/filter"
"github.com/cortezaproject/corteza/server/pkg/apigw/filter/proxy"
"github.com/cortezaproject/corteza/server/pkg/apigw/pipeline"
"github.com/cortezaproject/corteza/server/pkg/apigw/pipeline/chain"
"github.com/cortezaproject/corteza/server/pkg/apigw/profiler"
"github.com/cortezaproject/corteza/server/pkg/apigw/registry"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
f "github.com/cortezaproject/corteza/server/pkg/filter"
st "github.com/cortezaproject/corteza/server/system/types"
"github.com/go-chi/chi/v5"
"go.uber.org/zap"
)
type (
storer interface {
SearchApigwRoutes(ctx context.Context, f st.ApigwRouteFilter) (st.ApigwRouteSet, st.ApigwRouteFilter, error)
SearchApigwFilters(ctx context.Context, f st.ApigwFilterFilter) (st.ApigwFilterSet, st.ApigwFilterFilter, error)
}
apigw struct {
log *zap.Logger
reg *registry.Registry
routes []*route
mx *chi.Mux
pr *profiler.Profiler
storer storer
cfg types.Config
}
)
var (
// global service
apiGw *apigw
)
func Service() *apigw {
return apiGw
}
// Setup handles the singleton service
func Setup(cfg types.Config, log *zap.Logger, storer storer) {
if apiGw != nil {
return
}
apiGw = New(cfg, log, storer)
}
func New(cfg types.Config, logger *zap.Logger, storer storer) *apigw {
var (
pr = profiler.New()
)
reg := registry.NewRegistry(cfg)
reg.Preload()
return &apigw{
log: logger.Named("http.apigw"),
storer: storer,
reg: reg,
pr: pr,
cfg: cfg,
}
}
// ServeHTTP forwards the given HTTP request to the underlying chi mux which
// then handles the heavy lifting
//
// When reloading routes, make sure to replace the original mux
func (s *apigw) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if s.mx == nil {
http.Error(w, "Integration Gateway not initialized", http.StatusInternalServerError)
return
}
if len(s.routes) == 0 {
helperDefaultResponse(s.cfg, s.pr, s.log)(w, r)
return
}
// Remove route context for chi
//
// Without this, chi can not properly handle requests
// in API gateway's sub-router
r = r.WithContext(context.WithValue(r.Context(), chi.RouteCtxKey, nil))
// Handle api-gw request
s.mx.ServeHTTP(w, r)
}
// Reload reloads all routes and their filters
//
// The procedure constructs a new chi mux
func (s *apigw) Reload(ctx context.Context) (err error) {
routes, err := s.loadRoutes(ctx)
if err != nil {
s.log.Error("could not reload Integration Gateway routes", zap.Error(err))
return
}
s.Init(ctx, routes...)
// Rebuild the mux
s.mx = chi.NewMux()
for _, r := range s.routes {
// Register route handler on endpoint & method
s.mx.Method(r.method, r.endpoint, r)
}
// handling missed hits
// profiler gets the missed hit info also
{
var (
defaultMethodResponse = helperMethodNotAllowed(s.cfg, s.pr, s.log)
defaultResponse = helperDefaultResponse(s.cfg, s.pr, s.log)
)
s.mx.NotFound(defaultResponse)
s.mx.MethodNotAllowed(defaultMethodResponse)
}
return nil
}
// ReloadEndpoint reload a route and its filters
//
// The procedure use existing chi mux
func (s *apigw) ReloadEndpoint(ctx context.Context, method, endpoint string) (err error) {
var (
routes []*route
)
routes, err = s.loadRoute(ctx, method, endpoint)
if err != nil {
s.log.Error("could not reload Integration Gateway routes", zap.Error(err))
return
}
// rr := append(s.routes, routes...)
s.PrepRoutes(ctx, routes...)
if s.mx == nil {
// Rebuild the mux
s.mx = chi.NewMux()
}
for _, r := range routes {
// Register route handler on endpoint & method
s.mx.Method(r.method, r.endpoint, r)
}
// Make sure to append newly registered routes
s.AppendRoutes(routes...)
// handling missed hits
// profiler gets the missed hit info also
{
var (
defaultMethodResponse = helperMethodNotAllowed(s.cfg, s.pr, s.log)
defaultResponse = helperDefaultResponse(s.cfg, s.pr, s.log)
)
s.mx.NotFound(defaultResponse)
s.mx.MethodNotAllowed(defaultMethodResponse)
}
return nil
}
// Init all routes
func (s *apigw) Init(ctx context.Context, routes ...*route) {
s.PrepRoutes(ctx, routes...)
s.routes = routes
}
func (s *apigw) PrepRoutes(ctx context.Context, routes ...*route) {
var (
err error
defaultPostFilter types.Handler
)
s.loadInfo()
s.log.Debug("preparing routes", zap.Int("count", len(routes)))
defaultPostFilter, err = s.reg.Get("defaultJsonResponse")
if err != nil {
s.log.Error("could not register default filter", zap.Error(err))
}
for _, r := range routes {
var (
log = s.log.With(zap.String("route", r.String()))
pipe = pipeline.NewPipeline(log, chain.NewDefault())
regFilters []*st.ApigwFilter
)
// pipeline needs to know how to handle
// async processers
pipe.Async(r.meta.async)
r.cfg = s.cfg
r.log = log
r.pr = s.pr
regFilters, err = s.loadFilters(ctx, r.ID)
if err != nil {
log.Error("could not load filters for route", zap.Error(err))
continue
}
for _, rf := range regFilters {
flog := log.With(zap.String("ref", rf.Ref))
// make sure there is only one postfilter
// on async routes
if r.meta.async && rf.Kind == string(types.PostFilter) {
flog.Debug("not registering filter for async route")
continue
}
var ff *pipeline.Worker
ff, err = s.registerFilter(rf, r)
if err != nil {
flog.Error("could not register filter", zap.Error(err))
continue
}
pipe.Add(ff)
flog.Debug("registered filter")
}
// add default postfilter on async
// routes if not present
if r.meta.async {
log.Info("registering default postfilter", zap.Error(err))
pipe.Add(&pipeline.Worker{
Handler: defaultPostFilter.Handler(),
Name: defaultPostFilter.String(),
Type: types.PostFilter,
Weight: math.MaxInt8,
})
}
r.handler = pipe.Handler()
r.errHandler = pipe.Error()
log.Debug("successfully registered route")
}
}
func (s *apigw) AppendRoutes(routes ...*route) {
var (
rMap = make(map[string]*route)
uniq = func(r *route) string {
if routes == nil {
return ""
}
return r.method + r.endpoint
}
)
for _, r := range routes {
if r == nil {
continue
}
rMap[uniq(r)] = r
}
// update existing routes
for i, r := range s.routes {
if val, ok := rMap[uniq(r)]; ok && val != nil {
s.routes[i] = val
rMap[uniq(r)] = nil
}
}
// add new routes
for _, r := range rMap {
if r == nil {
continue
}
s.routes = append(s.routes, r)
}
return
}
func (s *apigw) NotFound(_ context.Context, method, endpoint string) {
if s.mx == nil || len(method) == 0 || len(endpoint) == 0 {
return
}
var (
defaultResponse = helperDefaultResponse(s.cfg, s.pr, s.log)
)
// Attach 404 handler
s.mx.Method(method, endpoint, defaultResponse)
}
func (s *apigw) registerFilter(f *st.ApigwFilter, r *route) (ff *pipeline.Worker, err error) {
handler, err := s.reg.Get(f.Ref)
if err != nil {
return
}
enc, err := json.Marshal(f.Params)
if err != nil {
err = fmt.Errorf("could not load params for filter: %s", err)
return
}
handler, err = s.reg.Merge(handler, enc, s.cfg)
if err != nil {
err = fmt.Errorf("could not merge params to handler: %s", err)
return
}
ff = &pipeline.Worker{
Async: r.meta.async && f.Kind == string(types.Processer),
Handler: handler.Handler(),
Name: handler.String(),
Type: types.FilterKind(f.Kind),
Weight: filter.FilterWeight(int(f.Weight), types.FilterKind(f.Kind)),
}
return
}
func (s *apigw) Funcs(kind string) (list types.FilterMetaList) {
list = s.reg.All()
if kind != "" {
list, _ = list.Filter(func(fm *types.FilterMeta) (bool, error) {
return string(fm.Kind) == kind, nil
})
}
return
}
func (s *apigw) ProxyAuthDef() (list []*proxy.ProxyAuthDefinition) {
list = proxy.ProxyAuthDef()
return
}
func (s *apigw) UpdateSettings(ctx context.Context, cfg types.Config) {
s.cfg = cfg
s.reg = registry.NewRegistry(cfg)
s.reg.Preload()
s.Reload(ctx)
}
func (s *apigw) loadRoutes(ctx context.Context) (rr []*route, err error) {
var (
routes st.ApigwRouteSet
agwf = st.ApigwRouteFilter{
Deleted: f.StateExcluded,
Disabled: f.StateExcluded,
}
)
routes, _, err = s.storer.SearchApigwRoutes(ctx, agwf)
if err != nil {
return
}
for _, r := range routes {
route := &route{
ID: r.ID,
endpoint: r.Endpoint,
method: r.Method,
meta: routeMeta{
debug: r.Meta.Debug,
async: r.Meta.Async,
},
}
rr = append(rr, route)
}
return
}
func (s *apigw) loadRoute(ctx context.Context, method, endpoint string) (rr []*route, err error) {
var (
routes st.ApigwRouteSet
agwf = st.ApigwRouteFilter{
Endpoint: endpoint,
Method: method,
Deleted: f.StateExcluded,
Disabled: f.StateExcluded,
}
)
routes, _, err = s.storer.SearchApigwRoutes(ctx, agwf)
if err != nil {
return
}
for _, r := range routes {
rr = append(rr, &route{
ID: r.ID,
endpoint: r.Endpoint,
method: r.Method,
meta: routeMeta{
debug: r.Meta.Debug,
async: r.Meta.Async,
},
})
}
return
}
func (s *apigw) loadFilters(ctx context.Context, route uint64) (ff []*st.ApigwFilter, err error) {
ff, _, err = s.storer.SearchApigwFilters(ctx, st.ApigwFilterFilter{
RouteID: route,
Deleted: f.StateExcluded,
Disabled: f.StateExcluded,
})
return
}
func (s *apigw) loadInfo() {
s.log.Info("loading Integration Gateway")
if s.cfg.Profiler.Enabled {
if !s.cfg.Profiler.Global {
s.log.Warn("profiler enabled only for routes with a profiler prefilter, use global setting to enable for all (APIGW_PROFILER_GLOBAL)")
}
} else {
if s.cfg.Profiler.Global {
s.log.Warn("profiler global is enabled, but profiler disabled, no routes will be profiled",
zap.Bool("APIGW_PROFILER_ENABLED", s.cfg.Profiler.Enabled),
zap.Bool("APIGW_PROFILER_GLOBAL", s.cfg.Profiler.Global))
}
}
}
func (s *apigw) Profiler() *profiler.Profiler {
return s.pr
}

View File

@@ -0,0 +1,249 @@
package apigw
import (
"context"
"fmt"
"testing"
"github.com/cortezaproject/corteza/server/pkg/apigw/registry"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
st "github.com/cortezaproject/corteza/server/system/types"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
type (
// overriding types.MockHandler with only
// the merge function
mockExistingHandler struct {
*types.MockHandler
merge func(params []byte) (types.Handler, error)
}
)
func Test_serviceLoadRoutes(t *testing.T) {
var (
ctx = context.Background()
req = require.New(t)
)
mockStorer := &types.MockStorer{
R: func(c context.Context, arf st.ApigwRouteFilter) (s st.ApigwRouteSet, f st.ApigwRouteFilter, err error) {
s = st.ApigwRouteSet{
{ID: 1, Endpoint: "/endpoint", Method: "GET", Enabled: true, Group: 0},
{ID: 2, Endpoint: "/endpoint2", Method: "POST", Enabled: true, Group: 0},
}
return
},
}
service := &apigw{
storer: mockStorer,
}
r, err := service.loadRoutes(ctx)
req.NoError(err)
req.Len(r, 2)
}
func Test_serviceLoadFunctions(t *testing.T) {
var (
ctx = context.Background()
req = require.New(t)
)
mockStorer := &types.MockStorer{
F: func(c context.Context, aff st.ApigwFilterFilter) (s st.ApigwFilterSet, f st.ApigwFilterFilter, err error) {
s = st.ApigwFilterSet{
{ID: 1, Route: 1},
{ID: 2, Route: 2},
}
return
},
}
service := &apigw{
storer: mockStorer,
}
r, err := service.loadFilters(ctx, 2)
req.NoError(err)
req.Len(r, 2)
}
func Test_serviceInit(t *testing.T) {
type (
tf struct {
name string
expLen int
st types.MockStorer
reg map[string]types.Handler
}
)
var (
tcc = []tf{
{
name: "could not register 1 function for route",
st: types.MockStorer{
R: func(c context.Context, arf st.ApigwRouteFilter) (s st.ApigwRouteSet, f st.ApigwRouteFilter, err error) {
s = st.ApigwRouteSet{
{ID: 1, Endpoint: "/endpoint", Method: "GET", Enabled: true, Group: 0},
}
return
},
F: func(c context.Context, aff st.ApigwFilterFilter) (s st.ApigwFilterSet, f st.ApigwFilterFilter, err error) {
s = st.ApigwFilterSet{
{ID: 1, Route: 1, Ref: "testExistingFilter"},
{ID: 2, Route: 1, Ref: "testNotExistingFunction"},
}
return
},
},
reg: map[string]types.Handler{"testExistingFilter": &types.MockHandler{}},
expLen: 1,
},
{
name: "successful register of 2 functions for route",
st: types.MockStorer{
R: func(c context.Context, arf st.ApigwRouteFilter) (s st.ApigwRouteSet, f st.ApigwRouteFilter, err error) {
s = st.ApigwRouteSet{
{ID: 1, Endpoint: "/endpoint", Method: "GET", Enabled: true, Group: 0},
}
return
},
F: func(c context.Context, aff st.ApigwFilterFilter) (s st.ApigwFilterSet, f st.ApigwFilterFilter, err error) {
s = st.ApigwFilterSet{
{ID: 1, Route: 1, Ref: "testExistingFilter"},
{ID: 2, Route: 1, Ref: "testExistingFilter"},
}
return
},
},
reg: map[string]types.Handler{"testExistingFilter": &types.MockHandler{}},
expLen: 2,
},
{
name: "could not merge params for function",
st: types.MockStorer{
R: func(c context.Context, arf st.ApigwRouteFilter) (s st.ApigwRouteSet, f st.ApigwRouteFilter, err error) {
s = st.ApigwRouteSet{
{ID: 1, Endpoint: "/endpoint", Method: "GET", Enabled: true, Group: 0},
}
return
},
F: func(c context.Context, aff st.ApigwFilterFilter) (s st.ApigwFilterSet, f st.ApigwFilterFilter, err error) {
s = st.ApigwFilterSet{
{ID: 1, Route: 1, Ref: "testExistingFilter", Params: st.ApigwFilterParams{}},
}
return
},
},
reg: map[string]types.Handler{"testExistingFilter": &mockExistingHandler{
MockHandler: &types.MockHandler{},
merge: func(params []byte) (types.Handler, error) {
return nil, fmt.Errorf("testttt")
},
}},
expLen: 0,
},
}
)
for _, tc := range tcc {
t.Run(tc.name, func(t *testing.T) {
var (
req = require.New(t)
ctx = context.Background()
)
reg := registry.NewRegistry(types.Config{})
for hn, h := range tc.reg {
reg.Add(hn, h)
}
service := &apigw{
log: zap.NewNop(),
storer: tc.st,
reg: reg,
}
rr, err := service.loadRoutes(ctx)
req.NoError(err)
service.Init(ctx, rr...)
req.NotEmpty(service.routes)
})
}
}
func (h mockExistingHandler) Merge(params []byte, cfg types.Config) (types.Handler, error) {
return h.merge(params)
}
func Test_serviceAppendRoutes(t *testing.T) {
var (
req = require.New(t)
routes = func(rr ...*route) []*route {
return rr
}
r1 = &route{
method: "GET",
endpoint: "/test",
}
r2 = &route{
method: "POST",
endpoint: "/test",
}
r3 = &route{
method: "PUT",
endpoint: "/test",
}
r4 = &route{
method: "DELETE",
endpoint: "/test",
}
r5 = &route{
endpoint: "GET",
method: "/test2",
}
tests = []struct {
name string
ag apigw
routes []*route
expected []*route
}{
{
name: "add new",
ag: apigw{
routes: routes(r1, r2, r3),
},
routes: routes(r4),
expected: routes(r1, r2, r3, r4),
},
{
name: "no duplicates",
ag: apigw{
routes: routes(r1, r2, r3, r4, r5),
},
routes: routes(r1, r5),
expected: routes(r1, r2, r3, r4, r5),
},
}
)
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.ag.AppendRoutes(tt.routes...)
req.Equal(tt.expected, tt.ag.routes)
})
}
}

View File

@@ -0,0 +1,19 @@
package types
import "time"
type (
Config struct {
Enabled bool
Profiler struct {
Enabled bool
Global bool
}
Proxy struct {
FollowRedirects bool
OutboundTimeout time.Duration
}
}
)

View File

@@ -0,0 +1,27 @@
package types
import (
"net/http"
"github.com/cortezaproject/corteza/server/pkg/errors"
"go.uber.org/zap"
)
type (
DefaultErrorHandler struct {
log *zap.Logger
}
)
func NewDefaultErrorHandler(log *zap.Logger) DefaultErrorHandler {
return DefaultErrorHandler{
log: log,
}
}
func (h DefaultErrorHandler) Handler() ErrorHandlerFunc {
return func(rw http.ResponseWriter, r *http.Request, err error) {
errors.ProperlyServeHTTP(rw, r, err, true)
h.log.Error(err.Error())
}
}

View File

@@ -0,0 +1,43 @@
package types
const (
PreFilter FilterKind = "prefilter"
PostFilter FilterKind = "postfilter"
Processer FilterKind = "processer"
)
type (
FilterKind string
FilterMeta struct {
Wgt int `json:"-"`
Name string `json:"name"`
Label string `json:"label"`
Kind FilterKind `json:"kind"`
Args []*FilterMetaArg `json:"params,omitempty"`
}
FilterMetaList []*FilterMeta
FilterMetaArg struct {
Label string `json:"label"`
Type string `json:"type"`
Example string `json:"example"`
Options map[string]interface{} `json:"options"`
}
)
func (fm FilterMetaList) Filter(f func(*FilterMeta) (bool, error)) (out FilterMetaList, err error) {
var ok bool
out = FilterMetaList{}
for i := range fm {
if ok, err = f(fm[i]); err != nil {
return
} else if ok {
out = append(out, fm[i])
}
}
return
}

View File

@@ -0,0 +1,29 @@
package types
import (
"fmt"
"net/http"
)
type (
HTTPHandler interface {
Handler() HandlerFunc
}
HTTPErrorHandler interface {
Handler() ErrorHandlerFunc
}
Handler interface {
HTTPHandler
fmt.Stringer
New(Config) Handler
Merge([]byte, Config) (Handler, error)
Meta() FilterMeta
Enabled() bool
}
HandlerFunc func(rw http.ResponseWriter, r *http.Request) error
ErrorHandlerFunc func(rw http.ResponseWriter, r *http.Request, err error)
)

View File

@@ -0,0 +1,122 @@
package types
import (
"fmt"
"net/http"
h "github.com/cortezaproject/corteza/server/pkg/http"
"github.com/cortezaproject/corteza/server/pkg/options"
)
type (
Scp map[string]interface{}
)
func (s Scp) Keys() (kk []string) {
for i := range s {
kk = append(kk, i)
}
return
}
func (s Scp) Request() *h.Request {
if _, ok := s["request"]; ok {
return s["request"].(*h.Request)
}
return nil
}
func (s Scp) Writer() http.ResponseWriter {
if _, ok := s["writer"]; ok {
return s["writer"].(http.ResponseWriter)
}
return nil
}
// Opts returns gateway options from the scope
//
// Routes store a Config (built from settings), older callers may store the
// options struct itself. Never returns nil; callers dereference the result
// directly.
func (s Scp) Opts() *options.ApigwOpt {
if ss, ok := s["opts"]; ok {
switch sss := ss.(type) {
case *options.ApigwOpt:
return withOptDefaults(sss)
case options.ApigwOpt:
return withOptDefaults(&sss)
case *Config:
return withOptDefaults(sss.toOpt())
case Config:
return withOptDefaults(sss.toOpt())
}
}
return withOptDefaults(&options.ApigwOpt{})
}
// toOpt maps the gateway config onto the options struct
//
// ProxyEnableDebugLog has no counterpart in settings and stays off.
func (c Config) toOpt() *options.ApigwOpt {
return &options.ApigwOpt{
Enabled: c.Enabled,
ProfilerEnabled: c.Profiler.Enabled,
ProfilerGlobal: c.Profiler.Global,
ProxyFollowRedirects: c.Proxy.FollowRedirects,
ProxyOutboundTimeout: c.Proxy.OutboundTimeout,
}
}
// withOptDefaults fills in values that would otherwise break the caller
//
// An unset outbound timeout yields an already-expired context, so fall back
// to the same default the options carry.
func withOptDefaults(o *options.ApigwOpt) *options.ApigwOpt {
if o.ProxyOutboundTimeout <= 0 {
o.ProxyOutboundTimeout = options.Apigw().ProxyOutboundTimeout
}
return o
}
func (s Scp) Set(k string, v interface{}) {
s[k] = v
}
func (s Scp) Get(k string) (v interface{}, err error) {
var ok bool
if v, ok = s[k]; !ok {
err = fmt.Errorf("could not get key on index: %s", k)
return
}
return
}
func (s *Scp) Dict() map[string]interface{} {
return *s
}
func (s *Scp) Filter(fn func(k string, v interface{}) bool) *Scp {
ss := Scp{}
for k, v := range *s {
if !fn(k, v) {
continue
}
ss[k] = v
}
return &ss
}
func (s Scp) Has(k string) (has bool) {
_, has = s[k]
return
}

View File

@@ -0,0 +1,104 @@
package types
import (
"testing"
"time"
"github.com/cortezaproject/corteza/server/pkg/options"
"github.com/stretchr/testify/require"
)
func TestScp_Opts(t *testing.T) {
var (
// what routes actually put in the scope
cfg = func() (c Config) {
c.Enabled = true
c.Profiler.Enabled = true
c.Profiler.Global = true
c.Proxy.FollowRedirects = true
c.Proxy.OutboundTimeout = time.Second * 5
return
}()
noTimeout = func() (c Config) {
c.Enabled = true
return
}()
defaultTimeout = options.Apigw().ProxyOutboundTimeout
)
tests := []struct {
name string
set func(s Scp)
want *options.ApigwOpt
}{
{
name: "config, as stored by the route",
set: func(s Scp) { s.Set("opts", cfg) },
want: &options.ApigwOpt{
Enabled: true,
ProfilerEnabled: true,
ProfilerGlobal: true,
ProxyFollowRedirects: true,
ProxyOutboundTimeout: time.Second * 5,
},
},
{
name: "config pointer",
set: func(s Scp) { s.Set("opts", &cfg) },
want: &options.ApigwOpt{
Enabled: true,
ProfilerEnabled: true,
ProfilerGlobal: true,
ProxyFollowRedirects: true,
ProxyOutboundTimeout: time.Second * 5,
},
},
{
// an unset timeout yields an already expired context
name: "config without an outbound timeout",
set: func(s Scp) { s.Set("opts", noTimeout) },
want: &options.ApigwOpt{Enabled: true, ProxyOutboundTimeout: defaultTimeout},
},
{
name: "options value",
set: func(s Scp) { s.Set("opts", options.ApigwOpt{Debug: true, ProxyOutboundTimeout: time.Second}) },
want: &options.ApigwOpt{Debug: true, ProxyOutboundTimeout: time.Second},
},
{
name: "options pointer keeps the debug log flag",
set: func(s Scp) {
s.Set("opts", &options.ApigwOpt{ProxyEnableDebugLog: true, ProxyOutboundTimeout: time.Second})
},
want: &options.ApigwOpt{ProxyEnableDebugLog: true, ProxyOutboundTimeout: time.Second},
},
{
name: "nothing in scope",
set: func(s Scp) {},
want: &options.ApigwOpt{ProxyOutboundTimeout: defaultTimeout},
},
{
name: "unexpected type in scope",
set: func(s Scp) { s.Set("opts", "nonsense") },
want: &options.ApigwOpt{ProxyOutboundTimeout: defaultTimeout},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var (
req = require.New(t)
s = Scp{}
)
tt.set(s)
opts := s.Opts()
// callers dereference this without checking
req.NotNil(opts)
req.Equal(tt.want, opts)
})
}
}

View File

@@ -0,0 +1,15 @@
package types
type (
Settings struct {
Enabled bool
ProfilerGlobal bool
ProfilerEnabled bool
}
AddSettings func(s Setter)
Setter interface {
UpdateSettings(st ...AddSettings)
}
)

View File

@@ -0,0 +1,5 @@
package types
type (
SecureStorager interface{}
)

View File

@@ -0,0 +1,76 @@
package types
import (
"bytes"
"context"
"encoding/json"
"net/http"
st "github.com/cortezaproject/corteza/server/system/types"
)
type (
MockExecer struct {
Exec_ func(context.Context, *Scp) (err error)
Type_ func() FilterKind
}
MockErrorHandler struct {
Handler_ ErrorHandlerFunc
}
MockHandler struct {
Foo string `json:"foo"`
Handler_ HandlerFunc
}
MockStorer struct {
F func(context.Context, st.ApigwFilterFilter) (st.ApigwFilterSet, st.ApigwFilterFilter, error)
R func(context.Context, st.ApigwRouteFilter) (st.ApigwRouteSet, st.ApigwRouteFilter, error)
}
MockRoundTripper func(*http.Request) (*http.Response, error)
)
func (h MockHandler) New(cfg Config) Handler {
return MockHandler{}
}
func (h MockHandler) Enabled() bool {
return true
}
func (h MockHandler) String() string {
return "MockHandler"
}
func (h MockHandler) Handler() HandlerFunc {
return h.Handler_
}
func (h MockHandler) Merge(params []byte, cfg Config) (Handler, error) {
err := json.NewDecoder(bytes.NewBuffer(params)).Decode(&h)
return h, err
}
func (h MockHandler) Meta() FilterMeta {
return FilterMeta{
Name: "return mocked filter metadata",
}
}
func (td MockStorer) SearchApigwRoutes(ctx context.Context, f st.ApigwRouteFilter) (s st.ApigwRouteSet, ff st.ApigwRouteFilter, err error) {
return td.R(ctx, f)
}
func (td MockStorer) SearchApigwFilters(ctx context.Context, f st.ApigwFilterFilter) (s st.ApigwFilterSet, ff st.ApigwFilterFilter, err error) {
return td.F(ctx, f)
}
func (h MockErrorHandler) Handler() ErrorHandlerFunc {
return h.Handler_
}
func (mrt MockRoundTripper) RoundTrip(rq *http.Request) (r *http.Response, err error) {
return mrt(rq)
}