snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)
Some checks failed
SonarQube / SonarQube (push) Failing after 5m28s
E2e test on main branch push / server and client test (map[name:admin port:8080]) (push) Failing after 3m33s
Server and lib tests / Library test (js) (push) Failing after 1m36s
Server and lib tests / Library test (vue) (push) Successful in 2m24s
Server and lib tests / Web client test (admin) (push) Successful in 2m52s
Server and lib tests / Web client test (compose) (push) Successful in 3m23s
Server and lib tests / Web client test (discovery) (push) Successful in 2m49s
Server and lib tests / Web client test (one) (push) Successful in 2m52s
Server and lib tests / Web client test (privacy) (push) Successful in 2m44s
Server and lib tests / Web client test (reporter) (push) Successful in 2m56s
Server and lib tests / Web client test (workflow) (push) Successful in 3m0s
Server and lib tests / Server tests (push) Failing after 4m55s
E2e test on main branch push / notify-failure (push) Failing after 9s
Server and lib tests / Send matrix testing notification (push) Successful in 2s
Mark stale issues and pull requests / stale (push) Successful in 40s

This commit is contained in:
upstream-mirror
2026-09-18 01:22:23 +05:00
commit 7e385fc126
8391 changed files with 1766236 additions and 0 deletions

View File

@@ -0,0 +1,40 @@
package auth
import (
"context"
"encoding/base64"
"fmt"
)
type (
ServicerBasic struct {
user string
pass string
}
BasicParams struct {
User string
Pass string
}
)
func NewBasic(p BasicParams) (s ServicerBasic, err error) {
if p.User == "" {
err = fmt.Errorf("invalid param username")
return
}
if p.Pass == "" {
err = fmt.Errorf("invalid param password")
return
}
s = ServicerBasic{user: p.User, pass: p.Pass}
return
}
func (s ServicerBasic) Do(ctx context.Context) string {
auth := s.user + ":" + s.pass
return base64.StdEncoding.EncodeToString([]byte(auth))
}

View File

@@ -0,0 +1,60 @@
package auth
import (
"context"
"testing"
"github.com/stretchr/testify/require"
)
func Test_basic(t *testing.T) {
type (
tf struct {
name string
err string
exp string
params BasicParams
}
)
var (
tcc = []tf{
{
name: "match basic headers fail username validation",
err: "invalid param username",
params: BasicParams{},
},
{
name: "match basic headers fail password validation",
err: "invalid param password",
params: BasicParams{
User: "user",
},
},
{
name: "match basic headers success",
params: BasicParams{
User: "thou",
Pass: "shallnotpass",
},
exp: "dGhvdTpzaGFsbG5vdHBhc3M=",
},
}
)
for _, tc := range tcc {
t.Run(tc.name, func(t *testing.T) {
var (
req = require.New(t)
s, err = NewBasic(tc.params)
)
if tc.err != "" {
req.EqualError(err, tc.err)
return
}
req.Equal(tc.exp, s.Do(context.Background()))
})
}
}

View File

@@ -0,0 +1,70 @@
package auth
import (
"context"
"fmt"
"net/http"
"net/url"
"github.com/cortezaproject/corteza/server/pkg/apigw/types"
"golang.org/x/oauth2"
"golang.org/x/oauth2/clientcredentials"
)
type (
ServicerOauth2 struct {
c *http.Client
client string
secret string
scope []string
tokenUrl *url.URL
}
Oauth2Params struct {
Client string
Secret string
Scope []string
TokenUrl *url.URL
}
)
func NewOauth2(p Oauth2Params, c *http.Client, s types.SecureStorager) (ss ServicerOauth2, err error) {
if p.Client == "" {
err = fmt.Errorf("invalid param client")
return
}
if p.Secret == "" {
err = fmt.Errorf("invalid param secret")
return
}
if p.TokenUrl == nil || p.TokenUrl.String() == "" {
err = fmt.Errorf("invalid param token url")
return
}
ss = ServicerOauth2{
c: c,
client: p.Client,
secret: p.Secret,
scope: p.Scope,
tokenUrl: p.TokenUrl,
}
return
}
func (s ServicerOauth2) Do(ctx context.Context) (t *oauth2.Token, err error) {
c := &clientcredentials.Config{
ClientID: s.client,
ClientSecret: s.secret,
Scopes: s.scope,
TokenURL: s.tokenUrl.String(),
}
ctx = context.WithValue(ctx, oauth2.HTTPClient, s.c)
return c.Token(ctx)
}

View File

@@ -0,0 +1,82 @@
package auth
import (
"net/http"
"net/url"
"testing"
"github.com/stretchr/testify/require"
)
func Test_oauth2(t *testing.T) {
type (
tf struct {
name string
err string
exp string
params Oauth2Params
}
)
var (
tcc = []tf{
{
name: "match oauth2 fail client validation",
err: "invalid param client",
params: Oauth2Params{},
},
{
name: "match oauth2 fail secret key validation",
err: "invalid param secret",
params: Oauth2Params{
Client: "client_ID",
},
},
{
name: "match oauth2 fail url validation",
err: "invalid param token url",
params: Oauth2Params{
Client: "client_ID",
Secret: "secret_KEY",
},
},
{
name: "match oauth2 fail url validation",
err: "invalid param token url",
params: Oauth2Params{
Client: "client_ID",
Secret: "secret_KEY",
TokenUrl: &url.URL{},
},
},
{
name: "match oauth2 fail validation",
params: Oauth2Params{
Client: "client_ID",
Secret: "secret_KEY",
TokenUrl: generateURL("http://example.com"),
},
},
}
)
for _, tc := range tcc {
t.Run(tc.name, func(t *testing.T) {
var (
req = require.New(t)
c = http.DefaultClient
_, err = NewOauth2(tc.params, c, struct{}{})
)
if tc.err != "" {
req.EqualError(err, tc.err)
return
}
})
}
}
func generateURL(s string) (u *url.URL) {
u, _ = url.Parse(s)
return
}

164
server/pkg/http/client.go Normal file
View File

@@ -0,0 +1,164 @@
package http
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net"
"net/http"
"net/http/httputil"
"strings"
"time"
"errors"
)
type (
Config struct {
BaseURL string
Timeout int
}
Client struct {
Transport *http.Transport
Client *http.Client
debugLevel DebugLevel
config *Config
}
DebugLevel string
)
const (
INFO DebugLevel = "info"
FULL DebugLevel = "full"
)
func New(flags *Config) (*Client, error) {
timeout := time.Duration(flags.Timeout) * time.Second
transport := &http.Transport{
Dial: (&net.Dialer{
Timeout: timeout,
}).Dial,
TLSHandshakeTimeout: timeout,
}
// @todo migrate to http.DefaultClient & http.DefaultTransport, see internal/http.SetupDefaults
client := &http.Client{
Timeout: timeout,
Transport: transport,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
}
return &Client{
Transport: transport,
Client: client,
config: flags,
}, nil
}
func (c *Client) Debug(level DebugLevel) *Client {
c.debugLevel = level
return c
}
func (c *Client) Get(url string) (*http.Request, error) {
return c.Request("GET", url, nil)
}
func (c *Client) Post(url string, body interface{}) (*http.Request, error) {
return c.Request("POST", url, body)
}
func (c *Client) Patch(url string, body interface{}) (*http.Request, error) {
return c.Request("PATCH", url, body)
}
func (c *Client) Delete(url string) (*http.Request, error) {
return c.Request("DELETE", url, nil)
}
func (c *Client) Request(method, url string, body interface{}) (*http.Request, error) {
if c.config.BaseURL != "" {
url = strings.TrimRight(c.config.BaseURL, "/") + "/" + strings.TrimLeft(url, "/")
}
if c.debugLevel == "info" {
fmt.Println("HTTP >>>", method, url)
}
request := func() (*http.Request, error) {
if body != nil {
b, err := json.Marshal(body)
if err != nil {
return nil, err
}
return http.NewRequest(method, url, bytes.NewBuffer(b))
}
return http.NewRequest(method, url, nil)
}
req, err := request()
if err != nil {
return nil, fmt.Errorf("creating request failed: %w", err)
}
req.Header.Add("Content-Type", "application/json")
return req, nil
}
func (c *Client) Do(req *http.Request) (*http.Response, error) {
if c.debugLevel == FULL {
fmt.Println("HTTP >>> (request)")
b, err := httputil.DumpRequestOut(req, true)
if err != nil {
fmt.Println("HTTP >>> Error:", err)
} else {
if b != nil {
fmt.Println(strings.TrimSpace(string(b)))
}
}
fmt.Println("---")
}
resp, err := c.Client.Do(req)
if c.debugLevel == FULL {
fmt.Println("HTTP <<< (response)")
if err != nil {
fmt.Println("HTTP <<< Error:", err)
} else {
b, err := httputil.DumpResponse(resp, true)
if err != nil {
fmt.Println("HTTP <<< Error:", err)
} else {
if b != nil {
fmt.Println(string(b))
}
}
}
fmt.Println("-----------------")
}
if err != nil {
if c.debugLevel == INFO {
fmt.Println("HTTP <<< Response error", err)
}
return nil, fmt.Errorf("request failed: %w", err)
}
if c.debugLevel == INFO {
fmt.Println("HTTP <<< Response", resp.StatusCode)
}
return resp, nil
}
func ToError(resp *http.Response) error {
body, err := ioutil.ReadAll(resp.Body)
if body == nil || err != nil {
return fmt.Errorf("unexpected response (%d, %s)", resp.StatusCode, err)
}
return errors.New(string(body))
}

View File

@@ -0,0 +1,14 @@
package http
import (
"fmt"
"net/http"
)
type Fortune struct{}
func (*Fortune) ServeHTTP(w http.ResponseWriter, r *http.Request) {
fortune := "Fortune favors the prepared mind. - Louis Pasteur"
_, _ = fmt.Fprint(w, fortune)
}

View File

@@ -0,0 +1,38 @@
package http
import (
"net/http/httptest"
"testing"
"github.com/stretchr/testify/require"
)
func TestHTTPClient(t *testing.T) {
handler := &Fortune{}
server := httptest.NewServer(handler)
defer server.Close()
client, err := New(&Config{
Timeout: 10,
})
require.True(t, err == nil, "%+v", err)
client.Debug(FULL)
req, err := client.Get(server.URL)
require.True(t, err == nil, "%+v", err)
resp, err := client.Do(req)
require.True(t, err == nil, "%+v", err)
err = func() error {
defer resp.Body.Close()
switch resp.StatusCode {
case 200:
return nil
default:
return ToError(resp)
}
}()
require.True(t, err == nil, "Invalid response: %+v", err)
}

View File

@@ -0,0 +1,34 @@
package http
import (
"crypto/tls"
"net"
"net/http"
"time"
)
// SetupDefaults Reconfigures defaults for HTTP client & transport
func SetupDefaults(timeout time.Duration, tlsInsecure bool) {
if tlsInsecure {
// This will allow HTTPS requests to insecure hosts (expired, wrong host, self signed, untrusted root...)
// With this enabled, features like OIDC auto-discovery should work on any of examples found on badssl.com.
//
// With SYSTEM_HTTP_CLIENT_TSL_INSECURE=0 (default) next command returns 404 error (expected)
// > ./system external-auth auto-discovery foo-tls-1 https://expired.badssl.com/
//
// Without SYSTEM_HTTP_CLIENT_TSL_INSECURE=1 next command returns "x509: certificate has expired or is not yet valid"
// > ./system external-auth auto-discovery foo-tls-1 https://expired.badssl.com/
//
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
http.DefaultTransport.(*http.Transport).DialContext = (&net.Dialer{Timeout: timeout}).DialContext
http.DefaultTransport.(*http.Transport).TLSHandshakeTimeout = timeout
}
if timeout > 0 {
http.DefaultClient.Timeout = timeout
}
http.DefaultClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
}
}

View File

@@ -0,0 +1,152 @@
package http
import (
"io/fs"
"net/http"
"path"
"strings"
"github.com/go-chi/chi/v5"
"go.uber.org/zap"
)
type (
// special file-server meant to be used for serving single page applications
// or anything that needs a bit of extra attention like fallback handling
// 404 handler, error handler and URL prefixing
fileServer struct {
// fileServer files
files http.FileSystem
urlPrefix string
fallbacks []string
// Final not-found handler
notFound http.HandlerFunc
// how errors are handled
errHandler func(w http.ResponseWriter, error string, code int)
logger *zap.Logger
}
configurator func(*fileServer) error
)
// MountSPA helper function, preconfigures FileServer for SPA serving
// and mounts it to chi Router
func MountSPA(r chi.Router, path string, root fs.FS, cc ...configurator) error {
path = "/" + strings.Trim(strings.TrimRight(path, "*"), "/") + "/"
cc = append(
[]configurator{UrlPrefix(path), Fallbacks("index.html")},
// appnd all configurators at the end and allow override of prefix & fallbacks
cc...
)
handler, err := FileServer(root, cc...)
if err != nil {
return err
}
r.Handle(
strings.TrimRight(path, "/"),
http.RedirectHandler("."+path, http.StatusTemporaryRedirect),
)
r.Handle(path+"*", handler)
return nil
}
func FileServer(files fs.FS, cc ...configurator) (h *fileServer, err error) {
h = &fileServer{
files: http.FS(files),
notFound: http.NotFound,
errHandler: http.Error,
logger: zap.NewNop(),
}
for _, configure := range cc {
if err = configure(h); err != nil {
return
}
}
return
}
func UrlPrefix(prefix string) configurator {
return func(s *fileServer) error {
s.urlPrefix = prefix
return nil
}
}
func Fallbacks(ff ...string) configurator {
return func(s *fileServer) error {
s.fallbacks = ff
return nil
}
}
func NotFound(h http.HandlerFunc) configurator {
return func(s *fileServer) error {
s.notFound = h
return nil
}
}
func Logger(l *zap.Logger) configurator {
return func(s *fileServer) error {
s.logger = l
return nil
}
}
// Serves the single-page-application
//
// This is file-server with some special logic for handling missing
// files (404s) and directories.
// In both cases we serve index file directly
func (h *fileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// catch requests for non-existing files and redirect to index.html
if h.files == nil {
h.errHandler(w, "configured without files", http.StatusInternalServerError)
return
}
trimmed := path.Clean(strings.TrimPrefix(r.URL.Path, h.urlPrefix))
h.logger.Debug(r.URL.Path, zap.String("trimmed", trimmed), zap.String("urlPrefix", h.urlPrefix))
r.URL.Path = trimmed
var (
err error
fh http.File
st fs.FileInfo
)
for _, candidate := range append([]string{r.URL.Path}, h.fallbacks...) {
if len(candidate) == 0 {
continue
}
if fh, err = h.files.Open(candidate); err != nil {
continue
} else if st, err = fh.Stat(); err != nil {
continue
} else if st.IsDir() {
// index
continue
}
break
}
if fh == nil || st == nil {
h.notFound(w, r)
return
}
http.ServeContent(w, r, st.Name(), st.ModTime(), fh)
}

View File

@@ -0,0 +1,133 @@
package http
import (
"embed"
"io/fs"
"net/http"
"net/http/httptest"
"os"
"testing"
"github.com/stretchr/testify/require"
)
var (
//go:embed test_data/file_server
testAssets embed.FS
)
func TestSPA(t *testing.T) {
testAssetsSub, err := fs.Sub(testAssets, "test_data/file_server")
if err != nil {
t.Errorf("failed to dive into subdirectory: %v", err)
}
testDirectAssets := os.DirFS("test_data/file_server")
var (
cases = []struct {
name string
fs fs.FS
cc []configurator
url string
rsp string
}{
{
name: "no special config",
fs: testAssetsSub,
url: "index.html",
rsp: "index html\n",
},
{
name: "empty path",
fs: testAssetsSub,
cc: []configurator{Fallbacks("index.html")},
url: "",
rsp: "index html\n",
},
{
name: "slash",
fs: testAssetsSub,
cc: []configurator{Fallbacks("index.html")},
url: "/",
rsp: "index html\n",
},
{
name: "prefix, no slash",
fs: testAssetsSub,
cc: []configurator{UrlPrefix("/my-spa"), Fallbacks("index.html")},
url: "/my-spa",
rsp: "index html\n",
},
{
name: "sub dir with url prefix",
fs: testAssetsSub,
cc: []configurator{UrlPrefix("/my-spa"), Fallbacks("index.html")},
url: "/my-spa/index.html",
rsp: "index html\n",
},
{
name: "sub dir with url prefix, none existing file",
fs: testAssetsSub,
cc: []configurator{UrlPrefix("/my-spa"), Fallbacks("index.html")},
url: "/not-here",
rsp: "index html\n",
},
{
name: "sub dir with url prefix, dir",
fs: testAssetsSub,
cc: []configurator{UrlPrefix("/my-spa"), Fallbacks("index.html")},
url: "/my-spa/sub",
rsp: "index html\n",
},
{
name: "sub dir with url prefix, sub index",
fs: testAssetsSub,
cc: []configurator{UrlPrefix("/my-spa"), Fallbacks("index.html")},
url: "/my-spa/sub/index.html",
rsp: "sub index html\n",
},
{
name: "sub dir with url prefix, sub file",
fs: testAssetsSub,
cc: []configurator{UrlPrefix("/my-spa"), Fallbacks("index.html")},
url: "/my-spa/sub/test.html",
rsp: "sub test html\n",
},
}
)
// another set of test cases for direct fs
total := len(cases)
for c := 0; c < total; c++ {
direct := cases[c]
cases[c].name += "; embedded"
direct.name += "; direct"
direct.fs = testDirectAssets
cases = append(cases, direct)
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
var (
handler http.Handler
req = require.New(t)
w = httptest.NewRecorder()
r, err = http.NewRequest(http.MethodGet, c.url, nil)
)
handler, err = FileServer(c.fs, c.cc...)
req.NoError(err)
handler.ServeHTTP(w, r)
req.Equal(c.rsp, w.Body.String())
req.Equal(http.StatusOK, w.Result().StatusCode)
})
}
}

168
server/pkg/http/request.go Normal file
View File

@@ -0,0 +1,168 @@
package http
import (
"bytes"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/url"
"strings"
)
type (
Request struct {
*http.Request
Body io.Reader
}
// The BufferedReader behaves exactly like a bytes.Reader, with the exception
// when the last block is read, it automatically rewinds the internal pointer to the start,
// so effectively, the content can be read again without calling Seek() externally.
BufferedReader struct {
s []byte
i int64 // current reading index
prevRune int // index of previous rune; or < 0
}
)
const maxMemory = 1 << 30 // 1GB
func NewRequest(r *http.Request) (rr *Request, err error) {
rr = &Request{
Request: r,
}
contentType := r.Header.Get("Content-Type")
if strings.HasPrefix(contentType, "multipart/form-data") {
// @todo consider increasing the limit; should be plenty for now
var body []byte
body, err = io.ReadAll(io.LimitReader(r.Body, maxMemory))
if err != nil {
return
}
r.Body.Close()
// Save a copy of the body
bCopy := make([]byte, len(body))
copy(bCopy, body)
// Restore the body so we can parse it
r.Body = io.NopCloser(bytes.NewReader(body))
// Pull files (this consumes the body)
err = getFilesAsReaders(r)
if err != nil {
return
}
// Restore & parse
rr.Body, err = NewBufferedReader(bytes.NewBuffer(bCopy))
if err != nil {
return
}
} else {
// @todo support for others when needed?
rr.Body, err = NewBufferedReader(r.Body)
if err != nil {
return
}
}
return
}
// NewBufferedReader copies original data to the
// BufferedReader
func NewBufferedReader(rr io.Reader) (bb *BufferedReader, err error) {
var (
buf = &bytes.Buffer{}
)
bb = &BufferedReader{}
_, err = io.Copy(buf, rr)
if err != nil {
return
}
return &BufferedReader{
s: buf.Bytes(),
i: 0,
prevRune: -1,
}, nil
}
func (r *BufferedReader) Read(b []byte) (n int, err error) {
if r.i >= int64(len(r.s)) {
n = 0
err = io.EOF
r.Seek(0, io.SeekStart)
return
}
r.prevRune = -1
n = copy(b, r.s[r.i:])
r.i += int64(n)
return
}
func (r *BufferedReader) Seek(offset int64, whence int) (int64, error) {
r.prevRune = -1
var abs int64
switch whence {
case io.SeekStart:
abs = offset
case io.SeekCurrent:
abs = r.i + offset
case io.SeekEnd:
abs = int64(len(r.s)) + offset
default:
return 0, fmt.Errorf("bytes.Reader.Seek: invalid whence")
}
if abs < 0 {
return 0, fmt.Errorf("bytes.Reader.Seek: negative position")
}
r.i = abs
return abs, nil
}
func (bb *Request) MarshalJSON() ([]byte, error) {
return json.Marshal(&struct {
Method string
URL *url.URL
Header http.Header
ContentLength int64
Host string
Form url.Values
PostForm url.Values
MultipartForm *multipart.Form
RemoteAddr string
RequestURI string
}{
Method: bb.Method,
URL: bb.URL,
Header: bb.Header,
ContentLength: bb.ContentLength,
Host: bb.Host,
Form: bb.Form,
PostForm: bb.PostForm,
MultipartForm: bb.MultipartForm,
RemoteAddr: bb.RemoteAddr,
RequestURI: bb.RequestURI,
})
}
func getFilesAsReaders(r *http.Request) (err error) {
err = r.ParseMultipartForm(32 << 20)
if err != nil {
return fmt.Errorf("error parsing multipart form: %w", err)
}
if r.MultipartForm == nil {
return fmt.Errorf("no multipart form found")
}
return nil
}

View File

@@ -0,0 +1,43 @@
package http
import (
"io"
h "net/http"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func Test_requestReadMultiple(t *testing.T) {
var (
req = require.New(t)
tt = `Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi placerat suscipit finibus. Morbi luctus et lorem sed euismod. Donec bibendum lorem non justo pretium, a sagittis augue mollis. In varius libero id purus convallis pretium. Vestibulum ac mauris aliquet, pulvinar massa eu, rhoncus ipsum. Cras sit amet euismod metus, in tincidunt sapien. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Sed scelerisque vulputate imperdiet. Nulla orci magna, fringilla sit amet tempor vel, tempus pulvinar urna.`
)
r, _ := h.NewRequest("POST", "/foo", strings.NewReader(tt))
rs, err := NewRequest(r)
req.NoError(err)
req.Equal(tt, must(io.ReadAll(rs.Body)))
req.Equal(tt, must(io.ReadAll(rs.Body)))
}
func Test_requestReadMultipleNoBody(t *testing.T) {
var req = require.New(t)
r, _ := h.NewRequest("POST", "/foo", h.NoBody)
rs, err := NewRequest(r)
req.NoError(err)
req.Equal(``, must(io.ReadAll(rs.Body)))
req.Equal(``, must(io.ReadAll(rs.Body)))
}
func must(b []byte, e error) string {
if e != nil {
panic(e)
}
return string(b)
}

View File

@@ -0,0 +1 @@
index html

View File

@@ -0,0 +1 @@
sub index html

View File

@@ -0,0 +1 @@
sub test html

View File

@@ -0,0 +1 @@
test html