snapshot of https://github.com/ory/keto.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
CI / Run lints and checks (push) Failing after 47s
CI / Run tests (push) Failing after 1s
CI / Run race tests (push) Failing after 5s
CI / Build CLI docs (push) Has been skipped
CI / Run buf actions (push) Failing after 5s
CodeQL / Analyze (go) (push) Failing after 3s
CodeQL / Analyze (javascript) (push) Failing after 2s
Docker Image Scanners / scanners (push) Failing after 3s
Format / format (push) Failing after 3s
Synchronize Issue Labels / Synchronize Issue Labels (push) Has been skipped
CI / Generate release (push) Has been skipped
CI / Draft newsletter (push) Has been skipped
CI / Pending approval Slack notification (push) Has been skipped
CI / Send newsletter (push) Has been skipped
Some checks failed
CI / Run lints and checks (push) Failing after 47s
CI / Run tests (push) Failing after 1s
CI / Run race tests (push) Failing after 5s
CI / Build CLI docs (push) Has been skipped
CI / Run buf actions (push) Failing after 5s
CodeQL / Analyze (go) (push) Failing after 3s
CodeQL / Analyze (javascript) (push) Failing after 2s
Docker Image Scanners / scanners (push) Failing after 3s
Format / format (push) Failing after 3s
Synchronize Issue Labels / Synchronize Issue Labels (push) Has been skipped
CI / Generate release (push) Has been skipped
CI / Draft newsletter (push) Has been skipped
CI / Pending approval Slack notification (push) Has been skipped
CI / Send newsletter (push) Has been skipped
This commit is contained in:
10
internal/driver/config/buildinfo.go
Normal file
10
internal/driver/config/buildinfo.go
Normal file
@@ -0,0 +1,10 @@
|
||||
// Copyright © 2023 Ory Corp
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package config
|
||||
|
||||
var (
|
||||
Version = "master"
|
||||
Date = "undefined"
|
||||
Commit = "undefined"
|
||||
)
|
||||
87
internal/driver/config/namespace_memory.go
Normal file
87
internal/driver/config/namespace_memory.go
Normal file
@@ -0,0 +1,87 @@
|
||||
// Copyright © 2023 Ory Corp
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"sync"
|
||||
|
||||
"github.com/ory/herodot"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/ory/keto/internal/namespace"
|
||||
)
|
||||
|
||||
type (
|
||||
memoryNamespaceManager struct {
|
||||
byName map[string]*namespace.Namespace
|
||||
sync.RWMutex
|
||||
}
|
||||
)
|
||||
|
||||
var _ namespace.Manager = &memoryNamespaceManager{}
|
||||
|
||||
func NewMemoryNamespaceManager(nn ...*namespace.Namespace) *memoryNamespaceManager {
|
||||
s := &memoryNamespaceManager{}
|
||||
s.set(nn)
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *memoryNamespaceManager) GetNamespaceByName(_ context.Context, name string) (*namespace.Namespace, error) {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
if n, ok := s.byName[name]; ok {
|
||||
return n, nil
|
||||
}
|
||||
|
||||
return nil, errors.WithStack(herodot.ErrNotFound().WithReasonf("Unknown namespace with name %q.", name))
|
||||
}
|
||||
|
||||
func (s *memoryNamespaceManager) GetNamespaceByConfigID(_ context.Context, id int32) (*namespace.Namespace, error) {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
for _, n := range s.byName {
|
||||
//lint:ignore SA1019 backwards compatibility
|
||||
//nolint:staticcheck
|
||||
if n.ID == id {
|
||||
return n, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, errors.WithStack(herodot.ErrNotFound().WithReasonf("Unknown namespace with id %d.", id))
|
||||
}
|
||||
|
||||
func (s *memoryNamespaceManager) Namespaces(_ context.Context) ([]*namespace.Namespace, error) {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
nn := make([]*namespace.Namespace, 0, len(s.byName))
|
||||
for _, n := range s.byName {
|
||||
nn = append(nn, n)
|
||||
}
|
||||
|
||||
return nn, nil
|
||||
}
|
||||
|
||||
func (s *memoryNamespaceManager) ShouldReload(newValue interface{}) bool {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
nn, _ := s.Namespaces(context.Background())
|
||||
|
||||
return !reflect.DeepEqual(newValue, nn)
|
||||
}
|
||||
|
||||
func (s *memoryNamespaceManager) set(nn []*namespace.Namespace) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
s.byName = make(map[string]*namespace.Namespace, len(nn))
|
||||
for _, n := range nn {
|
||||
s.byName[n.Name] = n
|
||||
}
|
||||
}
|
||||
25
internal/driver/config/namespace_memory_test.go
Normal file
25
internal/driver/config/namespace_memory_test.go
Normal file
@@ -0,0 +1,25 @@
|
||||
// Copyright © 2023 Ory Corp
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/ory/keto/internal/namespace"
|
||||
)
|
||||
|
||||
func TestNamespaceMemoryManager(t *testing.T) {
|
||||
t.Run("method=should reload", func(t *testing.T) {
|
||||
n := &namespace.Namespace{Name: "2"}
|
||||
nm := NewMemoryNamespaceManager(n)
|
||||
|
||||
assert.False(t, nm.ShouldReload([]*namespace.Namespace{n}))
|
||||
assert.False(t, nm.ShouldReload([]*namespace.Namespace{{Name: "2"}}))
|
||||
|
||||
assert.True(t, nm.ShouldReload([]*namespace.Namespace{{Name: "3"}}))
|
||||
assert.True(t, nm.ShouldReload("foo"))
|
||||
})
|
||||
}
|
||||
295
internal/driver/config/namespace_watcher.go
Normal file
295
internal/driver/config/namespace_watcher.go
Normal file
@@ -0,0 +1,295 @@
|
||||
// Copyright © 2023 Ory Corp
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/ghodss/yaml"
|
||||
"github.com/ory/herodot"
|
||||
"github.com/ory/x/logrusx"
|
||||
"github.com/ory/x/stringsx"
|
||||
"github.com/ory/x/urlx"
|
||||
"github.com/ory/x/watcherx"
|
||||
"github.com/pelletier/go-toml"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/ory/keto/internal/namespace"
|
||||
)
|
||||
|
||||
type (
|
||||
Parser func([]byte, interface{}) error
|
||||
|
||||
NamespaceFile struct {
|
||||
Name string
|
||||
Contents []byte
|
||||
Parser Parser
|
||||
|
||||
namespace *namespace.Namespace // last successfully parsed namespace. May be nil if there was a parse error
|
||||
}
|
||||
|
||||
NamespaceWatcher struct {
|
||||
sync.RWMutex
|
||||
namespaces map[string]*NamespaceFile
|
||||
logger *logrusx.Logger
|
||||
target string
|
||||
}
|
||||
|
||||
eventHandler interface {
|
||||
handleRemove(*watcherx.RemoveEvent)
|
||||
handleChange(*watcherx.ChangeEvent)
|
||||
handleError(*watcherx.ErrorEvent)
|
||||
}
|
||||
)
|
||||
|
||||
var _ namespace.Manager = (*NamespaceWatcher)(nil)
|
||||
|
||||
func NewNamespaceWatcher(ctx context.Context, l *logrusx.Logger, target string) (*NamespaceWatcher, error) {
|
||||
nw := NamespaceWatcher{
|
||||
logger: l,
|
||||
target: target,
|
||||
namespaces: make(map[string]*NamespaceFile),
|
||||
}
|
||||
|
||||
return &nw, watchTarget(ctx, target, &nw, l)
|
||||
}
|
||||
|
||||
func (nw *NamespaceWatcher) handleRemove(e *watcherx.RemoveEvent) {
|
||||
nw.Lock()
|
||||
defer nw.Unlock()
|
||||
|
||||
delete(nw.namespaces, e.Source())
|
||||
}
|
||||
|
||||
func (nw *NamespaceWatcher) handleChange(e *watcherx.ChangeEvent) {
|
||||
// the lock is acquired before parsing to ensure that the getters are
|
||||
// waiting for the updated values
|
||||
nw.Lock()
|
||||
defer nw.Unlock()
|
||||
|
||||
n := nw.readNamespaceFile(e.Reader(), e.Source())
|
||||
if n == nil {
|
||||
return
|
||||
} else if n.namespace == nil {
|
||||
// parse failed, rolling back to previous working version
|
||||
if existing, ok := nw.namespaces[e.Source()]; ok {
|
||||
existing.Contents = n.Contents
|
||||
} else {
|
||||
nw.namespaces[e.Source()] = n
|
||||
}
|
||||
} else {
|
||||
nw.namespaces[e.Source()] = n
|
||||
}
|
||||
}
|
||||
|
||||
func (nw *NamespaceWatcher) handleError(e *watcherx.ErrorEvent) {
|
||||
nw.logger.
|
||||
WithError(e).
|
||||
Errorf("Received error while watching namespace files at target %s.", nw.target)
|
||||
}
|
||||
|
||||
func watchTarget(ctx context.Context, target string, handler eventHandler, log *logrusx.Logger) error {
|
||||
var (
|
||||
eventCh = make(watcherx.EventChannel)
|
||||
watcher watcherx.Watcher
|
||||
)
|
||||
|
||||
targetUrl, err := urlx.Parse(target)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
info, err := os.Stat(targetUrl.Path) //nolint:gosec // Path is from a validated configuration URL, not user input.
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
watcher, err = watcherx.WatchDirectory(ctx, urlx.GetURLFilePath(targetUrl), eventCh)
|
||||
} else {
|
||||
watcher, err = watcherx.Watch(ctx, targetUrl, eventCh)
|
||||
}
|
||||
// this handles the watcher init error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// trigger initial load
|
||||
done, err := watcher.DispatchNow()
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
initialEventsProcessed := make(chan struct{})
|
||||
go startEventHandler(ctx, eventCh, handler, done, initialEventsProcessed, log)
|
||||
|
||||
// wait for initial load to be done
|
||||
<-initialEventsProcessed
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func startEventHandler(ctx context.Context,
|
||||
eventCh watcherx.EventChannel,
|
||||
handler eventHandler,
|
||||
done <-chan int,
|
||||
initialEventsProcessed chan<- struct{},
|
||||
log *logrusx.Logger) {
|
||||
|
||||
initialDone := false
|
||||
for {
|
||||
select {
|
||||
// because we use an unbuffered chan we can be sure that at least all
|
||||
// initial events are handled
|
||||
case <-done:
|
||||
initialDone = true
|
||||
close(initialEventsProcessed)
|
||||
// Disable this case after it fires once. watcherx closes done when
|
||||
// the watcher shuts down (e.g. directoryWatcher on ctx cancel); a
|
||||
// closed channel is always ready, so leaving done set would re-enter
|
||||
// this branch and close(initialEventsProcessed) a second time (panic).
|
||||
done = nil
|
||||
|
||||
case <-ctx.Done():
|
||||
return
|
||||
|
||||
case e, open := <-eventCh:
|
||||
if !open {
|
||||
return
|
||||
}
|
||||
|
||||
if initialDone {
|
||||
log.
|
||||
WithField("file", e.Source()).
|
||||
WithField("event_type", fmt.Sprintf("%T", e)).
|
||||
Info("A change to a namespace file was detected.")
|
||||
}
|
||||
|
||||
switch e := e.(type) {
|
||||
case *watcherx.RemoveEvent:
|
||||
handler.handleRemove(e)
|
||||
case *watcherx.ChangeEvent:
|
||||
handler.handleChange(e)
|
||||
case *watcherx.ErrorEvent:
|
||||
handler.handleError(e)
|
||||
default:
|
||||
log.Warnf("Ignored unknown event %T", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (nw *NamespaceWatcher) readNamespaceFile(r io.Reader, source string) *NamespaceFile {
|
||||
parse, err := GetParser(source)
|
||||
if err != nil {
|
||||
nw.logger.
|
||||
WithError(err).
|
||||
WithField("file_name", source).
|
||||
Warn("could not infer format from file extension")
|
||||
return nil
|
||||
}
|
||||
|
||||
raw, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
nw.logger.
|
||||
WithError(errors.WithStack(err)).
|
||||
WithField("file_name", source).
|
||||
Error("could not read namespace file")
|
||||
return nil
|
||||
}
|
||||
|
||||
n := namespace.Namespace{}
|
||||
if err := parse(raw, &n); err != nil {
|
||||
nw.logger.
|
||||
WithError(errors.WithStack(err)).
|
||||
WithField("file_name", source).
|
||||
Error("could not parse namespace file")
|
||||
return &NamespaceFile{Name: source, Contents: raw, Parser: parse}
|
||||
}
|
||||
|
||||
return &NamespaceFile{Name: source, Contents: raw, Parser: parse, namespace: &n}
|
||||
}
|
||||
|
||||
func (nw *NamespaceWatcher) GetNamespaceByName(_ context.Context, name string) (*namespace.Namespace, error) {
|
||||
nw.RLock()
|
||||
defer nw.RUnlock()
|
||||
|
||||
for _, nsf := range nw.namespaces {
|
||||
if nsf.namespace != nil && nsf.namespace.Name == name {
|
||||
return nsf.namespace, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, errors.WithStack(herodot.ErrNotFound().WithErrorf(
|
||||
"Unknown namespace with name %s", name))
|
||||
}
|
||||
|
||||
func (nw *NamespaceWatcher) GetNamespaceByConfigID(_ context.Context, id int32) (*namespace.Namespace, error) {
|
||||
nw.RLock()
|
||||
defer nw.RUnlock()
|
||||
|
||||
for _, nspace := range nw.namespaces {
|
||||
//lint:ignore SA1019 backwards compatibility
|
||||
//nolint:staticcheck
|
||||
if nspace.namespace.ID == id {
|
||||
return nspace.namespace, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, errors.WithStack(herodot.ErrNotFound().WithErrorf(
|
||||
"Unknown namespace with ID %d", id))
|
||||
}
|
||||
|
||||
func (nw *NamespaceWatcher) Namespaces(_ context.Context) ([]*namespace.Namespace, error) {
|
||||
nw.RLock()
|
||||
defer nw.RUnlock()
|
||||
|
||||
nspaces := make([]*namespace.Namespace, 0, len(nw.namespaces))
|
||||
for _, nsf := range nw.namespaces {
|
||||
if nsf.namespace != nil {
|
||||
nspaces = append(nspaces, nsf.namespace)
|
||||
}
|
||||
}
|
||||
return nspaces, nil
|
||||
}
|
||||
|
||||
func (nw *NamespaceWatcher) NamespaceFiles() []*NamespaceFile {
|
||||
nw.RLock()
|
||||
defer nw.RUnlock()
|
||||
|
||||
nsfs := make([]*NamespaceFile, 0, len(nw.namespaces))
|
||||
for _, nsf := range nw.namespaces {
|
||||
nsfs = append(nsfs, nsf)
|
||||
}
|
||||
return nsfs
|
||||
}
|
||||
|
||||
func (nw *NamespaceWatcher) ShouldReload(newValue interface{}) bool {
|
||||
v, ok := newValue.(string)
|
||||
if !ok {
|
||||
// the manager type changed
|
||||
return true
|
||||
}
|
||||
// reload if target changed
|
||||
return v != nw.target
|
||||
}
|
||||
|
||||
func GetParser(fn string) (Parser, error) {
|
||||
switch ext := stringsx.SwitchExact(filepath.Ext(fn)); {
|
||||
case ext.AddCase(".yaml"), ext.AddCase(".yml"):
|
||||
return func(b []byte, i interface{}) error {
|
||||
return yaml.Unmarshal(b, i)
|
||||
}, nil
|
||||
case ext.AddCase(".json"):
|
||||
return json.Unmarshal, nil
|
||||
case ext.AddCase(".toml"):
|
||||
return toml.Unmarshal, nil
|
||||
default:
|
||||
return nil, ext.ToUnknownCaseErr()
|
||||
}
|
||||
}
|
||||
250
internal/driver/config/namespace_watcher_test.go
Normal file
250
internal/driver/config/namespace_watcher_test.go
Normal file
@@ -0,0 +1,250 @@
|
||||
// Copyright © 2023 Ory Corp
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ghodss/yaml"
|
||||
"github.com/ory/x/logrusx"
|
||||
"github.com/ory/x/watcherx"
|
||||
"github.com/pelletier/go-toml"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/sirupsen/logrus/hooks/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/ory/keto/internal/namespace"
|
||||
)
|
||||
|
||||
func TestNamespaceProvider(t *testing.T) {
|
||||
setup := func(t *testing.T, target string) (*NamespaceWatcher, *test.Hook) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
hook := &test.Hook{}
|
||||
l := logrusx.New("", "", logrusx.WithHook(hook))
|
||||
|
||||
nw, err := NewNamespaceWatcher(ctx, l, target)
|
||||
require.NoError(t, err)
|
||||
|
||||
return nw, hook
|
||||
}
|
||||
|
||||
writeNamespace := func(t *testing.T, n *namespace.Namespace, fn string) {
|
||||
var marshal func(interface{}) ([]byte, error)
|
||||
switch ext := filepath.Ext(fn); ext {
|
||||
case ".yaml", ".yml":
|
||||
marshal = yaml.Marshal
|
||||
case ".toml":
|
||||
marshal = toml.Marshal
|
||||
case ".json":
|
||||
marshal = json.Marshal
|
||||
default:
|
||||
t.Logf("got unexpected file extension %s", ext)
|
||||
t.FailNow()
|
||||
}
|
||||
nEnc, err := marshal(n)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.WriteFile(fn, nEnc, 0600))
|
||||
}
|
||||
|
||||
writeDir := func(t *testing.T, dir string, files map[string]interface{}) {
|
||||
for fn, fc := range files {
|
||||
fp := filepath.Join(dir, fn)
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(fp), 0700))
|
||||
|
||||
if n, ok := fc.(*namespace.Namespace); ok {
|
||||
writeNamespace(t, n, fp)
|
||||
continue
|
||||
}
|
||||
|
||||
s, ok := fc.(string)
|
||||
if !ok {
|
||||
t.Logf("expected file content to be string or *namespace.Namespace but got %T", fc)
|
||||
t.FailNow()
|
||||
}
|
||||
|
||||
require.NoError(t, os.WriteFile(fp, []byte(s), 0600))
|
||||
}
|
||||
}
|
||||
|
||||
writeJsonNamespace := func(t *testing.T) (string, *namespace.Namespace) {
|
||||
dir := t.TempDir()
|
||||
n := &namespace.Namespace{
|
||||
Name: "test namespace 1",
|
||||
}
|
||||
fn := filepath.Join(dir, "n.json")
|
||||
|
||||
writeNamespace(t, n, fn)
|
||||
|
||||
return fn, n
|
||||
}
|
||||
|
||||
t.Run("case=loads JSON namespace file", func(t *testing.T) {
|
||||
fn, n := writeJsonNamespace(t)
|
||||
|
||||
ws, _ := setup(t, "file://"+fn)
|
||||
|
||||
nspaces, err := ws.Namespaces(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, []*namespace.Namespace{n}, nspaces)
|
||||
})
|
||||
|
||||
t.Run("case=reads namespace files from directory", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
files := map[string]interface{}{
|
||||
"b.yml": &namespace.Namespace{
|
||||
Name: "b",
|
||||
},
|
||||
"a.toml": &namespace.Namespace{
|
||||
Name: "a",
|
||||
},
|
||||
"c.json": &namespace.Namespace{
|
||||
Name: "c",
|
||||
},
|
||||
}
|
||||
|
||||
writeDir(t, dir, files)
|
||||
|
||||
nw, _ := setup(t, "file://"+dir)
|
||||
|
||||
nspaces, err := nw.Namespaces(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, n := range files {
|
||||
assert.Contains(t, nspaces, n.(*namespace.Namespace))
|
||||
}
|
||||
|
||||
nsfs := nw.NamespaceFiles()
|
||||
assert.Equal(t, len(nspaces), len(nsfs))
|
||||
for _, n := range nsfs {
|
||||
assert.NotNil(t, n.Contents)
|
||||
assert.NotNil(t, n.Parser)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("case=ignores but warns about unsupported file extensions", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
n := &namespace.Namespace{
|
||||
Name: "some name",
|
||||
}
|
||||
nJson, err := json.Marshal(n)
|
||||
require.NoError(t, err)
|
||||
|
||||
writeDir(t, dir, map[string]interface{}{
|
||||
"unsupported.file": "foo bar\n",
|
||||
"supported.json": string(nJson),
|
||||
})
|
||||
|
||||
nw, hook := setup(t, "file://"+dir)
|
||||
|
||||
require.Len(t, hook.Entries, 1)
|
||||
|
||||
assert.Equal(t, logrus.WarnLevel, hook.Entries[0].Level, "%+v", hook.Entries[0])
|
||||
assert.True(t, strings.HasSuffix(hook.Entries[0].Data["file_name"].(string), "unsupported.file"))
|
||||
|
||||
namespaces, err := nw.Namespaces(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Len(t, namespaces, 1)
|
||||
assert.Equal(t, n, namespaces[0])
|
||||
})
|
||||
|
||||
t.Run("case=still returns successful namespace if one errors", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
n := &namespace.Namespace{
|
||||
Name: "some name",
|
||||
}
|
||||
nJson, err := json.Marshal(n)
|
||||
require.NoError(t, err)
|
||||
|
||||
writeDir(t, dir, map[string]interface{}{
|
||||
"malformed.yml": "foo bar\n",
|
||||
"correct.json": string(nJson),
|
||||
})
|
||||
|
||||
nw, hook := setup(t, "file://"+dir)
|
||||
|
||||
require.Len(t, hook.Entries, 1)
|
||||
|
||||
assert.Equal(t, logrus.ErrorLevel, hook.Entries[0].Level, "%+v", hook.Entries[0])
|
||||
assert.True(t, strings.HasSuffix(hook.Entries[0].Data["file_name"].(string), "malformed.yml"))
|
||||
|
||||
namespaces, err := nw.Namespaces(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Len(t, namespaces, 1)
|
||||
assert.Equal(t, n, namespaces[0])
|
||||
|
||||
// files are included even if ns is unparsable
|
||||
nsfs := nw.NamespaceFiles()
|
||||
assert.Equal(t, 2, len(nsfs))
|
||||
})
|
||||
|
||||
t.Run("method=should reload", func(t *testing.T) {
|
||||
nw := &NamespaceWatcher{
|
||||
target: "foo",
|
||||
}
|
||||
assert.False(t, nw.ShouldReload("foo"))
|
||||
assert.True(t, nw.ShouldReload("bar"))
|
||||
assert.True(t, nw.ShouldReload([]*namespace.Namespace{}))
|
||||
})
|
||||
}
|
||||
|
||||
type noopEventHandler struct{}
|
||||
|
||||
func (noopEventHandler) handleRemove(*watcherx.RemoveEvent) {}
|
||||
func (noopEventHandler) handleChange(*watcherx.ChangeEvent) {}
|
||||
func (noopEventHandler) handleError(*watcherx.ErrorEvent) {}
|
||||
|
||||
func TestStartEventHandlerDoesNotCloseInitialDoneTwice(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
t.Cleanup(cancel)
|
||||
|
||||
eventCh := make(watcherx.EventChannel)
|
||||
done := make(chan int, 1)
|
||||
initialEventsProcessed := make(chan struct{})
|
||||
handlerReturned := make(chan any, 1)
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
handlerReturned <- recover()
|
||||
}()
|
||||
|
||||
startEventHandler(ctx, eventCh, noopEventHandler{}, done, initialEventsProcessed, logrusx.New("", ""))
|
||||
}()
|
||||
|
||||
done <- 0
|
||||
select {
|
||||
case <-initialEventsProcessed:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("timed out waiting for initial events to be processed")
|
||||
}
|
||||
|
||||
close(done)
|
||||
select {
|
||||
case recovered := <-handlerReturned:
|
||||
require.Nil(t, recovered)
|
||||
t.Fatal("event handler returned before context cancellation")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case recovered := <-handlerReturned:
|
||||
require.Nil(t, recovered)
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("timed out waiting for event handler to stop")
|
||||
}
|
||||
}
|
||||
193
internal/driver/config/opl_cache_test.go
Normal file
193
internal/driver/config/opl_cache_test.go
Normal file
@@ -0,0 +1,193 @@
|
||||
// Copyright © 2023 Ory Corp
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
|
||||
"github.com/hashicorp/go-retryablehttp"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"github.com/ory/herodot"
|
||||
"github.com/ory/x/configx"
|
||||
"github.com/ory/x/fetcher"
|
||||
"github.com/ory/x/logrusx"
|
||||
)
|
||||
|
||||
func TestNewOPLConfigWatcher(t *testing.T) {
|
||||
var hits atomic.Int64
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
if _, err := io.WriteString(w, testOPL); err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(ts.Close)
|
||||
ctx := context.Background()
|
||||
cfg, err := NewDefault(ctx, nil, logrusx.New("", ""), configx.SkipValidation())
|
||||
require.NoError(t, err)
|
||||
cw, err := newOPLConfigWatcher(ctx, cfg.l, cfg.Fetcher, ts.URL)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, 1, hits.Load(), "HTTP request made")
|
||||
_, err = cw.GetNamespaceByName(ctx, "User")
|
||||
require.NoError(t, err)
|
||||
_, err = cw.GetNamespaceByName(ctx, "Document")
|
||||
require.NoError(t, err)
|
||||
|
||||
cache.Wait()
|
||||
|
||||
cw, err = newOPLConfigWatcher(ctx, cfg.l, cfg.Fetcher, ts.URL)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, 1, hits.Load(), "content was cached")
|
||||
_, err = cw.GetNamespaceByName(ctx, "User")
|
||||
require.NoError(t, err)
|
||||
_, err = cw.GetNamespaceByName(ctx, "Document")
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestOPLConfigWatcher(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("caches parsed namespaces", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
ts, hits := newCountingServer(t, http.StatusOK, testOPL)
|
||||
cfg := newTestConfig(t)
|
||||
ctx := t.Context()
|
||||
|
||||
first, err := newOPLConfigWatcher(ctx, cfg.l, cfg.Fetcher, ts.URL)
|
||||
require.NoError(t, err)
|
||||
cache.Wait()
|
||||
|
||||
second, err := newOPLConfigWatcher(ctx, cfg.l, cfg.Fetcher, ts.URL)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, 1, hits.Load(), "content was fetched only once")
|
||||
|
||||
firstUser, err := first.GetNamespaceByName(ctx, "User")
|
||||
require.NoError(t, err)
|
||||
secondUser, err := second.GetNamespaceByName(ctx, "User")
|
||||
require.NoError(t, err)
|
||||
require.Same(t, firstUser, secondUser, "the parsed namespaces are shared, not re-parsed per construction")
|
||||
})
|
||||
|
||||
t.Run("deduplicates concurrent fetches", func(t *testing.T) {
|
||||
const parallel = 5
|
||||
const target = "https://opl.invalid/namespaces.ts"
|
||||
// The cache is process-global and outlives the test binary's runs.
|
||||
cache.Del(target)
|
||||
// The config warms package-level caches whose goroutines live outside
|
||||
// the bubble, so build it here.
|
||||
cfg := newTestConfig(t)
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
var hits atomic.Int64
|
||||
release := make(chan struct{})
|
||||
hc := retryablehttp.NewClient()
|
||||
hc.Logger = nil
|
||||
hc.HTTPClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
hits.Add(1)
|
||||
<-release
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Body: io.NopCloser(strings.NewReader(testOPL)),
|
||||
Header: make(http.Header),
|
||||
Request: r,
|
||||
}, nil
|
||||
})}
|
||||
newFetcher := func() *fetcher.Fetcher { return fetcher.NewFetcher(fetcher.WithClient(hc)) }
|
||||
ctx := t.Context()
|
||||
|
||||
watchers := make([]*oplConfigWatcher, parallel)
|
||||
var eg errgroup.Group
|
||||
for i := range parallel {
|
||||
eg.Go(func() error {
|
||||
cw, err := newOPLConfigWatcher(ctx, cfg.l, newFetcher, target)
|
||||
watchers[i] = cw
|
||||
return err
|
||||
})
|
||||
}
|
||||
synctest.Wait() // One caller is inside the fetch, the rest joined its flight.
|
||||
close(release)
|
||||
|
||||
require.NoError(t, eg.Wait())
|
||||
require.EqualValues(t, 1, hits.Load(), "concurrent constructions share a single fetch")
|
||||
|
||||
for _, cw := range watchers {
|
||||
_, err := cw.GetNamespaceByName(ctx, "User")
|
||||
require.NoError(t, err)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("retries failed fetches", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
ts, hits := newCountingServer(t, http.StatusNotFound, "")
|
||||
cfg := newTestConfig(t)
|
||||
ctx := t.Context()
|
||||
|
||||
_, err := newOPLConfigWatcher(ctx, cfg.l, cfg.Fetcher, ts.URL)
|
||||
require.ErrorContains(t, err, "404")
|
||||
cache.Wait()
|
||||
|
||||
_, err = newOPLConfigWatcher(ctx, cfg.l, cfg.Fetcher, ts.URL)
|
||||
require.ErrorContains(t, err, "404")
|
||||
require.EqualValues(t, 2, hits.Load(), "failed fetches are not cached")
|
||||
})
|
||||
|
||||
t.Run("caches parse failures", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
ts, hits := newCountingServer(t, http.StatusOK, "/* unclosed comment")
|
||||
cfg := newTestConfig(t)
|
||||
ctx := t.Context()
|
||||
|
||||
first, err := newOPLConfigWatcher(ctx, cfg.l, cfg.Fetcher, ts.URL)
|
||||
require.NoError(t, err, "a parse failure does not fail construction")
|
||||
_, err = first.GetNamespaceByName(ctx, "User")
|
||||
require.ErrorIs(t, err, herodot.ErrNotFound())
|
||||
cache.Wait()
|
||||
|
||||
second, err := newOPLConfigWatcher(ctx, cfg.l, cfg.Fetcher, ts.URL)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, 1, hits.Load(), "a parse failure is cached and does not trigger a fetch per construction")
|
||||
_, err = second.GetNamespaceByName(ctx, "User")
|
||||
require.ErrorIs(t, err, herodot.ErrNotFound())
|
||||
})
|
||||
}
|
||||
|
||||
func newTestConfig(t *testing.T) *Config {
|
||||
t.Helper()
|
||||
cfg, err := NewDefault(t.Context(), nil, logrusx.New("", ""), configx.SkipValidation())
|
||||
require.NoError(t, err)
|
||||
return cfg
|
||||
}
|
||||
|
||||
func newCountingServer(t *testing.T, status int, body string) (*httptest.Server, *atomic.Int64) {
|
||||
t.Helper()
|
||||
hits := new(atomic.Int64)
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
w.WriteHeader(status)
|
||||
_, _ = io.WriteString(w, body)
|
||||
}))
|
||||
t.Cleanup(ts.Close)
|
||||
return ts, hits
|
||||
}
|
||||
|
||||
var testOPL = `
|
||||
import { Namespace } from "@ory/keto-namespace-types"
|
||||
|
||||
class User implements Namespace {}
|
||||
class Document implements Namespace {}
|
||||
`
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
181
internal/driver/config/opl_config_namespace_watcher.go
Normal file
181
internal/driver/config/opl_config_namespace_watcher.go
Normal file
@@ -0,0 +1,181 @@
|
||||
// Copyright © 2023 Ory Corp
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/dgraph-io/ristretto/v2"
|
||||
"github.com/ory/x/fetcher"
|
||||
"github.com/ory/x/logrusx"
|
||||
"github.com/ory/x/urlx"
|
||||
"github.com/ory/x/watcherx"
|
||||
"github.com/pkg/errors"
|
||||
"golang.org/x/sync/singleflight"
|
||||
|
||||
"github.com/ory/keto/internal/namespace"
|
||||
"github.com/ory/keto/schema"
|
||||
)
|
||||
|
||||
type (
|
||||
configFiles struct {
|
||||
byPath map[string]io.Reader
|
||||
sync.Mutex
|
||||
}
|
||||
|
||||
oplConfigWatcher struct {
|
||||
logger *logrusx.Logger
|
||||
target string
|
||||
files configFiles
|
||||
|
||||
memoryNamespaceManager
|
||||
}
|
||||
)
|
||||
|
||||
var (
|
||||
_ namespace.Manager = (*oplConfigWatcher)(nil)
|
||||
|
||||
// cache holds the parsed namespaces per remote OPL location. Content at a location is
|
||||
// assumed to be immutable, therefore, a change to the OPL should create a new file location,
|
||||
// therefore a new cache entry.
|
||||
cache, _ = ristretto.NewCache(&ristretto.Config[string, []*namespace.Namespace]{
|
||||
MaxCost: 20_000_000, // 20 MB of estimated heap, each item ca. 10 KB => max 2000 items
|
||||
NumCounters: 20_000, // max 2000 items => 20000 counters
|
||||
BufferItems: 64,
|
||||
Cost: namespace.EstimatedSize,
|
||||
})
|
||||
|
||||
fetchGroup singleflight.Group
|
||||
)
|
||||
|
||||
func newOPLConfigWatcher(ctx context.Context, l *logrusx.Logger, newFetcher func() *fetcher.Fetcher, target string) (*oplConfigWatcher, error) {
|
||||
nw := &oplConfigWatcher{
|
||||
logger: l,
|
||||
target: target,
|
||||
files: configFiles{byPath: make(map[string]io.Reader)},
|
||||
memoryNamespaceManager: *NewMemoryNamespaceManager(),
|
||||
}
|
||||
|
||||
targetUrl, err := urlx.Parse(target)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
switch targetUrl.Scheme {
|
||||
case "file", "":
|
||||
return nw, watchTarget(ctx, target, nw, l)
|
||||
case "base64":
|
||||
file, err := newFetcher().FetchContext(ctx, target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nw.files.Lock()
|
||||
defer nw.files.Unlock()
|
||||
nw.files.byPath[targetUrl.String()] = file
|
||||
nw.parseFiles()
|
||||
return nw, err
|
||||
case "http", "https":
|
||||
if namespaces, ok := cache.Get(target); ok {
|
||||
nw.set(namespaces)
|
||||
return nw, nil
|
||||
}
|
||||
v, err, _ := fetchGroup.Do(target, func() (any, error) {
|
||||
// Detach from the leader's context so that its cancellation does not
|
||||
// fail the concurrent callers sharing this fetch.
|
||||
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
buf, err := newFetcher().FetchContext(ctx, target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b := buf.Bytes()
|
||||
nw.files.Lock()
|
||||
defer nw.files.Unlock()
|
||||
nw.files.byPath[targetUrl.String()] = bytes.NewReader(b)
|
||||
namespaces, ok := nw.parseFiles()
|
||||
if !ok {
|
||||
// Cache a parse failure as an empty result, so that a broken
|
||||
// OPL file does not trigger a fetch per construction.
|
||||
namespaces = []*namespace.Namespace{}
|
||||
}
|
||||
// Cost 0 lets the cache's Cost function compute the entry size.
|
||||
cache.SetWithTTL(target, namespaces, 0, 30*time.Minute)
|
||||
return namespaces, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if namespaces, ok := v.([]*namespace.Namespace); ok {
|
||||
nw.set(namespaces)
|
||||
}
|
||||
return nw, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected url scheme: %q", targetUrl.Scheme)
|
||||
}
|
||||
}
|
||||
|
||||
func (nw *oplConfigWatcher) handleChange(e *watcherx.ChangeEvent) {
|
||||
// the lock is acquired before parsing to ensure that the getters are
|
||||
// waiting for the updated values
|
||||
nw.files.Lock()
|
||||
defer nw.files.Unlock()
|
||||
nw.files.byPath[e.Source()] = e.Reader()
|
||||
nw.parseFiles()
|
||||
}
|
||||
|
||||
func (nw *oplConfigWatcher) handleRemove(e *watcherx.RemoveEvent) {
|
||||
nw.files.Lock()
|
||||
defer nw.files.Unlock()
|
||||
delete(nw.files.byPath, e.Source())
|
||||
nw.parseFiles()
|
||||
}
|
||||
|
||||
func (nw *oplConfigWatcher) handleError(e *watcherx.ErrorEvent) {
|
||||
nw.logger.
|
||||
WithError(e).
|
||||
Errorf("Received error while watching OPL config files at target %s.",
|
||||
nw.target)
|
||||
}
|
||||
|
||||
// parseFiles loops through all files, parsing each and getting the namespaces.
|
||||
// It then sets and returns the namespaces only if there were no errors.
|
||||
//
|
||||
// The caller must hold the lock to nw.files.
|
||||
func (nw *oplConfigWatcher) parseFiles() ([]*namespace.Namespace, bool) {
|
||||
var (
|
||||
namespaces = make([]*namespace.Namespace, 0)
|
||||
errs []error
|
||||
)
|
||||
for _, reader := range nw.files.byPath {
|
||||
content, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
nn, ee := schema.Parse(string(content))
|
||||
for _, e := range ee {
|
||||
errs = append(errs, e)
|
||||
}
|
||||
for _, n := range nn {
|
||||
namespaces = append(namespaces, &n)
|
||||
}
|
||||
}
|
||||
if len(errs) > 0 {
|
||||
for _, err := range errs {
|
||||
nw.logger.
|
||||
WithError(err).
|
||||
Errorf("Failed to parse OPL config files at target %s.",
|
||||
nw.target)
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
nw.set(namespaces)
|
||||
return namespaces, true
|
||||
}
|
||||
397
internal/driver/config/provider.go
Normal file
397
internal/driver/config/provider.go
Normal file
@@ -0,0 +1,397 @@
|
||||
// Copyright © 2023 Ory Corp
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha512"
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/ory/herodot"
|
||||
_ "github.com/ory/jsonschema/v3/httploader"
|
||||
"github.com/ory/x/configx"
|
||||
"github.com/ory/x/fetcher"
|
||||
"github.com/ory/x/httpx"
|
||||
"github.com/ory/x/logrusx"
|
||||
"github.com/ory/x/otelx"
|
||||
"github.com/ory/x/watcherx"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/cors"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/ory/keto/embedx"
|
||||
"github.com/ory/keto/internal/namespace"
|
||||
)
|
||||
|
||||
type EndpointType string
|
||||
|
||||
const (
|
||||
EndpointRead EndpointType = "read"
|
||||
EndpointWrite EndpointType = "write"
|
||||
EndpointMetrics EndpointType = "metrics"
|
||||
EndpointOPLSyntax EndpointType = "opl"
|
||||
|
||||
KeyDSN = "dsn"
|
||||
|
||||
KeyLimitMaxReadDepth = "limit.max_read_depth"
|
||||
KeyLimitMaxReadWidth = "limit.max_read_width"
|
||||
KeyLimitExpandMaxSize = "limit.max_expand_size"
|
||||
|
||||
KeyBatchCheckMaxBatchSize = "limit.max_batch_check_size"
|
||||
KeyBatchCheckParallelizationLimit = "limit.batch_check_max_parallelization"
|
||||
|
||||
KeyReadAPIHost = "serve." + string(EndpointRead) + ".host"
|
||||
KeyReadAPIPort = "serve." + string(EndpointRead) + ".port"
|
||||
KeyReadAPIListenFile = "serve." + string(EndpointRead) + ".write_listen_file"
|
||||
KeyWriteAPIHost = "serve." + string(EndpointWrite) + ".host"
|
||||
KeyWriteAPIPort = "serve." + string(EndpointWrite) + ".port"
|
||||
KeyWriteAPIListenFile = "serve." + string(EndpointWrite) + ".write_listen_file"
|
||||
KeyOPLSyntaxAPIHost = "serve." + string(EndpointOPLSyntax) + ".host"
|
||||
KeyOPLSyntaxAPIPort = "serve." + string(EndpointOPLSyntax) + ".port"
|
||||
KeyOPLSyntaxListenFile = "serve." + string(EndpointOPLSyntax) + ".write_listen_file"
|
||||
KeyMetricsHost = "serve." + string(EndpointMetrics) + ".host"
|
||||
KeyMetricsPort = "serve." + string(EndpointMetrics) + ".port"
|
||||
KeyMetricsListenFile = "serve." + string(EndpointMetrics) + ".write_listen_file"
|
||||
|
||||
KeyNamespaces = "namespaces"
|
||||
KeyNamespacesExperimentalStrictMode = KeyNamespaces + ".experimental_strict_mode"
|
||||
|
||||
KeyFeatureFlagExpandRewrites = "feature_flags.expand_rewrites"
|
||||
KeyFeatureFlagStrictMode = "feature_flags.strict_mode"
|
||||
|
||||
DSNMemory = "sqlite://file::memory:?_fk=true&"
|
||||
|
||||
KeySecretsPagination = "secrets.pagination"
|
||||
)
|
||||
|
||||
type (
|
||||
Config struct {
|
||||
p *configx.Provider
|
||||
l *logrusx.Logger
|
||||
ctx context.Context
|
||||
|
||||
nm namespace.Manager
|
||||
cancelNamespaceManager context.CancelFunc
|
||||
nmLock sync.Mutex
|
||||
}
|
||||
Provider interface {
|
||||
Config(ctx context.Context) *Config
|
||||
}
|
||||
)
|
||||
|
||||
func New(ctx context.Context, l *logrusx.Logger, p *configx.Provider) *Config {
|
||||
return &Config{
|
||||
p: p,
|
||||
l: l,
|
||||
ctx: ctx,
|
||||
}
|
||||
}
|
||||
|
||||
func NewDefault(ctx context.Context, flags *pflag.FlagSet, l *logrusx.Logger, opts ...configx.OptionModifier) (*Config, error) {
|
||||
c := New(ctx, l, nil)
|
||||
cp, err := NewProvider(ctx, flags, c, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.WithSource(cp)
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func NewProvider(ctx context.Context, flags *pflag.FlagSet, config *Config, opts ...configx.OptionModifier) (*configx.Provider, error) {
|
||||
p, err := configx.New(
|
||||
ctx,
|
||||
embedx.ConfigSchema,
|
||||
append(opts,
|
||||
configx.WithFlags(flags),
|
||||
configx.WithStderrValidationReporter(),
|
||||
configx.WithImmutables(KeyDSN, "serve"),
|
||||
configx.OmitKeysFromTracing(KeyDSN),
|
||||
configx.WithLogrusWatcher(config.l),
|
||||
configx.WithContext(ctx),
|
||||
configx.AttachWatcher(config.namespaceWatcher),
|
||||
)...,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (k *Config) Source() *configx.Provider {
|
||||
return k.p
|
||||
}
|
||||
|
||||
func (k *Config) WithSource(p *configx.Provider) {
|
||||
k.p = p
|
||||
k.l.UseConfig(p)
|
||||
}
|
||||
|
||||
func (k *Config) namespaceWatcher(_ watcherx.Event, err error) {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
nm, err := k.NamespaceManager()
|
||||
if err != nil {
|
||||
k.l.WithError(err).Error("got internal error in config namespace watcher: could not get namespace manager")
|
||||
return
|
||||
}
|
||||
|
||||
nnCfg, err := k.namespaceConfig()
|
||||
if err != nil {
|
||||
k.l.WithError(err).Error("could not get namespaces from config")
|
||||
return
|
||||
}
|
||||
if nm.ShouldReload(nnCfg.value()) {
|
||||
k.resetNamespaceManager()
|
||||
}
|
||||
}
|
||||
|
||||
func (k *Config) resetNamespaceManager() {
|
||||
k.nmLock.Lock()
|
||||
defer k.nmLock.Unlock()
|
||||
|
||||
if k.cancelNamespaceManager == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// cancel and delete old namespace manager
|
||||
// the next read request will result in a new one being created
|
||||
k.cancelNamespaceManager()
|
||||
k.nm, k.cancelNamespaceManager = nil, nil
|
||||
}
|
||||
|
||||
func (k *Config) Set(key string, v any) error {
|
||||
if err := k.p.Set(key, v); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if key == KeyNamespaces {
|
||||
k.resetNamespaceManager()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (k *Config) addressFor(endpoint EndpointType) (addr string, listenFile string) {
|
||||
return fmt.Sprintf(
|
||||
"%s:%d",
|
||||
k.p.StringF("serve."+string(endpoint)+".host", ""),
|
||||
k.p.IntF("serve."+string(endpoint)+".port", 0),
|
||||
), k.p.StringF("serve."+string(endpoint)+".write_listen_file", "")
|
||||
}
|
||||
|
||||
func (k *Config) ReadAPIListenOn() (addr string, listenFile string) {
|
||||
return k.addressFor(EndpointRead)
|
||||
}
|
||||
|
||||
func (k *Config) WriteAPIListenOn() (addr string, listenFile string) {
|
||||
return k.addressFor(EndpointWrite)
|
||||
}
|
||||
|
||||
func (k *Config) MetricsListenOn() (addr string, listenFile string) {
|
||||
return k.addressFor(EndpointMetrics)
|
||||
}
|
||||
|
||||
func (k *Config) OPLSyntaxAPIListenOn() (addr string, listenFile string) {
|
||||
return k.addressFor(EndpointOPLSyntax)
|
||||
}
|
||||
|
||||
func (k *Config) MaxReadDepth() int {
|
||||
return k.p.Int(KeyLimitMaxReadDepth)
|
||||
}
|
||||
|
||||
func (k *Config) MaxReadWidth() int {
|
||||
return k.p.Int(KeyLimitMaxReadWidth)
|
||||
}
|
||||
|
||||
func (k *Config) BatchCheckMaxBatchSize() int {
|
||||
return k.p.Int(KeyBatchCheckMaxBatchSize)
|
||||
}
|
||||
|
||||
func (k *Config) BatchCheckParallelizationLimit() int {
|
||||
return k.p.Int(KeyBatchCheckParallelizationLimit)
|
||||
}
|
||||
|
||||
func (k *Config) ExpandMaxTupleCount() int {
|
||||
return k.p.Int(KeyLimitExpandMaxSize)
|
||||
}
|
||||
|
||||
func (k *Config) CORS(iface string) (cors.Options, bool) {
|
||||
switch iface {
|
||||
case "read", "write", "metrics":
|
||||
default:
|
||||
panic("expected interface 'read' or 'write', but got unknown interface " + iface)
|
||||
}
|
||||
|
||||
return k.p.CORS("serve."+iface, cors.Options{
|
||||
AllowedMethods: []string{"GET", "POST", "PUT", "PATCH", "DELETE"},
|
||||
AllowedHeaders: []string{"Authorization", "Content-Type"},
|
||||
ExposedHeaders: []string{"Content-Type"},
|
||||
AllowCredentials: true,
|
||||
})
|
||||
}
|
||||
|
||||
func (k *Config) DSN() string {
|
||||
dsn := k.p.String(KeyDSN)
|
||||
if dsn == "memory" {
|
||||
return DSNMemory
|
||||
}
|
||||
return dsn
|
||||
}
|
||||
|
||||
func (k *Config) Fetcher() *fetcher.Fetcher {
|
||||
// Tracing still works correctly even though we pass a no-op tracer
|
||||
// here, because the otelhttp package will preferentially use the
|
||||
// tracer from the incoming request context over this one.
|
||||
opts := []httpx.ResilientOptions{}
|
||||
if k.p.Bool("clients.http.disallow_private_ip_ranges") {
|
||||
opts = append(opts, httpx.ResilientClientDisallowInternalIPs())
|
||||
}
|
||||
return fetcher.NewFetcher(
|
||||
fetcher.WithClient(httpx.NewResilientClient(opts...)),
|
||||
)
|
||||
}
|
||||
|
||||
func (k *Config) TracingServiceName() string {
|
||||
return k.p.StringF("tracing.service_name", "Ory Keto")
|
||||
}
|
||||
|
||||
func (k *Config) TracingProvider() string {
|
||||
return k.p.StringF("tracing.provider", "")
|
||||
}
|
||||
|
||||
func (k *Config) TracingConfig() *otelx.Config {
|
||||
return k.p.TracingConfig("Ory Keto")
|
||||
}
|
||||
|
||||
func (k *Config) NamespaceManager() (namespace.Manager, error) {
|
||||
k.nmLock.Lock()
|
||||
defer k.nmLock.Unlock()
|
||||
|
||||
if k.nm == nil {
|
||||
var ctx context.Context
|
||||
ctx, k.cancelNamespaceManager = context.WithCancel(k.ctx)
|
||||
|
||||
nnCfg, err := k.namespaceConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
k.nm, err = nnCfg.newManager()(ctx, k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return k.nm, nil
|
||||
}
|
||||
|
||||
func (k *Config) StrictMode() bool {
|
||||
return k.p.Bool(KeyFeatureFlagStrictMode) || k.p.Bool(KeyNamespacesExperimentalStrictMode)
|
||||
}
|
||||
|
||||
func (k *Config) ExpandRewrites() bool {
|
||||
return k.p.Bool(KeyFeatureFlagExpandRewrites)
|
||||
}
|
||||
|
||||
type (
|
||||
buildNamespaceFn func(context.Context, *Config) (namespace.Manager, error)
|
||||
|
||||
namespaceConfig interface {
|
||||
// newManager builds a new namespace manager.
|
||||
newManager() buildNamespaceFn
|
||||
// value returns the wrapped value (for comparing if we should reload)
|
||||
value() any
|
||||
}
|
||||
|
||||
legacyURINamespaceConfig string
|
||||
literalNamespaceConfig []*namespace.Namespace
|
||||
oplNamespaceConfig map[string]any
|
||||
)
|
||||
|
||||
func (uri legacyURINamespaceConfig) newManager() buildNamespaceFn {
|
||||
return func(ctx context.Context, c *Config) (namespace.Manager, error) {
|
||||
return NewNamespaceWatcher(ctx, c.l, string(uri))
|
||||
}
|
||||
}
|
||||
|
||||
func (uri legacyURINamespaceConfig) value() any {
|
||||
return string(uri)
|
||||
}
|
||||
|
||||
func (namespaces literalNamespaceConfig) newManager() buildNamespaceFn {
|
||||
return func(ctx context.Context, _ *Config) (namespace.Manager, error) {
|
||||
return NewMemoryNamespaceManager(namespaces...), nil
|
||||
}
|
||||
}
|
||||
|
||||
func (namespaces literalNamespaceConfig) value() any {
|
||||
return []*namespace.Namespace(namespaces)
|
||||
}
|
||||
|
||||
func (oplConfig oplNamespaceConfig) newManager() buildNamespaceFn {
|
||||
return func(ctx context.Context, c *Config) (namespace.Manager, error) {
|
||||
entry, ok := oplConfig["location"]
|
||||
if !ok {
|
||||
return nil, errors.New("location key not found")
|
||||
}
|
||||
target, ok := entry.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("config value must be string, was %T", entry)
|
||||
}
|
||||
return newOPLConfigWatcher(ctx, c.l, c.Fetcher, target)
|
||||
}
|
||||
}
|
||||
|
||||
func (oplConfig oplNamespaceConfig) value() any {
|
||||
return map[string]any(oplConfig)
|
||||
}
|
||||
|
||||
// namespaceConfig returns a namespace config, which can be either a URI (in
|
||||
// which case we want to watch that URI), or a literal list of namespaces (in
|
||||
// which case we just load them into memory), or a list of URIs referencing OPL
|
||||
// definitions (in which case we want to watch each URI and parse the content).
|
||||
func (k *Config) namespaceConfig() (namespaceConfig, error) {
|
||||
switch nTyped := k.p.GetF(KeyNamespaces, "file://./keto_namespaces").(type) {
|
||||
case string:
|
||||
return legacyURINamespaceConfig(nTyped), nil
|
||||
|
||||
case []*namespace.Namespace:
|
||||
return literalNamespaceConfig(nTyped), nil
|
||||
|
||||
case []any:
|
||||
nEnc, err := json.Marshal(nTyped)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
nn := make([]*namespace.Namespace, len(nTyped))
|
||||
|
||||
if err := json.Unmarshal(nEnc, &nn); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
return literalNamespaceConfig(nn), nil
|
||||
|
||||
case map[string]any:
|
||||
return oplNamespaceConfig(nTyped), nil
|
||||
|
||||
default:
|
||||
return nil, errors.WithStack(herodot.ErrInternalServerError().WithReasonf("could not infer namespaces for type %T", nTyped))
|
||||
}
|
||||
}
|
||||
|
||||
func (k *Config) PaginationEncryptionKeys() [][32]byte {
|
||||
secrets := k.p.Strings(KeySecretsPagination)
|
||||
|
||||
encryptionKeys := make([][32]byte, len(secrets))
|
||||
for i, key := range secrets {
|
||||
encryptionKeys[i] = sha512.Sum512_256([]byte(key))
|
||||
}
|
||||
|
||||
return encryptionKeys
|
||||
}
|
||||
302
internal/driver/config/provider_test.go
Normal file
302
internal/driver/config/provider_test.go
Normal file
@@ -0,0 +1,302 @@
|
||||
// Copyright © 2023 Ory Corp
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gobuffalo/httptest"
|
||||
"github.com/ory/x/configx"
|
||||
"github.com/ory/x/logrusx"
|
||||
"github.com/sirupsen/logrus/hooks/test"
|
||||
"github.com/spf13/pflag"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/ory/keto/embedx"
|
||||
"github.com/ory/keto/internal/namespace"
|
||||
)
|
||||
|
||||
// createFile writes the content to a temporary file, returning the path.
|
||||
// Good for testing config files.
|
||||
func createFile(t *testing.T, content string) (path string) {
|
||||
t.Helper()
|
||||
|
||||
f, err := os.CreateTemp(t.TempDir(), "config-*.yaml")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Cleanup(func() { _ = os.Remove(f.Name()) }) //nolint:gosec // Test file path from os.CreateTemp, not user input.
|
||||
|
||||
n, err := f.WriteString(content)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != len(content) {
|
||||
t.Fatal("failed to write the complete content")
|
||||
}
|
||||
|
||||
return f.Name()
|
||||
}
|
||||
|
||||
// createFileF writes the content format string with the applied args to a
|
||||
// temporary file, returning the path. Good for testing config files.
|
||||
func createFileF(t *testing.T, contentF string, args ...any) (path string) {
|
||||
return createFile(t, fmt.Sprintf(contentF, args...))
|
||||
}
|
||||
|
||||
func setup(t *testing.T, configFile string) (*test.Hook, *Config) {
|
||||
t.Helper()
|
||||
hook := test.Hook{}
|
||||
l := logrusx.New("test", "today", logrusx.WithHook(&hook))
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
|
||||
t.Cleanup(cancel)
|
||||
|
||||
config, err := NewDefault(
|
||||
ctx,
|
||||
pflag.NewFlagSet("test", pflag.ContinueOnError),
|
||||
l,
|
||||
configx.WithConfigFiles(configFile),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
return &hook, config
|
||||
}
|
||||
|
||||
func assertNamespaces(t *testing.T, p *Config, nn ...*namespace.Namespace) {
|
||||
t.Helper()
|
||||
|
||||
nm, err := p.NamespaceManager()
|
||||
require.NoError(t, err)
|
||||
actualNamespaces, err := nm.Namespaces(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, len(nn), len(actualNamespaces))
|
||||
assert.ElementsMatch(t, nn, actualNamespaces)
|
||||
}
|
||||
|
||||
// The new way to configure namespaces is through the Ory Permissions Language.
|
||||
// We check here that we still support enumerating the namespaces directly in
|
||||
// the config or through a file reference, in which case there should be no
|
||||
// rewrites configured.
|
||||
func TestLegacyNamespaceConfig(t *testing.T) {
|
||||
t.Run("case=creates memory namespace manager when namespaces are set", func(t *testing.T) {
|
||||
config := createFile(t, `
|
||||
dsn: memory
|
||||
namespaces:
|
||||
- name: n0
|
||||
- name: n1
|
||||
- name: n2`)
|
||||
|
||||
run := func(namespaces []*namespace.Namespace) func(*testing.T) {
|
||||
return func(t *testing.T) {
|
||||
_, p := setup(t, config)
|
||||
|
||||
assertNamespaces(t, p, namespaces...)
|
||||
|
||||
nm, err := p.NamespaceManager()
|
||||
require.NoError(t, err)
|
||||
_, ok := nm.(*memoryNamespaceManager)
|
||||
assert.True(t, ok)
|
||||
assert.False(t, p.StrictMode())
|
||||
}
|
||||
}
|
||||
|
||||
nn := []*namespace.Namespace{
|
||||
{Name: "n0"},
|
||||
{Name: "n1"},
|
||||
{Name: "n2"},
|
||||
}
|
||||
nnJson, err := json.Marshal(nn)
|
||||
require.NoError(t, err)
|
||||
nnValue := make([]interface{}, 0)
|
||||
require.NoError(t, json.Unmarshal(nnJson, &nnValue))
|
||||
|
||||
t.Run(
|
||||
"type=[]*namespace.Namespace",
|
||||
run(nn),
|
||||
)
|
||||
})
|
||||
|
||||
t.Run("case=reloads namespace manager when namespaces are updated using Set()", func(t *testing.T) {
|
||||
_, p := setup(t, createFile(t, "dsn: memory"))
|
||||
|
||||
n0 := &namespace.Namespace{
|
||||
Name: "n0",
|
||||
}
|
||||
n1 := &namespace.Namespace{
|
||||
Name: "n1",
|
||||
}
|
||||
|
||||
require.NoError(t, p.Set(KeyNamespaces, []*namespace.Namespace{n0}))
|
||||
assertNamespaces(t, p, n0)
|
||||
|
||||
require.NoError(t, p.Set(KeyNamespaces, []*namespace.Namespace{n1}))
|
||||
assertNamespaces(t, p, n1)
|
||||
})
|
||||
|
||||
t.Run("case=creates watcher manager when namespaces is string URL", func(t *testing.T) {
|
||||
_, p := setup(t, createFileF(t, `
|
||||
dsn: memory
|
||||
namespaces: file://%s`,
|
||||
t.TempDir()))
|
||||
|
||||
nm, err := p.NamespaceManager()
|
||||
require.NoError(t, err)
|
||||
_, ok := nm.(*NamespaceWatcher)
|
||||
assert.True(t, ok)
|
||||
assert.False(t, p.StrictMode())
|
||||
})
|
||||
|
||||
t.Run("case=uses passed configx provider", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cp, err := configx.New(ctx, embedx.ConfigSchema, configx.WithValue(KeyDSN, "foobar"))
|
||||
require.NoError(t, err)
|
||||
|
||||
p := New(ctx, logrusx.New("test", "today"), cp)
|
||||
assert.Equal(t, "foobar", p.DSN())
|
||||
assert.Same(t, cp, p.p)
|
||||
})
|
||||
}
|
||||
|
||||
func TestInvalidNamespaceConfig(t *testing.T) {
|
||||
config := createFile(t, `
|
||||
dsn: memory
|
||||
namespaces:
|
||||
- name: n0
|
||||
id: 2147483648
|
||||
`)
|
||||
hook := test.Hook{}
|
||||
l := logrusx.New("test", "today", logrusx.WithHook(&hook))
|
||||
_, err := NewDefault(
|
||||
context.Background(),
|
||||
pflag.NewFlagSet("test", pflag.ContinueOnError),
|
||||
l,
|
||||
configx.WithConfigFiles(config),
|
||||
)
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "must be <= 2.147483647e+09 but found 2147483648")
|
||||
}
|
||||
|
||||
// Test that the namespaces can be configured through the Ory Permission
|
||||
// Language.
|
||||
func TestRewritesNamespaceConfig(t *testing.T) {
|
||||
oplContent := `
|
||||
class User implements Namespace {
|
||||
related: {
|
||||
manager: User[]
|
||||
}
|
||||
}
|
||||
|
||||
class Group implements Namespace {
|
||||
related: {
|
||||
members: (User | Group)[]
|
||||
}
|
||||
}`
|
||||
|
||||
oplConfigFile := createFile(t, oplContent)
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, err := w.Write([]byte(oplContent))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(func() { srv.Close() })
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
location string
|
||||
disallowPrivateIPRanges bool
|
||||
expectErr bool
|
||||
}{{
|
||||
name: "local file",
|
||||
location: "file://" + oplConfigFile,
|
||||
}, {
|
||||
name: "HTTP url forbidden",
|
||||
location: srv.URL,
|
||||
disallowPrivateIPRanges: true,
|
||||
expectErr: true,
|
||||
}, {
|
||||
name: "HTTP url allowed",
|
||||
location: srv.URL,
|
||||
}}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run("case="+tc.name, func(t *testing.T) {
|
||||
config := createFileF(t, `
|
||||
dsn: memory
|
||||
clients:
|
||||
http:
|
||||
disallow_private_ip_ranges: %v
|
||||
namespaces:
|
||||
location: %s`, tc.disallowPrivateIPRanges, tc.location)
|
||||
|
||||
_, p := setup(t, config)
|
||||
nm, err := p.NamespaceManager()
|
||||
if tc.expectErr {
|
||||
assert.Error(t, err)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
namespaces, err := nm.Namespaces(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Len(t, namespaces, 2)
|
||||
|
||||
names, relationNames := []string{namespaces[0].Name, namespaces[1].Name}, []string{namespaces[0].Relations[0].Name, namespaces[1].Relations[0].Name}
|
||||
|
||||
assert.False(t, p.StrictMode())
|
||||
assert.ElementsMatch(t, names, []string{"User", "Group"})
|
||||
assert.ElementsMatch(t, relationNames, []string{"manager", "members"})
|
||||
})
|
||||
}
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
deprecated bool
|
||||
new bool
|
||||
want bool
|
||||
}{
|
||||
{name: "neither key set", deprecated: false, new: false, want: false},
|
||||
{name: "deprecated key only", deprecated: true, new: false, want: true},
|
||||
{name: "new key only", deprecated: false, new: true, want: true},
|
||||
{name: "both keys true", deprecated: true, new: true, want: true},
|
||||
} {
|
||||
t.Run("case=strict_mode/"+tt.name, func(t *testing.T) {
|
||||
cfg := createFileF(t, `
|
||||
dsn: memory
|
||||
namespaces:
|
||||
location: file://%s
|
||||
experimental_strict_mode: %v
|
||||
feature_flags:
|
||||
strict_mode: %v`, oplConfigFile, tt.deprecated, tt.new)
|
||||
|
||||
_, p := setup(t, cfg)
|
||||
assert.Equal(t, tt.want, p.StrictMode())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvider_DefaultReadAPIListenOn(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
config, err := NewDefault(
|
||||
ctx,
|
||||
pflag.NewFlagSet("test", pflag.ContinueOnError),
|
||||
logrusx.New("", ""),
|
||||
configx.WithValue("dsn", "foo"),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
addr, listenFile := config.ReadAPIListenOn()
|
||||
assert.Equal(t, ":4466", addr)
|
||||
assert.Zero(t, listenFile)
|
||||
}
|
||||
Reference in New Issue
Block a user