snapshot of https://github.com/ory/keto.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
CI / Run lints and checks (push) Failing after 47s
CI / Run tests (push) Failing after 1s
CI / Run race tests (push) Failing after 5s
CI / Build CLI docs (push) Has been skipped
CI / Run buf actions (push) Failing after 5s
CodeQL / Analyze (go) (push) Failing after 3s
CodeQL / Analyze (javascript) (push) Failing after 2s
Docker Image Scanners / scanners (push) Failing after 3s
Format / format (push) Failing after 3s
Synchronize Issue Labels / Synchronize Issue Labels (push) Has been skipped
CI / Generate release (push) Has been skipped
CI / Draft newsletter (push) Has been skipped
CI / Pending approval Slack notification (push) Has been skipped
CI / Send newsletter (push) Has been skipped

This commit is contained in:
upstream-mirror
2026-09-16 01:08:34 +05:00
commit 9456a4e922
2560 changed files with 120911 additions and 0 deletions

View File

@@ -0,0 +1,78 @@
// Copyright © 2023 Ory Corp
// SPDX-License-Identifier: Apache-2.0
package watcherx
import (
"context"
"fmt"
"net/url"
"github.com/pkg/errors"
)
type (
errSchemeUnknown struct {
scheme string
}
EventChannel chan Event
Watcher interface {
// DispatchNow fires the watcher and causes an event.
//
// WARNING: The returned channel must be read or no further events will
// be propagated due to a deadlock.
DispatchNow() (<-chan int, error)
}
dispatcher struct {
ctx context.Context
trigger chan struct{}
done chan int
}
)
var (
// ErrSchemeUnknown is just for checking with errors.Is()
ErrSchemeUnknown = &errSchemeUnknown{}
ErrWatcherNotRunning = fmt.Errorf("watcher is not running")
)
func (e *errSchemeUnknown) Is(other error) bool {
_, ok := other.(*errSchemeUnknown)
return ok
}
func (e *errSchemeUnknown) Error() string {
return fmt.Sprintf("unknown scheme '%s' to watch", e.scheme)
}
func newDispatcher(ctx context.Context) *dispatcher {
return &dispatcher{
ctx: ctx,
trigger: make(chan struct{}),
done: make(chan int),
}
}
func (d *dispatcher) DispatchNow() (<-chan int, error) {
if d.trigger == nil {
return nil, ErrWatcherNotRunning
}
// The trigger send must respect cancellation. Once the watcher's context is
// done its receiver goroutine has returned, so a bare send would block
// forever and wedge the caller.
select {
case d.trigger <- struct{}{}:
return d.done, nil
case <-d.ctx.Done():
return nil, errors.WithStack(ErrWatcherNotRunning)
}
}
func Watch(ctx context.Context, u *url.URL, c EventChannel) (Watcher, error) {
switch u.Scheme {
// see urlx.Parse for why the empty string is also file
case "file", "":
return WatchFile(ctx, u.Path, c)
}
return nil, &errSchemeUnknown{u.Scheme}
}

173
oryx/watcherx/directory.go Normal file
View File

@@ -0,0 +1,173 @@
// Copyright © 2023 Ory Corp
// SPDX-License-Identifier: Apache-2.0
package watcherx
import (
"context"
"os"
"path/filepath"
"sync"
"github.com/fsnotify/fsnotify"
"github.com/pkg/errors"
)
func WatchDirectory(ctx context.Context, dir string, c EventChannel) (Watcher, error) {
w, err := fsnotify.NewWatcher()
if err != nil {
return nil, errors.WithStack(err)
}
subDirs := make(map[string]struct{})
if err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return errors.WithStack(err)
}
if info.IsDir() {
if err := w.Add(path); err != nil {
return errors.WithStack(err)
}
subDirs[path] = struct{}{}
}
return nil
}); err != nil {
return nil, err
}
dw := &directoryWatcher{
dispatcher: newDispatcher(ctx),
c: c,
dir: dir,
subDirs: subDirs,
w: w,
}
go dw.streamDirectoryEvents(ctx)
return dw, nil
}
type directoryWatcher struct {
*dispatcher
c EventChannel
dir string
// subDirsMtx guards subDirs, which is read by the watcher goroutine and
// potentially by external callers (e.g. tests) concurrently.
subDirsMtx sync.RWMutex
subDirs map[string]struct{}
w *fsnotify.Watcher
}
func (w *directoryWatcher) handleEvent(ctx context.Context, e fsnotify.Event) {
if e.Has(fsnotify.Remove) {
w.subDirsMtx.Lock()
_, isSubDir := w.subDirs[e.Name]
if isSubDir {
// we do not want any event on deletion of a directory
delete(w.subDirs, e.Name)
}
w.subDirsMtx.Unlock()
if isSubDir {
return
}
w.maybeSend(ctx, &RemoveEvent{
source: source(e.Name),
})
return
} else if e.Has(fsnotify.Write | fsnotify.Create) {
if stats, err := os.Stat(e.Name); err != nil {
w.maybeSend(ctx, &ErrorEvent{
error: errors.WithStack(err),
source: source(e.Name),
})
return
} else if stats.IsDir() {
if err := w.w.Add(e.Name); err != nil {
w.maybeSend(ctx, &ErrorEvent{
error: errors.WithStack(err),
source: source(e.Name),
})
}
w.subDirsMtx.Lock()
w.subDirs[e.Name] = struct{}{}
w.subDirsMtx.Unlock()
return
}
//#nosec G304 -- false positive
data, err := os.ReadFile(e.Name)
if err != nil {
w.maybeSend(ctx, &ErrorEvent{
error: err,
source: source(e.Name),
})
} else {
w.maybeSend(ctx, &ChangeEvent{
data: data,
source: source(e.Name),
})
}
}
}
func (w *directoryWatcher) maybeSend(ctx context.Context, e Event) bool {
select {
case <-ctx.Done():
return false
case w.c <- e:
return true
}
}
func (w *directoryWatcher) streamDirectoryEvents(ctx context.Context) {
defer func() {
close(w.done)
close(w.c)
_ = w.w.Close()
}()
for {
select {
case <-ctx.Done():
return
case e := <-w.w.Events:
w.handleEvent(ctx, e)
case <-w.trigger:
var eventsSent int
if err := filepath.Walk(w.dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
//#nosec G304 -- false positive
data, err := os.ReadFile(path)
if err != nil {
if !w.maybeSend(ctx, &ErrorEvent{
error: err,
source: source(path),
}) {
return errors.WithStack(context.Canceled)
}
} else {
if !w.maybeSend(ctx, &ChangeEvent{
data: data,
source: source(path),
}) {
return errors.WithStack(context.Canceled)
}
}
eventsSent++
}
return nil
}); err != nil {
if !w.maybeSend(ctx, &ErrorEvent{
error: err,
source: source(w.dir),
}) {
return
}
eventsSent++
}
w.done <- eventsSent
}
}
}

109
oryx/watcherx/event.go Normal file
View File

@@ -0,0 +1,109 @@
// Copyright © 2023 Ory Corp
// SPDX-License-Identifier: Apache-2.0
package watcherx
import (
"bytes"
"encoding/json"
"fmt"
"io"
)
type (
Event interface {
// MarshalJSON is required to work multiple times
json.Marshaler
Reader() io.Reader
Source() string
String() string
setSource(string)
}
source string
ErrorEvent struct {
error
source
}
ChangeEvent struct {
data []byte
source
}
RemoveEvent struct {
source
}
serialEventType string
serialEvent struct {
Type serialEventType `json:"type"`
Data []byte `json:"data"`
Source source `json:"source"`
}
)
func NewErrorEvent(err error, source_ string) *ErrorEvent {
return &ErrorEvent{
error: err,
source: source(source_),
}
}
const (
serialTypeChange serialEventType = "change"
serialTypeRemove serialEventType = "remove"
serialTypeError serialEventType = "error"
)
func (e *ErrorEvent) Reader() io.Reader {
return bytes.NewBufferString(e.Error())
}
func (e *ErrorEvent) MarshalJSON() ([]byte, error) {
return json.Marshal(serialEvent{
Type: serialTypeError,
Data: []byte(e.Error()),
Source: e.source,
})
}
func (e *ErrorEvent) String() string {
return fmt.Sprintf("error: %+v; source: %s", e.error, e.source)
}
func (e source) Source() string {
return string(e)
}
func (e *source) setSource(nsrc string) {
*e = source(nsrc)
}
func (e *ChangeEvent) Reader() io.Reader {
return bytes.NewBuffer(e.data)
}
func (e *ChangeEvent) MarshalJSON() ([]byte, error) {
return json.Marshal(serialEvent{
Type: serialTypeChange,
Data: e.data,
Source: e.source,
})
}
func (e *ChangeEvent) String() string {
return fmt.Sprintf("data: %s; source: %s", e.data, e.source)
}
func (e *RemoveEvent) Reader() io.Reader {
return nil
}
func (e *RemoveEvent) MarshalJSON() ([]byte, error) {
return json.Marshal(serialEvent{
Type: serialTypeRemove,
Source: e.source,
})
}
func (e *RemoveEvent) String() string {
return fmt.Sprintf("removed source: %s", e.source)
}

174
oryx/watcherx/file.go Normal file
View File

@@ -0,0 +1,174 @@
// Copyright © 2023 Ory Corp
// SPDX-License-Identifier: Apache-2.0
package watcherx
import (
"context"
"os"
"path/filepath"
"github.com/fsnotify/fsnotify"
"github.com/pkg/errors"
)
// WatchFile spawns a background goroutine to watch file, reporting any changes
// to c. Watching stops when ctx is canceled.
func WatchFile(ctx context.Context, file string, c EventChannel) (Watcher, error) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
return nil, errors.WithStack(err)
}
dir := filepath.Dir(file)
if err := watcher.Add(dir); err != nil {
return nil, errors.WithStack(err)
}
resolvedFile, err := filepath.EvalSymlinks(file)
if err != nil {
if pathError := new(os.PathError); !errors.As(err, &pathError) {
return nil, errors.WithStack(err)
}
// The file does not exist. The watcher should still watch the directory
// to get notified about file creation.
resolvedFile = ""
} else if resolvedFile != file {
// If `resolvedFile` != `file` then `file` is a symlink and we have to explicitly watch the referenced file.
// This is because fsnotify follows symlinks and watches the destination file, not the symlink
// itself. That is at least the case for unix systems. See: https://github.com/fsnotify/fsnotify/issues/199
if err := watcher.Add(file); err != nil {
return nil, errors.WithStack(err)
}
}
d := newDispatcher(ctx)
go streamFileEvents(ctx, watcher, c, d.trigger, d.done, file, resolvedFile)
return d, nil
}
// streamFileEvents watches for file changes and supports symlinks which requires several workarounds due to limitations of fsnotify.
// Argument `resolvedFile` is the resolved symlink path of the file, or it is the watchedFile name itself. If `resolvedFile` is empty, then the watchedFile does not exist.
func streamFileEvents(ctx context.Context, watcher *fsnotify.Watcher, c EventChannel, sendNow <-chan struct{}, sendNowDone chan<- int, watchedFile, resolvedFile string) {
eventSource := source(watchedFile)
removeDirectFileWatcher := func() {
_ = watcher.Remove(watchedFile)
}
addDirectFileWatcher := func() {
// check if the watchedFile (symlink) exists
// if it does not the dir watcher will notify us when it gets created
if _, err := os.Lstat(watchedFile); err == nil {
if err := watcher.Add(watchedFile); err != nil {
c <- &ErrorEvent{
error: errors.WithStack(err),
source: eventSource,
}
}
}
}
defer watcher.Close()
for {
select {
case <-ctx.Done():
return
case <-sendNow:
if resolvedFile == "" {
// The file does not exist. Announce this by sending a RemoveEvent.
c <- &RemoveEvent{eventSource}
} else {
// The file does exist. Announce the current content by sending a ChangeEvent.
// #nosec G304 -- false positive
data, err := os.ReadFile(watchedFile)
if err != nil {
select {
case c <- &ErrorEvent{
error: errors.WithStack(err),
source: eventSource,
}:
case <-ctx.Done():
return
}
continue
}
select {
case c <- &ChangeEvent{
data: data,
source: eventSource,
}:
case <-ctx.Done():
return
}
}
// in any of the above cases we send exactly one event
select {
case sendNowDone <- 1:
case <-ctx.Done():
return
}
case e, ok := <-watcher.Events:
if !ok {
return
}
// filter events to only watch watchedFile
// e.Name contains the name of the watchedFile (regardless whether it is a symlink), not the resolved file name
if filepath.Clean(e.Name) == watchedFile {
recentlyResolvedFile, err := filepath.EvalSymlinks(watchedFile)
// when there is no error the file exists and any symlinks can be resolved
if err != nil {
// check if the watchedFile (or the file behind the symlink) was removed
if _, ok := err.(*os.PathError); ok {
select {
case c <- &RemoveEvent{eventSource}:
case <-ctx.Done():
return
}
removeDirectFileWatcher()
continue
}
select {
case c <- &ErrorEvent{
error: errors.WithStack(err),
source: eventSource,
}:
case <-ctx.Done():
return
}
continue
}
// This catches following three cases:
// 1. the watchedFile was written or created
// 2. the watchedFile is a symlink and has changed (k8s config map updates)
// 3. the watchedFile behind the symlink was written or created
switch {
case recentlyResolvedFile != resolvedFile:
resolvedFile = recentlyResolvedFile
// watch the symlink again to update the actually watched file
removeDirectFileWatcher()
addDirectFileWatcher()
// we fallthrough because we also want to read the file in this case
fallthrough
case e.Has(fsnotify.Write | fsnotify.Create):
// #nosec G304 -- false positive
data, err := os.ReadFile(watchedFile)
if err != nil {
select {
case c <- &ErrorEvent{
error: errors.WithStack(err),
source: eventSource,
}:
case <-ctx.Done():
return
}
continue
}
select {
case c <- &ChangeEvent{
data: data,
source: eventSource,
}:
case <-ctx.Done():
return
}
}
}
}
}
}

View File

@@ -0,0 +1,7 @@
event_logger.yml
configmap.yml
Makefile
README.md
eventlog_snapshot

View File

@@ -0,0 +1 @@
tmp_snapshot

View File

@@ -0,0 +1,21 @@
FROM golang:1.26-alpine3.23 AS builder
RUN apk -U --no-cache add build-base
WORKDIR /go/src/github.com/ory/x
ADD go.mod go.mod
ADD go.sum go.sum
RUN go mod download
ADD . .
RUN go build -o /usr/bin/eventlogger ./watcherx/integrationtest
FROM alpine:3.22
COPY --from=builder /usr/bin/eventlogger /usr/bin/eventlogger
ENTRYPOINT ["eventlogger"]
CMD ["/etc/config/mock-config"]

View File

@@ -0,0 +1,65 @@
SHELL=/bin/bash -euo pipefail
CLUSTER_NAME=watcherx-integration-test
SNAPSHOT_FILE=eventlog_snapshot
define generate_snapshot
sleep 5
make update
sleep 1
kubectl logs eventlogger --context kind-${CLUSTER_NAME} >> $(1)
make apply
sleep 1
kubectl logs eventlogger --context kind-${CLUSTER_NAME} >> $(1)
make update
sleep 1
kubectl logs eventlogger --context kind-${CLUSTER_NAME} >> $(1)
endef
.PHONY: build
build:
docker build -f Dockerfile -t eventlogger:latest ../..
.PHONY: create
create:
kind create cluster --name ${CLUSTER_NAME} --wait 1m || true
.PHONY: load
load:
kind load docker-image eventlogger:latest --name ${CLUSTER_NAME}
.PHONY: apply
apply:
kubectl apply -f configmap.yml -f event_logger.yml --context kind-${CLUSTER_NAME}
.PHONY: delete
delete:
kind delete cluster --name ${CLUSTER_NAME}
.PHONY: setup
setup: build create load apply
.PHONY: snapshot
snapshot: setup container-restart
rm ${SNAPSHOT_FILE}
${call generate_snapshot,$(SNAPSHOT_FILE)}
.PHONY: check
check: setup container-restart
rm tmp_snapshot || true
${call generate_snapshot,tmp_snapshot}
diff tmp_snapshot ${SNAPSHOT_FILE}
.PHONY: logs
logs:
kubectl logs eventlogger --context kind-${CLUSTER_NAME}
.PHONY: container-restart
container-restart:
kubectl delete -f event_logger.yml --context kind-${CLUSTER_NAME}
kubectl apply -f event_logger.yml --context kind-${CLUSTER_NAME}
.PHONY: update
update:
cat configmap.yml | sed 's/somevalue/othervalue/' | kubectl apply -f - --context kind-${CLUSTER_NAME}
cat event_logger.yml | sed 's/somevalue/othervalue/' | kubectl apply -f - --context kind-${CLUSTER_NAME}

View File

@@ -0,0 +1,27 @@
# Integration Test for watcherx/FileWatcher
As kubernetes has a special way to change mounted config map values we want to
make sure our file watcher is compatible with that.
## Perquisites
The versions are the ones that definitely work.
- kind (v0.8.1)
- kubectl (v1.18.5)
- docker (v19.03.12-ce)
- make (v4.3)
## Structure
The `main.go` just logs all events it gets. It is deployed to a kind kubernetes
cluster together with a configmap that gets updated during the test. For details
on the test steps have a look at the `Makefile`.
## Running
To generate the log snapshot run `make snapshot`. That snapshot should be
committed. To check if the FileWatcher works run `make check`. For debugging
purposes single steps of the setup have descriptive make target names and can be
run separately. It is safe to delete the cluster at any point or rerun snapshot
generation.

View File

@@ -0,0 +1,6 @@
kind: ConfigMap
apiVersion: v1
metadata:
name: changing-config
data:
mock-config: somevalue

View File

@@ -0,0 +1,19 @@
kind: Pod
apiVersion: v1
metadata:
name: eventlogger
annotations:
variant: somevalue
spec:
containers:
- name: eventlogger
image: eventlogger:latest
imagePullPolicy: Never
volumeMounts:
- name: changing-config
mountPath: /etc/config
restartPolicy: Never
volumes:
- name: changing-config
configMap:
name: changing-config

View File

@@ -0,0 +1,21 @@
watching file /etc/config/mock-config
got change event:
Data: othervalue,
Src: /etc/config/mock-config
watching file /etc/config/mock-config
got change event:
Data: othervalue,
Src: /etc/config/mock-config
got change event:
Data: somevalue,
Src: /etc/config/mock-config
watching file /etc/config/mock-config
got change event:
Data: othervalue,
Src: /etc/config/mock-config
got change event:
Data: somevalue,
Src: /etc/config/mock-config
got change event:
Data: othervalue,
Src: /etc/config/mock-config

View File

@@ -0,0 +1,45 @@
// Copyright © 2023 Ory Corp
// SPDX-License-Identifier: Apache-2.0
package main
import (
"context"
"fmt"
"io"
"os"
"github.com/ory/x/watcherx"
)
func main() {
if len(os.Args) != 2 {
_, _ = fmt.Fprintf(os.Stderr, "expected 1 comand line argument but got %d\n", len(os.Args)-1)
os.Exit(1)
}
c := make(chan watcherx.Event)
_, err := watcherx.WatchFile(context.Background(), os.Args[1], c)
if err != nil {
_, _ = fmt.Fprintf(os.Stderr, "could not initialize file watcher: %+v\n", err)
os.Exit(1)
}
fmt.Printf("watching file %s\n", os.Args[1])
for {
switch e := (<-c).(type) {
case *watcherx.ChangeEvent:
var data []byte
data, err = io.ReadAll(e.Reader())
if err != nil {
_, _ = fmt.Fprintf(os.Stderr, "could not read data: %+v\n", err)
os.Exit(1)
}
fmt.Printf("got change event:\nData: %s,\nSrc: %s\n", data, e.Source())
case *watcherx.RemoveEvent:
fmt.Printf("got remove event:\nSrc: %s\n", e.Source())
case *watcherx.ErrorEvent:
fmt.Printf("got error event:\nError: %s\n", e.Error())
default:
fmt.Println("got unknown event")
}
}
}

View File

@@ -0,0 +1,80 @@
// Copyright © 2023 Ory Corp
// SPDX-License-Identifier: Apache-2.0
package watcherx
import (
"os"
"path"
"path/filepath"
"runtime"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func KubernetesAtomicWrite(t *testing.T, dir, fileName, content string) {
// atomic write according to https://github.com/kubernetes/kubernetes/blob/master/pkg/volume/util/atomic_writer.go
const (
dataDirName = "..data"
newDataDirName = "..data_tmp"
)
// (2)
dataDirPath := filepath.Join(dir, dataDirName)
oldTsDir, err := os.Readlink(dataDirPath)
if err != nil {
require.True(t, os.IsNotExist(err), "%+v", err)
// although Readlink() returns "" on err, don't be fragile by relying on it (since it's not specified in docs)
// empty oldTsDir indicates that it didn't exist
oldTsDir = ""
}
oldTsPath := filepath.Join(dir, oldTsDir)
// (3) we are not interested in the case where a file gets deleted as we just operate on one file
// (4) we assume the file needs an update
// (5)
tsDir, err := os.MkdirTemp(dir, time.Now().UTC().Format("..2006_01_02_15_04_05."))
require.NoError(t, err)
tsDirName := filepath.Base(tsDir)
// (6)
require.NoError(
t,
os.WriteFile(path.Join(tsDir, fileName), []byte(content), 0600),
)
// (7)
_, err = os.Readlink(filepath.Join(dir, fileName))
if err != nil && os.IsNotExist(err) {
// The link into the data directory for this path doesn't exist; create it
require.NoError(
t,
os.Symlink(filepath.Join(dataDirName, fileName), filepath.Join(dir, fileName)),
)
}
// (8)
newDataDirPath := filepath.Join(dir, newDataDirName)
require.NoError(
t,
os.Symlink(tsDirName, newDataDirPath),
)
// (9)
if runtime.GOOS == "windows" {
require.NoError(t, os.Remove(dataDirPath))
require.NoError(t, os.Symlink(tsDirName, dataDirPath))
require.NoError(t, os.Remove(newDataDirPath))
} else {
require.NoError(t, os.Rename(newDataDirPath, dataDirPath))
}
// (10) in our case there is nothing to remove
// (11)
if len(oldTsDir) > 0 {
require.NoError(t, os.RemoveAll(oldTsPath))
}
}