snapshot of https://github.com/ory/kratos.git (master, 2026-09-15; blobs > 131072B omitted)

oryx/fetcher/fetcher.go

@@ -0,0 +1,187 @@
+// Copyright © 2023 Ory Corp
+// SPDX-License-Identifier: Apache-2.0
+
+package fetcher
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	stderrors "errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/dgraph-io/ristretto/v2"
+	"github.com/hashicorp/go-retryablehttp"
+	"github.com/pkg/errors"
+
+	"github.com/ory/x/httpx"
+)
+
+// Fetcher is able to load file contents from http, https, file, and base64 locations.
+type Fetcher struct {
+	hc      *retryablehttp.Client
+	limit   int64
+	cache   *ristretto.Cache[[]byte, []byte]
+	ttl     time.Duration
+	schemes []string
+}
+
+type opts struct {
+	hc      *retryablehttp.Client
+	limit   int64
+	cache   *ristretto.Cache[[]byte, []byte]
+	ttl     time.Duration
+	schemes []string
+}
+
+var ErrUnknownScheme = stderrors.New("unknown scheme")
+
+// WithClient sets the http.Client the fetcher uses.
+func WithClient(hc *retryablehttp.Client) Modifier {
+	return func(o *opts) {
+		o.hc = hc
+	}
+}
+
+// WithMaxHTTPMaxBytes reads at most limit bytes from the HTTP response body,
+// returning bytes.ErrToLarge if the limit would be exceeded.
+func WithMaxHTTPMaxBytes(limit int64) Modifier {
+	return func(o *opts) {
+		o.limit = limit
+	}
+}
+
+func WithCache(cache *ristretto.Cache[[]byte, []byte], ttl time.Duration) Modifier {
+	return func(o *opts) {
+		if ttl < 0 {
+			return
+		}
+		o.cache = cache
+		o.ttl = ttl
+	}
+}
+
+func WithAllowedSchemes(schemes ...string) Modifier {
+	return func(o *opts) {
+		o.schemes = slices.Clone(schemes)
+	}
+}
+
+func newOpts() *opts {
+	return &opts{
+		hc:      httpx.NewResilientClient(),
+		schemes: []string{"http", "https", "file", "base64"},
+	}
+}
+
+type Modifier func(*opts)
+
+// redactedSource returns a safe representation of the source for use in error
+// messages. base64:// sources embed the full payload (which may contain
+// secrets such as private keys) and are therefore redacted; all other sources
+// are returned unchanged.
+func redactedSource(source string) string {
+	if strings.HasPrefix(source, "base64://") {
+		return "base64://[redacted]"
+	}
+	return source
+}
+
+// NewFetcher creates a new fetcher instance.
+func NewFetcher(opts ...Modifier) *Fetcher {
+	o := newOpts()
+	for _, f := range opts {
+		f(o)
+	}
+	return &Fetcher{hc: o.hc, limit: o.limit, cache: o.cache, ttl: o.ttl, schemes: o.schemes}
+}
+
+// FetchContext fetches the file contents from the source and allows to pass a
+// context that is used for HTTP requests.
+func (f *Fetcher) FetchContext(ctx context.Context, source string) (*bytes.Buffer, error) {
+	b, err := f.FetchBytes(ctx, source)
+	if err != nil {
+		return nil, err
+	}
+	return bytes.NewBuffer(b), nil
+}
+
+// FetchBytes fetches the file contents from the source and allows to pass a
+// context that is used for HTTP requests.
+func (f *Fetcher) FetchBytes(ctx context.Context, source string) ([]byte, error) {
+	if !slices.ContainsFunc(f.schemes, func(scheme string) bool {
+		return strings.HasPrefix(source, scheme+"://")
+	}) {
+		return nil, errors.WithStack(fmt.Errorf("%w: in source %q: allowed schemes: %s", ErrUnknownScheme, redactedSource(source), strings.Join(f.schemes, ", ")))
+	}
+	switch {
+	case strings.HasPrefix(source, "http://"), strings.HasPrefix(source, "https://"):
+		return f.fetchRemote(ctx, source)
+	case strings.HasPrefix(source, "file://"):
+		b, err := os.ReadFile(strings.TrimPrefix(source, "file://"))
+		if err != nil {
+			return nil, errors.Wrapf(err, "read file: %s", redactedSource(source))
+		}
+		return b, nil
+	case strings.HasPrefix(source, "base64://"):
+		src, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(source, "base64://"))
+		if err != nil {
+			return nil, errors.Wrapf(err, "base64decode: %s", redactedSource(source))
+		}
+		return src, nil
+	default:
+		return nil, errors.Wrap(ErrUnknownScheme, "unknown scheme in source: "+redactedSource(source))
+	}
+}
+
+func (f *Fetcher) fetchRemote(ctx context.Context, source string) (b []byte, err error) {
+	if f.cache != nil {
+		cacheKey := sha256.Sum256([]byte(source))
+		if v, ok := f.cache.Get(cacheKey[:]); ok {
+			b = make([]byte, len(v))
+			copy(b, v)
+			return b, nil
+		}
+		defer func() {
+			if err == nil && len(b) > 0 {
+				toCache := make([]byte, len(b))
+				copy(toCache, b)
+				f.cache.SetWithTTL(cacheKey[:], toCache, int64(len(toCache)), f.ttl)
+			}
+		}()
+	}
+
+	req, err := retryablehttp.NewRequestWithContext(ctx, http.MethodGet, source, nil)
+	if err != nil {
+		return nil, errors.Wrapf(err, "new request: %s", redactedSource(source))
+	}
+	res, err := f.hc.Do(req)
+	if err != nil {
+		return nil, errors.Wrap(err, redactedSource(source))
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return nil, errors.Errorf("expected http response status code 200 but got %d when fetching: %s", res.StatusCode, redactedSource(source))
+	}
+
+	if f.limit > 0 {
+		var buf bytes.Buffer
+		n, err := io.Copy(&buf, io.LimitReader(res.Body, f.limit+1))
+		if n > f.limit {
+			return nil, bytes.ErrTooLarge
+		}
+		if err != nil {
+			return nil, err
+		}
+		return buf.Bytes(), nil
+	}
+	return io.ReadAll(res.Body)
+}