snapshot of https://github.com/ory/keto.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
CI / Run lints and checks (push) Failing after 47s
CI / Run tests (push) Failing after 1s
CI / Run race tests (push) Failing after 5s
CI / Build CLI docs (push) Has been skipped
CI / Run buf actions (push) Failing after 5s
CodeQL / Analyze (go) (push) Failing after 3s
CodeQL / Analyze (javascript) (push) Failing after 2s
Docker Image Scanners / scanners (push) Failing after 3s
Format / format (push) Failing after 3s
Synchronize Issue Labels / Synchronize Issue Labels (push) Has been skipped
CI / Generate release (push) Has been skipped
CI / Draft newsletter (push) Has been skipped
CI / Pending approval Slack notification (push) Has been skipped
CI / Send newsletter (push) Has been skipped

This commit is contained in:
upstream-mirror
2026-09-16 01:08:34 +05:00
commit 9456a4e922
2560 changed files with 120911 additions and 0 deletions

View File

@@ -0,0 +1,67 @@
{
"e": "2023-01-01T01:00:00Z",
"c": [
{
"n": "id",
"v": "token"
},
{
"n": "string",
"o": 1,
"v": "str"
},
{
"n": "string-zero",
"o": 1,
"v": ""
},
{
"n": "time",
"vt": "2023-01-01T00:00:00Z"
},
{
"n": "uuid",
"vu": "b652bbbd-1a84-4fb7-9437-6b78ba2686db"
},
{
"n": "nulluuid",
"vu": "b652bbbd-1a84-4fb7-9437-6b78ba2686da"
},
{
"n": "int64",
"vi": 64
},
{
"n": "int64-zero",
"vi": 0
},
{
"n": "empty-nullint64",
"vi": 0
},
{
"n": "nullint64",
"vi": 64
},
{
"n": "nullstring",
"v": "str"
},
{
"n": "empty-nullstring",
"v": ""
},
{
"n": "null-nulluuid",
"vn": true
},
{
"n": "null-nullstring",
"vn": true
},
{
"n": "null-nullint64",
"vn": true
}
]
}

View File

@@ -0,0 +1,245 @@
// Copyright © 2023 Ory Corp
// SPDX-License-Identifier: Apache-2.0
package keysetpagination
import (
"crypto/rand"
"database/sql"
"encoding/base64"
"encoding/json"
"time"
"github.com/gofrs/uuid"
"github.com/pkg/errors"
"github.com/ory/herodot"
"github.com/ory/x/aead"
)
// pageTokenContext is passed as additional authenticated data to the AEAD.
// It binds the ciphertext to its purpose: even if the encryption key is
// reused in another context (for example when it is derived from a shared
// system secret), ciphertexts from that context are rejected as page tokens
// and vice versa.
const pageTokenContext = "ory/keysetpagination_v2/page_token"
// fallbackEncryptionKey seals page tokens when no pagination secrets are
// configured. It is well known, so those tokens are only obfuscated, not
// authenticated. Configure pagination secrets to authenticate tokens.
var fallbackEncryptionKey [32]byte
type (
PageToken struct {
testNow func() time.Time
cols []Column
}
jsonPageToken = struct {
ExpiresAt time.Time `json:"e"`
Cols []jsonColumn `json:"c"`
}
jsonColumn = struct {
Name string `json:"n"`
Order Order `json:"o,omitempty"`
Nullable bool `json:"nl,omitempty"`
HasConstraint bool `json:"hc,omitempty"`
ValueAny any `json:"v,omitempty"`
ValueTime *time.Time `json:"vt,omitempty"`
ValueUUID *uuid.UUID `json:"vu,omitempty"`
ValueInt64 *int64 `json:"vi,omitempty"`
ValueNull bool `json:"vn,omitempty"`
}
Column struct {
Name string
Order Order
Value any
Nullable bool
// HasConstraint marks if the column is already constrained by WHERE condition.
HasConstraint bool
}
)
func (t PageToken) Columns() []Column { return t.cols }
// Encrypt encrypts the page token using the first key in the provided keyset.
// It uses a fallback key if no keys are provided.
func (t PageToken) Encrypt(keys [][32]byte) string {
key := fallbackEncryptionKey
if len(keys) > 0 {
key = keys[0]
}
enc, err := t.encrypt(key)
if err != nil {
// This should basically never happen, only if reading from the random source or marshaling the token fails.
// In both cases, we have a bigger problem than just not being able to generate the page token.
// Therefore, if we do get an error, there is no point in returning it to the client,
// as we already have a working result set. With this string, the next page will return an error,
// but that is better than breaking the current page.
return "internal error: failed to generate page token"
}
return enc
}
func (t PageToken) MarshalJSON() ([]byte, error) {
now := time.Now
if t.testNow != nil {
now = t.testNow
}
toEncode := jsonPageToken{
ExpiresAt: now().Add(time.Hour).UTC(),
Cols: make([]jsonColumn, len(t.cols)),
}
for i, col := range t.cols {
toEncode.Cols[i] = jsonColumn{
Name: col.Name,
Order: col.Order,
Nullable: col.Nullable,
HasConstraint: col.HasConstraint,
}
switch v := col.Value.(type) {
case time.Time:
toEncode.Cols[i].ValueTime = new(v)
case uuid.UUID:
toEncode.Cols[i].ValueUUID = new(v)
case uuid.NullUUID:
if v.Valid {
toEncode.Cols[i].ValueUUID = new(v.UUID)
} else {
toEncode.Cols[i].ValueNull = true
}
case sql.NullString:
if v.Valid {
toEncode.Cols[i].ValueAny = new(v.String)
} else {
toEncode.Cols[i].ValueNull = true
}
case int64:
toEncode.Cols[i].ValueInt64 = new(v)
case sql.NullInt64:
if v.Valid {
toEncode.Cols[i].ValueInt64 = new(v.Int64)
} else {
toEncode.Cols[i].ValueNull = true
}
default:
toEncode.Cols[i].ValueAny = v
}
}
return json.Marshal(toEncode)
}
func ErrPageTokenExpired() *herodot.DefaultError {
return herodot.ErrBadRequest().WithError("page token expired, do not persist page tokens")
}
func (t *PageToken) UnmarshalJSON(data []byte) error {
rawToken := jsonPageToken{}
if err := json.Unmarshal(data, &rawToken); err != nil {
return err
}
t.cols = make([]Column, len(rawToken.Cols))
for i, col := range rawToken.Cols {
t.cols[i] = Column{
Name: col.Name,
Order: col.Order,
Nullable: col.Nullable,
HasConstraint: col.HasConstraint,
}
switch {
case col.ValueNull:
t.cols[i].Value = nil
case col.ValueAny != nil:
t.cols[i].Value = col.ValueAny
case col.ValueUUID != nil:
t.cols[i].Value = *col.ValueUUID
case col.ValueTime != nil:
t.cols[i].Value = *col.ValueTime
case col.ValueInt64 != nil:
t.cols[i].Value = *col.ValueInt64
}
}
now := time.Now
if t.testNow != nil {
now = t.testNow
}
if rawToken.ExpiresAt.Before(now().UTC()) {
return errors.WithStack(ErrPageTokenExpired())
}
return nil
}
func NewPageToken(cols ...Column) PageToken { return PageToken{cols: cols} }
func (t *PageToken) encrypt(key [32]byte) (string, error) {
raw, err := json.Marshal(t)
if err != nil {
return "", errors.Wrap(err, "cannot marshal page token")
}
a, err := aead.New(key)
if err != nil {
return "", errors.Wrap(err, "cannot create AEAD")
}
// The nonce is prepended to the ciphertext. AEADs that manage the nonce
// internally report a nonce size of zero, so this also covers them.
nonce := make([]byte, a.NonceSize())
if _, err := rand.Read(nonce); err != nil {
return "", errors.Wrap(err, "cannot generate nonce")
}
return base64.URLEncoding.EncodeToString(a.Seal(nonce, nonce, raw, []byte(pageTokenContext))), nil
}
func (t *PageToken) decrypt(key [32]byte, s string) error {
if s == "" {
return errors.WithStack(ErrInvalidPaginationToken())
}
raw, err := base64.URLEncoding.DecodeString(s)
if err != nil {
return errors.WithStack(ErrInvalidPaginationToken())
}
dec, err := openAEAD(key, raw)
if err != nil {
// Tokens issued before the switch to a context-bound AEAD are sealed
// with NaCl secretbox. Remove this fallback once all tokens issued by
// the previous version have expired.
var ok bool
dec, ok = aead.OpenLegacySecretbox(key, raw)
if !ok {
return errors.WithStack(ErrInvalidPaginationToken())
}
}
if err := json.Unmarshal(dec, t); err != nil {
if errors.As(err, new(*herodot.DefaultError)) {
return err
}
return errors.WithStack(herodot.ErrInternalServerError().WithReason("unable to unmarshal page token").WithDebug(err.Error()))
}
return nil
}
func openAEAD(key [32]byte, raw []byte) ([]byte, error) {
a, err := aead.New(key)
if err != nil {
return nil, errors.Wrap(err, "cannot create AEAD")
}
if len(raw) < a.NonceSize() {
return nil, errors.New("ciphertext too short")
}
nonce, ciphertext := raw[:a.NonceSize()], raw[a.NonceSize():]
bs, err := a.Open(nil, nonce, ciphertext, []byte(pageTokenContext))
if err != nil {
return nil, errors.Wrap(err, "cannot open AEAD")
}
return bs, nil
}

View File

@@ -0,0 +1,192 @@
// Copyright © 2023 Ory Corp
// SPDX-License-Identifier: Apache-2.0
package keysetpagination
import (
"cmp"
"reflect"
"github.com/jmoiron/sqlx/reflectx"
"github.com/ory/herodot"
)
func ErrInvalidPaginationToken() *herodot.DefaultError {
return herodot.ErrBadRequest().WithError("invalid pagination token")
}
type (
Paginator struct {
token, defaultToken PageToken
size, defaultSize, maxSize int
isLast bool
}
Option func(*Paginator)
)
const (
DefaultSize = 100
DefaultMaxSize = 500
)
var dbStructTagMapper = reflectx.NewMapper("db")
func (p *Paginator) DefaultToken() PageToken { return p.defaultToken }
func (p *Paginator) IsLast() bool { return p.isLast }
func (p *Paginator) PageToken() PageToken {
if p.token.cols != nil {
return p.token
}
return p.defaultToken
}
func (p *Paginator) Size() int {
defaultSize := cmp.Or(p.defaultSize, DefaultSize)
maxSize := cmp.Or(p.maxSize, DefaultMaxSize)
size := p.size
if size <= 0 {
size = defaultSize
}
if size > maxSize {
size = maxSize
}
return size
}
func (p *Paginator) ToOptions() []Option {
opts := make([]Option, 0, 6)
if p.token.cols != nil {
opts = append(opts, WithToken(p.token))
}
if p.defaultToken.cols != nil {
opts = append(opts, WithDefaultToken(p.defaultToken))
}
if p.size > 0 {
opts = append(opts, WithSize(p.size))
}
if p.defaultSize != DefaultSize {
opts = append(opts, WithDefaultSize(p.defaultSize))
}
if p.maxSize != DefaultMaxSize {
opts = append(opts, WithMaxSize(p.maxSize))
}
if p.isLast {
opts = append(opts, withIsLast(p.isLast))
}
return opts
}
// Result removes the last item (if applicable) and returns the paginator for the next page.
func Result[I any](items []I, p *Paginator) ([]I, *Paginator) {
return ResultFunc(items, p, func(last I, colName string) any {
lastItemVal := reflect.ValueOf(last)
return dbStructTagMapper.FieldByName(lastItemVal, colName).Interface()
})
}
// ResultFunc removes the last item (if applicable) and returns the paginator for the next page.
// The extractor function is used to extract the column values from the last item.
func ResultFunc[I any](items []I, p *Paginator, extractor func(last I, colName string) any) ([]I, *Paginator) {
if len(items) <= p.Size() {
return items, &Paginator{
isLast: true,
defaultToken: p.defaultToken,
size: p.size,
defaultSize: p.defaultSize,
maxSize: p.maxSize,
}
}
items = items[:p.Size()]
lastItem := items[len(items)-1]
currentCols := p.PageToken().Columns()
newCols := make([]Column, len(currentCols))
for i, col := range currentCols {
newCols[i] = Column{
Name: col.Name,
Order: col.Order,
Nullable: col.Nullable,
HasConstraint: col.HasConstraint,
Value: extractor(lastItem, col.Name),
}
}
return items, &Paginator{
token: NewPageToken(newCols...),
defaultToken: p.defaultToken,
size: p.size,
defaultSize: p.defaultSize,
maxSize: p.maxSize,
}
}
func WithSize(size int) Option {
return func(p *Paginator) { p.size = size }
}
func WithDefaultSize(size int) Option {
return func(p *Paginator) { p.defaultSize = size }
}
func WithMaxSize(size int) Option {
return func(p *Paginator) { p.maxSize = size }
}
func WithToken(t PageToken) Option {
return func(p *Paginator) { p.token = t }
}
func WithDefaultToken(t PageToken) Option {
return func(p *Paginator) { p.defaultToken = t }
}
func withIsLast(isLast bool) Option {
return func(p *Paginator) { p.isLast = isLast }
}
func NewPaginator(modifiers ...Option) (*Paginator, error) {
p := &Paginator{
// these can still be overridden by the modifiers, but they should never be unset
maxSize: DefaultMaxSize,
defaultSize: DefaultSize,
}
for _, f := range modifiers {
f(p)
}
if err := p.validatePageToken(); err != nil {
return nil, err
}
return p, nil
}
// validatePageToken ensures the page token columns strictly match DefaultToken's schema in order.
func (p *Paginator) validatePageToken() error {
tokenCols := p.PageToken().Columns()
if len(tokenCols) == 0 {
return nil
}
if columnsMatch(tokenCols, p.defaultToken.Columns()) {
return nil
}
return ErrInvalidPaginationToken()
}
func columnsMatch(tokenCols, templateCols []Column) bool {
if len(templateCols) == 0 || len(tokenCols) != len(templateCols) {
return false
}
for i, col := range tokenCols {
def := templateCols[i]
if def.Name != col.Name || def.Order != col.Order || def.Nullable != col.Nullable {
return false
}
}
return true
}

View File

@@ -0,0 +1,35 @@
// Copyright © 2023 Ory Corp
// SPDX-License-Identifier: Apache-2.0
package keysetpagination
import (
"net/http"
"net/url"
"github.com/peterhellberg/link"
)
// ParseHeader parses the response header's Link and returns the first and next page tokens.
func ParseHeader(r *http.Response) (first, next string, isLast bool) {
links := link.ParseResponse(r)
first, _ = findRel(links, "first")
next, hasNext := findRel(links, "next")
return first, next, !hasNext
}
func findRel(links link.Group, rel string) (string, bool) {
for idx, l := range links {
if idx == rel {
parsed, err := url.Parse(l.URI)
if err != nil {
continue
}
q := parsed.Query()
return q.Get("page_token"), q.Has("page_token")
}
}
return "", false
}

View File

@@ -0,0 +1,163 @@
// Copyright © 2025 Ory Corp
// SPDX-License-Identifier: Apache-2.0
package keysetpagination
import (
"database/sql/driver"
"fmt"
"strings"
"github.com/ory/pop/v6"
)
type Order int
const (
OrderAscending Order = iota
OrderDescending
)
func (o Order) extract() (string, string) {
switch o {
case OrderAscending:
return ">", "ASC"
case OrderDescending:
return "<", "DESC"
default:
panic(fmt.Sprintf("keyset pagination: unknown order %d", o))
}
}
// Paginate returns a function that paginates a pop.Query.
// Usage:
//
// q := c.Where("foo = ?", foo).Scope(keysetpagination.Paginate[MyItemType](paginator))
func Paginate[I any](p *Paginator) pop.ScopeFunc {
model := pop.Model{Value: *new(I)}
tableName := model.Alias()
return func(q *pop.Query) *pop.Query {
quoteAndContextualize := func(name string) string {
quote := q.Connection.Dialect.Quote
return quote(tableName) + "." + quote(name)
}
dialect := q.Connection.Dialect.Name()
where, args, order := BuildWhereAndOrder(p.PageToken().Columns(), quoteAndContextualize, dialect)
// IMPORTANT: Ensures correct query logic by grouping conditions.
// Without parentheses, `WHERE otherCond AND pageCond1 OR pageCond2` would be
// evaluated as `(otherCond = ? AND pageCond1) OR pageCond2`, potentially returning
// rows that do not match `otherCond`.
// We fix it by forcing the query to be: `WHERE otherCond AND (paginationCond1 OR paginationCond2)`.
where = "(" + where + ")"
return q.
Where(where, args...).
Order(order).
Limit(p.Size() + 1)
}
}
func BuildWhereAndOrder(columns []Column, quote func(string) string, dialect string) (string, []any, string) {
var whereBuilder, orderByBuilder, prevEqual strings.Builder
keysetCols := make([]Column, 0, len(columns))
// ORDER BY includes all columns (even constrained ones)
for i, part := range columns {
column := quote(part.Name)
_, keyword := part.Order.extract()
if i > 0 {
orderByBuilder.WriteString(", ")
}
orderByBuilder.WriteString(column + " " + keyword)
// Postgres orders NULLs differently depending on sort direction;
// (ASC → NULLS LAST, DESC → NULLS FIRST), which does not match the
// assumptions of our keyset pagination logic and other supported DBs.
// We therefore make NULL ordering explicit on Postgres (and YugabyteDB,
// which shares Postgres NULL ordering) to keep pagination stable and
// consistent with sqlite/mysql/cockroachdb.
if (dialect == "postgres" || dialect == "yugabyte") && part.Nullable {
if part.Order == OrderAscending {
orderByBuilder.WriteString(" NULLS FIRST")
} else {
orderByBuilder.WriteString(" NULLS LAST")
}
}
// Build keyset WHERE only from unconstrained columns
if !part.HasConstraint {
keysetCols = append(keysetCols, part)
}
}
// If everything is constrained, no keyset predicate is needed.
if len(keysetCols) == 0 {
return "", nil, orderByBuilder.String()
}
args := make([]any, 0, len(keysetCols)*(len(keysetCols)+1)/2)
prevEqualArgs := make([]any, 0, len(keysetCols))
whereBuilder.WriteRune('(')
for i, part := range keysetCols {
column := quote(part.Name)
sign, _ := part.Order.extract()
if i > 0 {
whereBuilder.WriteString(") OR (")
}
whereBuilder.WriteString(prevEqual.String())
if prevEqual.Len() > 0 {
whereBuilder.WriteString(" AND ")
}
isNull := part.Nullable && isSQLNull(part.Value)
if !part.Nullable {
whereBuilder.WriteString(column + " " + sign + " ?")
} else if !isNull {
whereBuilder.WriteString(column + " IS NOT NULL AND " + column + " " + sign + " ?")
} else {
whereBuilder.WriteString(column + " IS NOT NULL")
}
if i > 0 {
prevEqual.WriteString(" AND ")
}
if !part.Nullable {
prevEqual.WriteString(column + " = ?")
prevEqualArgs = append(prevEqualArgs, part.Value)
} else if !isNull {
prevEqual.WriteString(column + " = ?")
prevEqualArgs = append(prevEqualArgs, part.Value)
} else {
prevEqual.WriteString(column + " IS NULL")
}
args = append(args, prevEqualArgs...)
}
whereBuilder.WriteRune(')')
return whereBuilder.String(), args, orderByBuilder.String()
}
// isSQLNull reports whether v represents a SQL NULL value.
func isSQLNull(v any) bool {
if v == nil {
return true
}
if valuer, ok := v.(driver.Valuer); ok {
val, err := valuer.Value()
return err == nil && val == nil
}
return false
}

View File

@@ -0,0 +1,126 @@
// Copyright © 2023 Ory Corp
// SPDX-License-Identifier: Apache-2.0
package keysetpagination
import (
"cmp"
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
"github.com/pkg/errors"
)
// Pagination Request Parameters
//
// For details on pagination please head over to the [pagination documentation](https://www.ory.com/docs/ecosystem/api-design#pagination).
//
// swagger:model keysetPaginationRequestParameters
type RequestParameters struct {
// Items per Page
//
// This is the number of items per page to return.
// For details on pagination please head over to the [pagination documentation](https://www.ory.com/docs/ecosystem/api-design#pagination).
//
// required: false
// in: query
// default: 250
// min: 1
// max: 1000
PageSize int `json:"page_size"`
// Next Page Token
//
// The next page token.
// For details on pagination please head over to the [pagination documentation](https://www.ory.com/docs/ecosystem/api-design#pagination).
//
// required: false
// in: query
PageToken string `json:"page_token"`
}
// Pagination Response Header
//
// The `Link` HTTP header contains multiple links (`first`, `next`) formatted as:
// `<https://{project-slug}.projects.oryapis.com/admin/sessions?page_size=250&page_token=>; rel="first"`
//
// For details on pagination please head over to the [pagination documentation](https://www.ory.com/docs/ecosystem/api-design#pagination).
//
// swagger:model keysetPaginationResponseHeaders
type ResponseHeaders struct {
// The Link HTTP Header
//
// The `Link` header contains a comma-delimited list of links to the following pages:
//
// - first: The first page of results.
// - next: The next page of results.
//
// Pages are omitted if they do not exist. For example, if there is no next page, the `next` link is omitted. Examples:
//
// </admin/sessions?page_size=250&page_token={last_item_uuid}; rel="first",/admin/sessions?page_size=250&page_token=>; rel="next"
//
Link string `json:"link"`
}
// SetLinkHeader adds the Link header for the page encoded by the paginator.
// It contains links to the first and next page, if one exists.
func SetLinkHeader(w http.ResponseWriter, keys [][32]byte, u *url.URL, p *Paginator) {
size := p.Size()
link := []string{linkPart(u, "first", p.DefaultToken().Encrypt(keys), size)}
if !p.isLast {
link = append(link, linkPart(u, "next", p.PageToken().Encrypt(keys), size))
}
w.Header().Set("Link", strings.Join(link, ","))
}
func linkPart(u *url.URL, rel, token string, size int) string {
q := u.Query()
q.Set("page_token", token)
q.Set("page_size", strconv.Itoa(size))
u.RawQuery = q.Encode()
return fmt.Sprintf("<%s>; rel=%q", u.String(), rel)
}
// ParseQueryParams extracts the pagination options from the URL query.
func ParseQueryParams(keys [][32]byte, q url.Values) ([]Option, error) {
var opts []Option
if t := cmp.Or(q["page_token"]...); t != "" {
raw, err := url.QueryUnescape(t)
if err != nil {
return nil, errors.WithStack(err)
}
token, err := ParsePageToken(keys, raw)
if err != nil {
return nil, err
}
opts = append(opts, WithToken(token))
}
if s := cmp.Or(q["page_size"]...); s != "" {
size, err := strconv.Atoi(s)
if err != nil {
return nil, errors.WithStack(err)
}
opts = append(opts, WithSize(size))
}
return opts, nil
}
// ParsePageToken parses a page token from the given raw string using the provided keys.
// If decryption fails with all provided keys (including when no keys are given), it
// falls back to using the fallbackEncryptionKey and returns an error if that also fails.
func ParsePageToken(keys [][32]byte, raw string) (t PageToken, err error) {
for i := range keys {
err = errors.WithStack(t.decrypt(keys[i], raw))
if errors.Is(err, ErrInvalidPaginationToken()) {
continue
}
// either we successfully decrypted the token, or we got an error that is not ErrInvalidPaginationToken, in both cases we should return immediately
return t, err
}
// as a last resort, try the fallback key
err = t.decrypt(fallbackEncryptionKey, raw)
return t, errors.WithStack(err)
}