snapshot of https://github.com/ory/keto.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
CI / Run lints and checks (push) Failing after 47s
CI / Run tests (push) Failing after 1s
CI / Run race tests (push) Failing after 5s
CI / Build CLI docs (push) Has been skipped
CI / Run buf actions (push) Failing after 5s
CodeQL / Analyze (go) (push) Failing after 3s
CodeQL / Analyze (javascript) (push) Failing after 2s
Docker Image Scanners / scanners (push) Failing after 3s
Format / format (push) Failing after 3s
Synchronize Issue Labels / Synchronize Issue Labels (push) Has been skipped
CI / Generate release (push) Has been skipped
CI / Draft newsletter (push) Has been skipped
CI / Pending approval Slack notification (push) Has been skipped
CI / Send newsletter (push) Has been skipped
Some checks failed
CI / Run lints and checks (push) Failing after 47s
CI / Run tests (push) Failing after 1s
CI / Run race tests (push) Failing after 5s
CI / Build CLI docs (push) Has been skipped
CI / Run buf actions (push) Failing after 5s
CodeQL / Analyze (go) (push) Failing after 3s
CodeQL / Analyze (javascript) (push) Failing after 2s
Docker Image Scanners / scanners (push) Failing after 3s
Format / format (push) Failing after 3s
Synchronize Issue Labels / Synchronize Issue Labels (push) Has been skipped
CI / Generate release (push) Has been skipped
CI / Draft newsletter (push) Has been skipped
CI / Pending approval Slack notification (push) Has been skipped
CI / Send newsletter (push) Has been skipped
This commit is contained in:
23
oryx/sqlcon/connector.go
Normal file
23
oryx/sqlcon/connector.go
Normal file
@@ -0,0 +1,23 @@
|
||||
// Copyright © 2023 Ory Corp
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package sqlcon provides helpers for dealing with SQL connectivity.
|
||||
package sqlcon
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// GetDriverName returns the driver name of a given DSN.
|
||||
func GetDriverName(dsn string) string {
|
||||
return strings.Split(dsn, "://")[0]
|
||||
}
|
||||
func maxParallelism() int {
|
||||
maxProcs := runtime.GOMAXPROCS(0)
|
||||
numCPU := runtime.NumCPU()
|
||||
if maxProcs < numCPU {
|
||||
return maxProcs
|
||||
}
|
||||
return numCPU
|
||||
}
|
||||
22
oryx/sqlcon/dockertest/cockroach.go
Normal file
22
oryx/sqlcon/dockertest/cockroach.go
Normal file
@@ -0,0 +1,22 @@
|
||||
// Copyright © 2023 Ory Corp
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package dockertest
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/cockroachdb/cockroach-go/v2/testserver"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func NewLocalTestCRDBServer(t testing.TB) string {
|
||||
ts, err := testserver.NewTestServer(testserver.CustomVersionOpt("25.3.3"))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(ts.Stop)
|
||||
|
||||
require.NoError(t, ts.WaitForInit())
|
||||
|
||||
ts.PGURL().Scheme = "cockroach"
|
||||
return ts.PGURL().String()
|
||||
}
|
||||
195
oryx/sqlcon/dockertest/test_helper.go
Normal file
195
oryx/sqlcon/dockertest/test_helper.go
Normal file
@@ -0,0 +1,195 @@
|
||||
// Copyright © 2023 Ory Corp
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package dockertest
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/moby/moby/client"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/ory/pop/v6"
|
||||
|
||||
"github.com/ory/dockertest/v4"
|
||||
"github.com/ory/x/stringsx"
|
||||
)
|
||||
|
||||
func ConnectPop(t require.TestingT, url string) (c *pop.Connection) {
|
||||
require.EventuallyWithT(t, func(t *assert.CollectT) {
|
||||
var err error
|
||||
c, err = pop.NewConnection(&pop.ConnectionDetails{
|
||||
URL: url,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, c.Open())
|
||||
require.NoError(t, c.RawQuery("select version()").Exec())
|
||||
}, 5*time.Minute, time.Second*5, "could not connect to database at %s", url)
|
||||
return
|
||||
}
|
||||
|
||||
// ## PostgreSQL ##
|
||||
|
||||
func startPostgreSQL(t testing.TB, version string) dockertest.Resource {
|
||||
pool := dockertest.NewPoolT(t, "")
|
||||
return pool.RunT(t, "postgres",
|
||||
dockertest.WithTag(cmp.Or(version, "16")),
|
||||
dockertest.WithoutReuse(),
|
||||
dockertest.WithEnv([]string{"PGUSER=postgres", "POSTGRES_PASSWORD=secret", "POSTGRES_DB=postgres"}),
|
||||
)
|
||||
}
|
||||
|
||||
// RunTestPostgreSQL runs a PostgreSQL database and returns the URL to it.
|
||||
// If a docker container is started for the database, the container be removed
|
||||
// at the end of the test.
|
||||
func RunTestPostgreSQL(t testing.TB) string {
|
||||
if dsn := os.Getenv("TEST_DATABASE_POSTGRESQL"); dsn != "" {
|
||||
t.Logf("Skipping Docker setup because environment variable TEST_DATABASE_POSTGRESQL is set to: %s", dsn)
|
||||
return dsn
|
||||
}
|
||||
r := startPostgreSQL(t, "")
|
||||
return fmt.Sprintf("postgres://postgres:secret@127.0.0.1:%s/postgres?sslmode=disable", r.GetPort("5432/tcp"))
|
||||
}
|
||||
|
||||
// RunTestPostgreSQLWithVersion connects to a PostgreSQL database.
|
||||
func RunTestPostgreSQLWithVersion(t testing.TB, version string) string {
|
||||
if dsn := os.Getenv("TEST_DATABASE_POSTGRESQL"); dsn != "" {
|
||||
return dsn
|
||||
}
|
||||
r := startPostgreSQL(t, version)
|
||||
return fmt.Sprintf("postgres://postgres:secret@127.0.0.1:%s/postgres?sslmode=disable", r.GetPort("5432/tcp"))
|
||||
}
|
||||
|
||||
// ## MySQL ##
|
||||
|
||||
func startMySQL(t testing.TB, version string) dockertest.Resource {
|
||||
pool := dockertest.NewPoolT(t, "")
|
||||
return pool.RunT(t, "mysql",
|
||||
dockertest.WithTag(cmp.Or(version, "8.0")),
|
||||
dockertest.WithoutReuse(),
|
||||
dockertest.WithEnv([]string{
|
||||
"MYSQL_ROOT_PASSWORD=secret",
|
||||
"MYSQL_ROOT_HOST=%",
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
// RunTestMySQL runs a MySQL database and returns the URL to it.
|
||||
// If a docker container is started for the database, the container be removed
|
||||
// at the end of the test.
|
||||
func RunTestMySQL(t testing.TB) string {
|
||||
if dsn := os.Getenv("TEST_DATABASE_MYSQL"); dsn != "" {
|
||||
t.Logf("Skipping Docker setup because environment variable TEST_DATABASE_MYSQL is set to: %s", dsn)
|
||||
return dsn
|
||||
}
|
||||
r := startMySQL(t, "")
|
||||
return fmt.Sprintf("mysql://root:secret@tcp(localhost:%s)/mysql?parseTime=true&multiStatements=true", r.GetPort("3306/tcp"))
|
||||
}
|
||||
|
||||
// RunTestMySQLWithVersion runs a MySQL database in the specified version and returns the URL to it.
|
||||
// If a docker container is started for the database, the container be removed
|
||||
// at the end of the test.
|
||||
func RunTestMySQLWithVersion(t testing.TB, version string) string {
|
||||
if dsn := os.Getenv("TEST_DATABASE_MYSQL"); dsn != "" {
|
||||
t.Logf("Skipping Docker setup because environment variable TEST_DATABASE_MYSQL is set to: %s", dsn)
|
||||
return dsn
|
||||
}
|
||||
r := startMySQL(t, version)
|
||||
return fmt.Sprintf("mysql://root:secret@tcp(localhost:%s)/mysql?parseTime=true&multiStatements=true", r.GetPort("3306/tcp"))
|
||||
}
|
||||
|
||||
// ## CockroachDB
|
||||
|
||||
func startCockroachDB(t testing.TB, version string) dockertest.Resource {
|
||||
pool := dockertest.NewPoolT(t, "")
|
||||
return pool.RunT(t, "cockroachdb/cockroach",
|
||||
dockertest.WithTag(cmp.Or(version, "latest-v25.4")),
|
||||
dockertest.WithoutReuse(),
|
||||
dockertest.WithCmd([]string{"start-single-node", "--insecure"}),
|
||||
)
|
||||
}
|
||||
|
||||
// RunTestCockroachDB runs a CockroachDB database and returns the URL to it.
|
||||
// If a docker container is started for the database, the container be removed
|
||||
// at the end of the test.
|
||||
func RunTestCockroachDB(t testing.TB) string {
|
||||
return RunTestCockroachDBWithVersion(t, "")
|
||||
}
|
||||
|
||||
// RunTestCockroachDBWithVersion runs a CockroachDB database and returns the URL to it.
|
||||
// If a docker container is started for the database, the container be removed
|
||||
// at the end of the test.
|
||||
func RunTestCockroachDBWithVersion(t testing.TB, version string) string {
|
||||
if dsn := os.Getenv("TEST_DATABASE_COCKROACHDB"); dsn != "" {
|
||||
t.Logf("Skipping Docker setup because environment variable TEST_DATABASE_COCKROACHDB is set to: %s", dsn)
|
||||
return dsn
|
||||
}
|
||||
r := startCockroachDB(t, version)
|
||||
return fmt.Sprintf("cockroach://root@localhost:%s/defaultdb?sslmode=disable", r.GetPort("26257/tcp"))
|
||||
}
|
||||
|
||||
func DumpSchema(t testing.TB, c *pop.Connection) string {
|
||||
name, database, port := c.Dialect.Name(), c.Dialect.Details().Database, c.Dialect.Details().Port
|
||||
t.Logf("Dumping schema for dialect %s, database %s on port %s", name, database, port)
|
||||
|
||||
var cmd []string
|
||||
var appendToDump string
|
||||
switch dialects := stringsx.SwitchExact(name); {
|
||||
case dialects.AddCase("sqlite3"):
|
||||
return dumpSQLiteSchema(t, c)
|
||||
case dialects.AddCase("postgres"):
|
||||
cmd = []string{"pg_dump", "--username", "postgres", "--schema-only", "--dbname", database}
|
||||
// we need to set the search path because the postgres dump always unsets it
|
||||
appendToDump = "SET search_path TO public;\n"
|
||||
case dialects.AddCase("mysql"):
|
||||
cmd = []string{"mysqldump", "--user", "root", "--password=secret", "--no-data", database}
|
||||
case dialects.AddCase("cockroach"):
|
||||
cmd = []string{"cockroach", "sql", "--insecure", "--database", database, "--execute", "SHOW CREATE ALL TABLES; SHOW CREATE ALL TYPES;", "--format", "raw"}
|
||||
default:
|
||||
t.Log(dialects.ToUnknownCaseErr())
|
||||
t.FailNow()
|
||||
return ""
|
||||
}
|
||||
|
||||
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
|
||||
require.NoError(t, err)
|
||||
list, err := cli.ContainerList(t.Context(), client.ContainerListOptions{
|
||||
Filters: client.Filters{}.Add("publish", port),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Lenf(t, list.Items, 1, "expected exactly one %s container with port %s", name, port)
|
||||
|
||||
process, err := cli.ExecCreate(t.Context(), list.Items[0].ID, client.ExecCreateOptions{
|
||||
TTY: true,
|
||||
AttachStdout: true,
|
||||
Cmd: cmd,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
resp, err := cli.ExecAttach(t.Context(), process.ID, client.ExecAttachOptions{
|
||||
TTY: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
dump, err := io.ReadAll(resp.Reader)
|
||||
require.NoErrorf(t, err, "%s", dump)
|
||||
|
||||
d := string(dump) + appendToDump
|
||||
d = regexp.MustCompile(`(--|#|\\|mysqldump|SHOW CREATE)[^\n]*\n`).ReplaceAllLiteralString(d, "") // comments and other non-schema lines
|
||||
d = strings.ReplaceAll(d, "\r\n", "\n")
|
||||
d = regexp.MustCompile(`\n\n+`).ReplaceAllLiteralString(d, "\n\n")
|
||||
return d
|
||||
}
|
||||
|
||||
func dumpSQLiteSchema(t testing.TB, c *pop.Connection) string {
|
||||
var sqls []string
|
||||
require.NoError(t, c.RawQuery("SELECT sql FROM sqlite_master WHERE type IN ('table', 'index', 'trigger', 'view') AND name NOT LIKE 'sqlite_%' ORDER BY name").All(&sqls))
|
||||
return strings.Join(sqls, ";\n") + ";\n"
|
||||
}
|
||||
150
oryx/sqlcon/error.go
Normal file
150
oryx/sqlcon/error.go
Normal file
@@ -0,0 +1,150 @@
|
||||
// Copyright © 2023 Ory Corp
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package sqlcon
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
stderrs "errors"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/pkg/errors"
|
||||
"google.golang.org/grpc/codes"
|
||||
"modernc.org/sqlite"
|
||||
|
||||
"github.com/ory/herodot"
|
||||
)
|
||||
|
||||
// ErrUniqueViolation is returned when a SQL INSERT / UPDATE command returns a conflict.
|
||||
func ErrUniqueViolation() *herodot.DefaultError {
|
||||
return &herodot.DefaultError{
|
||||
CodeField: http.StatusConflict,
|
||||
GRPCCodeField: codes.AlreadyExists,
|
||||
StatusField: http.StatusText(http.StatusConflict),
|
||||
ErrorField: "Unable to insert or update resource because a resource with that value exists already",
|
||||
}
|
||||
}
|
||||
|
||||
// ErrNoRows is returned when a SQL SELECT statement returns no rows.
|
||||
func ErrNoRows() *herodot.DefaultError {
|
||||
return &herodot.DefaultError{
|
||||
CodeField: http.StatusNotFound,
|
||||
GRPCCodeField: codes.NotFound,
|
||||
StatusField: http.StatusText(http.StatusNotFound),
|
||||
ErrorField: "Unable to locate the resource",
|
||||
}
|
||||
}
|
||||
|
||||
// ErrConcurrentUpdate is returned when the database is unable to serialize access due to a concurrent update.
|
||||
func ErrConcurrentUpdate() *herodot.DefaultError {
|
||||
return &herodot.DefaultError{
|
||||
CodeField: http.StatusBadRequest,
|
||||
GRPCCodeField: codes.Aborted,
|
||||
StatusField: http.StatusText(http.StatusBadRequest),
|
||||
ErrorField: "Unable to serialize access due to a concurrent update in another session",
|
||||
}
|
||||
}
|
||||
|
||||
func ErrNoSuchTable() *herodot.DefaultError {
|
||||
return &herodot.DefaultError{
|
||||
CodeField: http.StatusInternalServerError,
|
||||
GRPCCodeField: codes.Internal,
|
||||
StatusField: http.StatusText(http.StatusInternalServerError),
|
||||
ErrorField: "Unable to locate the table",
|
||||
}
|
||||
}
|
||||
|
||||
func ErrNoSuchColumn() *herodot.DefaultError {
|
||||
return &herodot.DefaultError{
|
||||
CodeField: http.StatusInternalServerError,
|
||||
GRPCCodeField: codes.Internal,
|
||||
StatusField: http.StatusText(http.StatusInternalServerError),
|
||||
ErrorField: "Unable to locate the column",
|
||||
}
|
||||
}
|
||||
|
||||
func handlePostgres(err error, sqlState string) error {
|
||||
switch sqlState {
|
||||
case "23505": // "unique_violation"
|
||||
return errors.WithStack(ErrUniqueViolation().WithWrap(err))
|
||||
case "40001", // "serialization_failure" in CRDB
|
||||
"CR000": // "serialization_failure"
|
||||
return errors.WithStack(ErrConcurrentUpdate().WithWrap(err))
|
||||
case "42P01": // "no such table"
|
||||
return errors.WithStack(ErrNoSuchTable().WithWrap(err))
|
||||
case "42703": // "no such column"
|
||||
return errors.WithStack(ErrNoSuchColumn().WithWrap(err))
|
||||
}
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
// HandleError returns the right sqlcon.Err* depending on the input error.
|
||||
func HandleError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
type stater = interface {
|
||||
error
|
||||
SQLState() string
|
||||
}
|
||||
|
||||
if stderrs.Is(err, sql.ErrNoRows) {
|
||||
return errors.WithStack(ErrNoRows())
|
||||
}
|
||||
if e, ok := stderrs.AsType[stater](err); ok {
|
||||
return errors.WithStack(handlePostgres(err, e.SQLState()))
|
||||
}
|
||||
if e, ok := stderrs.AsType[*pgconn.PgError](err); ok {
|
||||
return errors.WithStack(handlePostgres(err, e.Code))
|
||||
}
|
||||
if e, ok := stderrs.AsType[*mysql.MySQLError](err); ok {
|
||||
switch e.Number {
|
||||
case 1062: // https://dev.mysql.com/doc/mysql-errors/8.0/en/server-error-reference.html#error_er_dup_entry
|
||||
return errors.WithStack(ErrUniqueViolation().WithWrap(err))
|
||||
case 1146: // https://dev.mysql.com/doc/mysql-errors/8.0/en/server-error-reference.html#error_er_no_such_table
|
||||
return errors.WithStack(ErrNoSuchTable().WithWrap(e))
|
||||
case 1054: // https://dev.mysql.com/doc/mysql-errors/8.0/en/server-error-reference.html#error_er_bad_field_error
|
||||
return errors.WithStack(ErrNoSuchColumn().WithWrap(e))
|
||||
case 1213: // https://dev.mysql.com/doc/mysql-errors/8.0/en/server-error-reference.html#error_er_lock_deadlock
|
||||
return errors.WithStack(ErrConcurrentUpdate().WithWrap(e))
|
||||
}
|
||||
}
|
||||
if e, ok := stderrs.AsType[*sqlite.Error](err); ok {
|
||||
return handleSqlite(e, err)
|
||||
}
|
||||
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
// IsForeignKeyViolation reports whether err is a foreign-key constraint
|
||||
// violation, e.g. deleting a parent row that a child row still references
|
||||
// under ON DELETE RESTRICT or NO ACTION.
|
||||
func IsForeignKeyViolation(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
type stater = interface {
|
||||
error
|
||||
SQLState() string
|
||||
}
|
||||
|
||||
if e, ok := stderrs.AsType[stater](err); ok {
|
||||
return e.SQLState() == "23503" // "foreign_key_violation" in Postgres / CockroachDB
|
||||
}
|
||||
if e, ok := stderrs.AsType[*pgconn.PgError](err); ok {
|
||||
return e.Code == "23503"
|
||||
}
|
||||
if e, ok := stderrs.AsType[*mysql.MySQLError](err); ok {
|
||||
// 1451: cannot delete or update a parent row (a foreign key constraint fails).
|
||||
// https://dev.mysql.com/doc/mysql-errors/8.0/en/server-error-reference.html#error_er_row_is_referenced_2
|
||||
return e.Number == 1451
|
||||
}
|
||||
if e, ok := stderrs.AsType[*sqlite.Error](err); ok {
|
||||
return isSqliteForeignKeyViolation(e)
|
||||
}
|
||||
return false
|
||||
}
|
||||
47
oryx/sqlcon/error_sqlite.go
Normal file
47
oryx/sqlcon/error_sqlite.go
Normal file
@@ -0,0 +1,47 @@
|
||||
// Copyright © 2023 Ory Corp
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package sqlcon
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"modernc.org/sqlite"
|
||||
sqlite3lib "modernc.org/sqlite/lib"
|
||||
)
|
||||
|
||||
// isSqliteForeignKeyViolation reports whether e is an SQLite foreign-key
|
||||
// constraint failure. SQLite's extended result code for this case is not
|
||||
// reported consistently by the driver, so we match on the stable error
|
||||
// message instead.
|
||||
func isSqliteForeignKeyViolation(e *sqlite.Error) bool {
|
||||
return strings.Contains(e.Error(), "FOREIGN KEY constraint failed")
|
||||
}
|
||||
|
||||
// handleSqlite handles the error iff (if and only if) it is an sqlite error
|
||||
func handleSqlite(e *sqlite.Error, err error) error {
|
||||
// Code() returns the full extended error code.
|
||||
switch e.Code() {
|
||||
case sqlite3lib.SQLITE_CONSTRAINT_UNIQUE,
|
||||
sqlite3lib.SQLITE_CONSTRAINT_PRIMARYKEY:
|
||||
return errors.WithStack(ErrUniqueViolation().WithWrap(err))
|
||||
case sqlite3lib.SQLITE_ERROR:
|
||||
if strings.Contains(e.Error(), "no such table") {
|
||||
return errors.WithStack(ErrNoSuchTable().WithWrap(err))
|
||||
}
|
||||
case sqlite3lib.SQLITE_LOCKED,
|
||||
sqlite3lib.SQLITE_BUSY,
|
||||
sqlite3lib.SQLITE_BUSY_RECOVERY,
|
||||
sqlite3lib.SQLITE_BUSY_SNAPSHOT,
|
||||
sqlite3lib.SQLITE_BUSY_TIMEOUT:
|
||||
return errors.WithStack(ErrConcurrentUpdate().WithWrap(err))
|
||||
}
|
||||
|
||||
if strings.HasPrefix(e.Error(), "no such column:") ||
|
||||
strings.Contains(e.Error(), "has no column named") {
|
||||
return errors.WithStack(ErrNoSuchColumn().WithWrap(err))
|
||||
}
|
||||
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
87
oryx/sqlcon/message.go
Normal file
87
oryx/sqlcon/message.go
Normal file
@@ -0,0 +1,87 @@
|
||||
// Copyright © 2023 Ory Corp
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package sqlcon
|
||||
|
||||
// HelpMessage returns a string explaining how to set up SQL using environment variables.
|
||||
func HelpMessage() string {
|
||||
return `- DATABASE_URL: A DSN to a persistent backend. Various backends are supported:
|
||||
|
||||
- Changes are lost on process death (ephemeral storage):
|
||||
|
||||
- Memory: If DATABASE_URL is "memory", data will be written to memory and is lost when you restart this instance.
|
||||
Example: DATABASE_URL=memory
|
||||
|
||||
- Changes are kept after process death (persistent storage):
|
||||
|
||||
- SQL Databases: Officially, PostgreSQL, MySQL and CockroachDB are supported. This project works best with PostgreSQL.
|
||||
|
||||
- PostgreSQL: If DATABASE_URL is a DSN starting with postgres://, PostgreSQL will be used as storage backend.
|
||||
Example: DATABASE_URL=postgres://user:password@host:123/database
|
||||
|
||||
Additionally, the following query/DSN parameters are supported:
|
||||
|
||||
* max_conns (number): Sets the maximum number of open connections to the database. Defaults to the number of CPU cores times 2.
|
||||
* max_idle_conns (number): Sets the maximum number of connections in the idle. Defaults to the number of CPU cores.
|
||||
* max_conn_lifetime (duratino): Sets the maximum amount of time ("ms", "s", "m", "h") a connection may be reused.
|
||||
Defaults to 0s (disabled).
|
||||
* sslmode (string): Whether or not to use SSL (default is require)
|
||||
* disable - No SSL
|
||||
* require - Always SSL (skip verification)
|
||||
* verify-ca - Always SSL (verify that the certificate presented by the
|
||||
server was signed by a trusted CA)
|
||||
* verify-full - Always SSL (verify that the certification presented by
|
||||
the server was signed by a trusted CA and the server host name
|
||||
matches the one in the certificate)
|
||||
* fallback_application_name (string): An application_name to fall back to if one isn't provided.
|
||||
* connect_timeout (number): Maximum wait for connection, in seconds. Zero or
|
||||
not specified means wait indefinitely.
|
||||
* sslcert (string): Cert file location. The file must contain PEM encoded data.
|
||||
* sslkey (string): Key file location. The file must contain PEM encoded data.
|
||||
* sslrootcert (string): The location of the root certificate file. The file
|
||||
must contain PEM encoded data.
|
||||
Example: DATABASE_URL=postgres://user:password@host:123/database?sslmode=verify-full
|
||||
|
||||
- MySQL: If DATABASE_URL is a DSN starting with mysql:// MySQL will be used as storage backend.
|
||||
Be aware that the ?parseTime=true parameter is mandatory, or timestamps will not work.
|
||||
Example: DATABASE_URL=mysql://user:password@tcp(host:123)/database?parseTime=true
|
||||
|
||||
Additionally, the following query/DSN parameters are supported:
|
||||
* collation (string): Sets the collation used for client-server interaction on connection. In contrast to charset,
|
||||
collation does not issue additional queries. If the specified collation is unavailable on the target server,
|
||||
the connection will fail.
|
||||
* loc (string): Sets the location for time.Time values. Note that this sets the location for time.Time values
|
||||
but does not change MySQL's time_zone setting. For that set the time_zone DSN parameter. Please keep in mind,
|
||||
that param values must be url.QueryEscape'ed. Alternatively you can manually replace the / with %2F.
|
||||
For example US/Pacific would be loc=US%2FPacific.
|
||||
* maxAllowedPacket (number): Max packet size allowed in bytes. The default value is 4 MiB and should be
|
||||
adjusted to match the server settings. maxAllowedPacket=0 can be used to automatically fetch the max_allowed_packet variable from server on every connection.
|
||||
* readTimeout (duration): I/O read timeout. The value must be a decimal number with a unit suffix
|
||||
("ms", "s", "m", "h"), such as "30s", "0.5m" or "1m30s".
|
||||
* timeout (duration): Timeout for establishing connections, aka dial timeout. The value must be a decimal number with a unit suffix
|
||||
("ms", "s", "m", "h"), such as "30s", "0.5m" or "1m30s".
|
||||
* tls (bool / string): tls=true enables TLS / SSL encrypted connection to the server. Use skip-verify if
|
||||
you want to use a self-signed or invalid certificate (server side).
|
||||
* writeTimeout (duration): I/O write timeout. The value must be a decimal number with a unit suffix
|
||||
("ms", "s", "m", "h"), such as "30s", "0.5m" or "1m30s".
|
||||
Example: DATABASE_URL=mysql://user:password@tcp(host:123)/database?parseTime=true&writeTimeout=123s
|
||||
|
||||
- CockroachDB: If DATABASE_URL is a DSN starting with cockroach://, CockroachDB will be used as storage backend.
|
||||
Example: DATABASE_URL=cockroach://user:password@host:123/database
|
||||
|
||||
Additionally, the following query/DSN parameters are supported:
|
||||
* sslmode (string): Whether or not to use SSL (default is require)
|
||||
* disable - No SSL
|
||||
* require - Always SSL (skip verification)
|
||||
* verify-ca - Always SSL (verify that the certificate presented by the
|
||||
server was signed by a trusted CA)
|
||||
* verify-full - Always SSL (verify that the certification presented by
|
||||
the server was signed by a trusted CA and the server host name
|
||||
matches the one in the certificate)
|
||||
* application_name (string): An initial value for the application_name session variable.
|
||||
* sslcert (string): Cert file location. The file must contain PEM encoded data.
|
||||
* sslkey (string): Key file location. The file must contain PEM encoded data.
|
||||
* sslrootcert (string): The location of the root certificate file. The file
|
||||
must contain PEM encoded data.
|
||||
Example: DATABASE_URL=cockroach://user:password@host:123/database?sslmode=verify-full`
|
||||
}
|
||||
118
oryx/sqlcon/parse_opts.go
Normal file
118
oryx/sqlcon/parse_opts.go
Normal file
@@ -0,0 +1,118 @@
|
||||
// Copyright © 2023 Ory Corp
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package sqlcon
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ory/x/logrusx"
|
||||
)
|
||||
|
||||
// ParseConnectionOptions parses values for max_conns, max_idle_conns, max_conn_lifetime from DSNs.
|
||||
// It also returns the URI without those query parameters.
|
||||
func ParseConnectionOptions(l *logrusx.Logger, dsn string) (maxConns int, maxIdleConns int, maxConnLifetime, maxIdleConnTime time.Duration, cleanedDSN string) {
|
||||
maxConns = maxParallelism() * 2
|
||||
maxIdleConns = maxParallelism()
|
||||
cleanedDSN = dsn
|
||||
|
||||
hostPath, rawQuery, ok := strings.Cut(dsn, "?")
|
||||
if !ok {
|
||||
l.
|
||||
WithField("sql_max_connections", maxConns).
|
||||
WithField("sql_max_idle_connections", maxIdleConns).
|
||||
WithField("sql_max_connection_lifetime", maxConnLifetime).
|
||||
WithField("sql_max_idle_connection_time", maxIdleConnTime).
|
||||
Debugf("No SQL connection options have been defined, falling back to default connection options.")
|
||||
return
|
||||
}
|
||||
|
||||
query, err := url.ParseQuery(rawQuery)
|
||||
if err != nil {
|
||||
l.
|
||||
WithField("sql_max_connections", maxConns).
|
||||
WithField("sql_max_idle_connections", maxIdleConns).
|
||||
WithField("sql_max_connection_lifetime", maxConnLifetime).
|
||||
WithField("sql_max_idle_connection_time", maxIdleConnTime).
|
||||
WithError(err).
|
||||
Warnf("Unable to parse SQL DSN query, falling back to default connection options.")
|
||||
return
|
||||
}
|
||||
|
||||
if v := query.Get("max_conns"); v != "" {
|
||||
s, err := strconv.ParseInt(v, 10, 64)
|
||||
if err != nil {
|
||||
l.WithError(err).Warnf(`SQL DSN query parameter "max_conns" value %v could not be parsed to int, falling back to default value %d`, v, maxConns)
|
||||
} else {
|
||||
maxConns = int(s)
|
||||
}
|
||||
query.Del("max_conns")
|
||||
}
|
||||
|
||||
if v := query.Get("max_idle_conns"); v != "" {
|
||||
s, err := strconv.ParseInt(v, 10, 64)
|
||||
if err != nil {
|
||||
l.WithError(err).Warnf(`SQL DSN query parameter "max_idle_conns" value %v could not be parsed to int, falling back to default value %d`, v, maxIdleConns)
|
||||
} else {
|
||||
maxIdleConns = int(s)
|
||||
}
|
||||
query.Del("max_idle_conns")
|
||||
}
|
||||
|
||||
if v := query.Get("max_conn_lifetime"); v != "" {
|
||||
s, err := time.ParseDuration(v)
|
||||
if err != nil {
|
||||
l.WithError(err).Warnf(`SQL DSN query parameter "max_conn_lifetime" value %v could not be parsed to duration, falling back to default value %d`, v, maxConnLifetime)
|
||||
} else {
|
||||
maxConnLifetime = s
|
||||
}
|
||||
query.Del("max_conn_lifetime")
|
||||
}
|
||||
|
||||
if v := query.Get("max_conn_idle_time"); v != "" {
|
||||
s, err := time.ParseDuration(v)
|
||||
if err != nil {
|
||||
l.WithError(err).Warnf(`SQL DSN query parameter "max_conn_idle_time" value %v could not be parsed to duration, falling back to default value %d`, v, maxIdleConnTime)
|
||||
} else {
|
||||
maxIdleConnTime = s
|
||||
}
|
||||
query.Del("max_conn_idle_time")
|
||||
}
|
||||
cleanedDSN = fmt.Sprintf("%s?%s", hostPath, query.Encode())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// FinalizeDSN will return a finalized DSN URI.
|
||||
func FinalizeDSN(l *logrusx.Logger, dsn string) string {
|
||||
if !strings.HasPrefix(dsn, "mysql://") {
|
||||
return dsn
|
||||
}
|
||||
|
||||
var q url.Values
|
||||
hostPath, query, ok := strings.Cut(dsn, "?")
|
||||
|
||||
if !ok {
|
||||
q = make(url.Values)
|
||||
} else {
|
||||
var err error
|
||||
q, err = url.ParseQuery(query)
|
||||
if err != nil {
|
||||
l.WithError(err).Warnf("Unable to parse SQL DSN query, could not finalize the DSN URI.")
|
||||
return dsn
|
||||
}
|
||||
}
|
||||
|
||||
q.Set("multiStatements", "true")
|
||||
q.Set("parseTime", "true")
|
||||
|
||||
// This causes an UPDATE to return the number of matching rows instead of
|
||||
// the number of rows changed. This ensures compatibility with PostgreSQL and SQLite behavior.
|
||||
q.Set("clientFoundRows", "true")
|
||||
|
||||
return fmt.Sprintf("%s?%s", hostPath, q.Encode())
|
||||
}
|
||||
Reference in New Issue
Block a user