snapshot of https://github.com/centrifugal/centrifugo.git (master, 2026-09-16; blobs > 131072B omitted)
This commit is contained in:
31
internal/pgstreambroker/config_test.go
Normal file
31
internal/pgstreambroker/config_test.go
Normal file
@@ -0,0 +1,31 @@
|
||||
package pgstreambroker
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/centrifugal/centrifuge"
|
||||
)
|
||||
|
||||
// A table prefix reaches DDL unquoted, so one PostgreSQL can't parse has to be
|
||||
// caught at construction — otherwise it surfaces much later as a syntax error
|
||||
// naming a generated table rather than the setting behind it.
|
||||
func TestNewPostgresStreamBroker_RejectsBadTablePrefix(t *testing.T) {
|
||||
node, err := centrifuge.New(centrifuge.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("centrifuge.New: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = node.Shutdown(t.Context()) })
|
||||
|
||||
// Rejected before any connection is attempted, so this needs no Postgres.
|
||||
_, err = NewPostgresStreamBroker(node, PostgresStreamBrokerConfig{
|
||||
DSN: "postgres://test:test@localhost:5432/test",
|
||||
TablePrefix: "cf-prod",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for a prefix PostgreSQL cannot parse unquoted")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "table_prefix") {
|
||||
t.Fatalf("error should name the setting at fault, got: %v", err)
|
||||
}
|
||||
}
|
||||
243
internal/pgstreambroker/ensure_schema.go
Normal file
243
internal/pgstreambroker/ensure_schema.go
Normal file
@@ -0,0 +1,243 @@
|
||||
package pgstreambroker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/centrifugal/centrifugo/v6/internal/pgschema"
|
||||
)
|
||||
|
||||
// reconcileShardLock ensures the shard_lock table (for both variants) has
|
||||
// exactly one row for each shard in [0, NumShards). Called on every
|
||||
// EnsureSchema including the fast path so a NumShards change between runs
|
||||
// is picked up. A stale shard_lock is load-bearing, not cosmetic: a missing
|
||||
// row makes `PERFORM 1 FROM shard_lock WHERE shard_id = X FOR UPDATE`
|
||||
// lock nothing, breaking per-shard publish serialization, which in turn
|
||||
// lets stream IDs commit out of order and the outbox cursor skip rows.
|
||||
// bothVariantsPresent reports whether the primary table exists for both the
|
||||
// jsonb and binary variants, so EnsureSchema's fast path doesn't trust a partial
|
||||
// install where only one variant was created.
|
||||
func (e *PostgresStreamBroker) bothVariantsPresent(ctx context.Context) bool {
|
||||
for _, prefix := range []string{e.names.jsonbPrefix, e.names.binaryPrefix} {
|
||||
table := strings.TrimRight(prefix, "_")
|
||||
if _, err := e.pool.Exec(ctx, fmt.Sprintf(`SELECT 1 FROM %s LIMIT 0`, table)); err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (e *PostgresStreamBroker) reconcileShardLock(ctx context.Context) error {
|
||||
for _, prefix := range []string{e.names.jsonbPrefix, e.names.binaryPrefix} {
|
||||
shardLock := prefix + "shard_lock"
|
||||
if _, err := e.pool.Exec(ctx, fmt.Sprintf(
|
||||
`INSERT INTO %s (shard_id) SELECT generate_series(0, $1 - 1) ON CONFLICT DO NOTHING`,
|
||||
shardLock), e.conf.NumShards); err != nil {
|
||||
return &SchemaError{
|
||||
Object: SchemaObject{Type: "table", Name: shardLock},
|
||||
Op: "create",
|
||||
Err: fmt.Errorf("populate shard_lock: %w", err),
|
||||
}
|
||||
}
|
||||
if _, err := e.pool.Exec(ctx, fmt.Sprintf(
|
||||
`DELETE FROM %s WHERE shard_id >= $1`,
|
||||
shardLock), e.conf.NumShards); err != nil {
|
||||
return &SchemaError{
|
||||
Object: SchemaObject{Type: "table", Name: shardLock},
|
||||
Op: "create",
|
||||
Err: fmt.Errorf("trim shard_lock: %w", err),
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// versionTables returns the per-variant schema_version table names — used as
|
||||
// the targets for every UPDATE schema_version performed during the schema
|
||||
// lifecycle (specifically the final SetSchemaVersion after DDL).
|
||||
func (e *PostgresStreamBroker) versionTables() []string {
|
||||
return []string{
|
||||
e.names.jsonbPrefix + "schema_version",
|
||||
e.names.binaryPrefix + "schema_version",
|
||||
}
|
||||
}
|
||||
|
||||
// migrationVariants renders `template` once per variant (jsonb + binary) and
|
||||
// pairs each rendered SQL with its variant's schema_version table.
|
||||
func (e *PostgresStreamBroker) migrationVariants(template string) []pgschema.MigrationVariant {
|
||||
return []pgschema.MigrationVariant{
|
||||
{
|
||||
SQL: renderSchemaTemplate(template, e.names.jsonbPrefix, false),
|
||||
VersionTable: e.names.jsonbPrefix + "schema_version",
|
||||
},
|
||||
{
|
||||
SQL: renderSchemaTemplate(template, e.names.binaryPrefix, true),
|
||||
VersionTable: e.names.binaryPrefix + "schema_version",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// runMigrationsUnderLock — see pgmapbroker.runMigrationsUnderLock; identical
|
||||
// shape and rationale, scoped to this broker's prefix tables.
|
||||
func (e *PostgresStreamBroker) runMigrationsUnderLock(ctx context.Context, label string) error {
|
||||
release, err := pgschema.AcquireMigrationLock(ctx, e.pool, label)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
|
||||
dbVersion, isFresh, err := pgschema.ReadSchemaVersion(ctx, e.pool, e.names.schemaVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isFresh {
|
||||
return nil
|
||||
}
|
||||
if err := pgschema.CheckDowngrade(label, dbVersion, schemaVersion); err != nil {
|
||||
return err
|
||||
}
|
||||
for v := dbVersion + 1; v <= schemaVersion; v++ {
|
||||
sql, ok := schemaMigrations[v]
|
||||
if !ok {
|
||||
return fmt.Errorf("%s: missing schemaMigrations[%d] at runtime (schemaVersion=%d)", label, v, schemaVersion)
|
||||
}
|
||||
if err := pgschema.ApplyMigrationInTx(ctx, e.pool, label, v, e.migrationVariants(sql)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EnsureSchema creates all required database objects idempotently.
|
||||
//
|
||||
// Flow:
|
||||
// 1. Read schema_version with error discrimination (transient errors must
|
||||
// propagate — see pgschema.ReadSchemaVersion).
|
||||
// 2. Fast path: version matches AND probe ok AND partitioned shape verified
|
||||
// → reconcile shard_lock + lookahead partitions and return.
|
||||
// 3. Reject downgrade.
|
||||
// 4. Run pending migrations BEFORE DDL, each in its own transaction with
|
||||
// atomic schema_version bump.
|
||||
// 5. Render and run DDL for both variants (CREATE OR REPLACE picks up the
|
||||
// latest function bodies; CREATE TABLE IF NOT EXISTS is a no-op when
|
||||
// migrations have already brought the shape current).
|
||||
// 6. Reconcile shard_lock; verify partitioned shape; pre-create partitions.
|
||||
// 7. Final schema_version write — fatal on failure.
|
||||
func (e *PostgresStreamBroker) EnsureSchema(ctx context.Context) error {
|
||||
const label = "pgstreambroker"
|
||||
|
||||
dbVersion, isFresh, err := pgschema.ReadSchemaVersion(ctx, e.pool, e.names.schemaVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Fast path: version current, both variants' history tables exist, partition
|
||||
// shape ok. Probe BOTH variants (not just the active one): the two variants
|
||||
// are created by separate Exec calls, so a partial fresh install can commit
|
||||
// one and fail the other transiently. Trusting only the active variant would
|
||||
// take the fast path and then fail in reconcileShardLock (which touches both)
|
||||
// on the missing tables, before reaching the idempotent DDL that recreates
|
||||
// them — wedging the broker on every restart. If a variant is missing, fall
|
||||
// through and re-apply DDL to self-heal.
|
||||
if !isFresh && dbVersion == schemaVersion {
|
||||
if e.bothVariantsPresent(ctx) {
|
||||
if err := e.verifyPartitionedShape(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := e.reconcileShardLock(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return e.ensureInitialPartitions(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
if !isFresh {
|
||||
if err := pgschema.CheckDowngrade(label, dbVersion, schemaVersion); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Migrations run BEFORE DDL, under a Postgres advisory lock so concurrent
|
||||
// rolling-deploy upgrades can't run the same migration chain twice.
|
||||
// See pgmapbroker.EnsureSchema for the full rationale.
|
||||
if !isFresh {
|
||||
if err := e.runMigrationsUnderLock(ctx, label); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Render and run DDL for both variants.
|
||||
jsonbSQL := renderSchema(e.names.jsonbPrefix, false)
|
||||
binarySQL := renderSchema(e.names.binaryPrefix, true)
|
||||
jsonbDDL, jsonbFuncs := splitSchemaSQL(jsonbSQL)
|
||||
binaryDDL, binaryFuncs := splitSchemaSQL(binarySQL)
|
||||
for _, sql := range []string{jsonbDDL, binaryDDL, jsonbFuncs, binaryFuncs} {
|
||||
if sql == "" {
|
||||
continue
|
||||
}
|
||||
if err := e.execSchemaWithRetry(ctx, sql); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := e.reconcileShardLock(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := e.verifyPartitionedShape(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := e.ensureInitialPartitions(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Final schema_version write — see pgmapbroker for the rationale on
|
||||
// "fatal on failure".
|
||||
return pgschema.SetSchemaVersion(ctx, e.pool, label, schemaVersion, e.versionTables())
|
||||
}
|
||||
|
||||
// verifyPartitionedShape probes pg_partitioned_table for the active variant's
|
||||
// history table. If a non-partitioned table exists (from an earlier build of
|
||||
// pgstreambroker), fail loudly with a clear "drop the legacy tables" message.
|
||||
func (e *PostgresStreamBroker) verifyPartitionedShape(ctx context.Context) error {
|
||||
var isPartitioned bool
|
||||
err := e.pool.QueryRow(ctx, `
|
||||
SELECT EXISTS(
|
||||
SELECT 1 FROM pg_partitioned_table
|
||||
WHERE partrelid = $1::regclass
|
||||
)
|
||||
`, e.names.stream).Scan(&isPartitioned)
|
||||
if err != nil {
|
||||
// Table doesn't exist yet (or other lookup error). Fall through —
|
||||
// the schema template's CREATE TABLE will run later.
|
||||
return nil
|
||||
}
|
||||
if !isPartitioned {
|
||||
return &SchemaError{
|
||||
Object: SchemaObject{Type: "table", Name: e.names.stream},
|
||||
Op: "verify",
|
||||
Err: fmt.Errorf(
|
||||
"%s exists but is not partitioned. pgstreambroker schema requires "+
|
||||
"a partitioned table and this build does not include migration logic. "+
|
||||
"Drop the existing tables manually with: DROP TABLE IF EXISTS %s, %s, %s, %s, %s CASCADE",
|
||||
e.names.stream,
|
||||
e.names.stream, e.names.meta, e.names.idempotency, e.names.shardLock, e.names.schemaVersion,
|
||||
),
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureInitialPartitions pre-creates today's and lookahead partitions
|
||||
// via pgoutbox.Partitioner.EnsureLookaheadPartitions.
|
||||
func (e *PostgresStreamBroker) ensureInitialPartitions(ctx context.Context) error {
|
||||
p := e.newPartitioner()
|
||||
if err := p.EnsureLookaheadPartitions(ctx); err != nil {
|
||||
return &SchemaError{
|
||||
Object: SchemaObject{Type: "table", Name: e.names.stream},
|
||||
Op: "create",
|
||||
Err: err,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
190
internal/pgstreambroker/history.go
Normal file
190
internal/pgstreambroker/history.go
Normal file
@@ -0,0 +1,190 @@
|
||||
package pgstreambroker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/centrifugal/centrifuge"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// History reads publications from a channel's history table, applying the
|
||||
// per-channel TTL filter at read time. Unlike the Redis broker, the PG stream
|
||||
// broker does NOT refresh meta TTL on read — meta TTL is refreshed on Publish
|
||||
// (via the 3-tier MetaTTL fallback). This means channels with active readers
|
||||
// but no publishers will eventually have their meta expire. This matches the
|
||||
// map broker's ReadState/ReadStream behavior, and allows History to use read
|
||||
// replicas for scaling.
|
||||
//
|
||||
// If the channel has no meta row (never published, or meta expired), History
|
||||
// returns an empty result with a zero StreamPosition — matching the map
|
||||
// broker's behavior on ErrNoRows.
|
||||
func (e *PostgresStreamBroker) History(ch string, opts centrifuge.HistoryOptions) ([]*centrifuge.Publication, centrifuge.StreamPosition, error) {
|
||||
ctx := context.Background()
|
||||
pool := e.getReadPool(ch, true) // can use replicas — this is a pure read
|
||||
|
||||
// Read meta — pure SELECT, no UPSERT.
|
||||
metaQuery := fmt.Sprintf(
|
||||
`SELECT top_offset, epoch, history_ttl, history_size FROM %s WHERE channel = $1`,
|
||||
e.names.meta,
|
||||
)
|
||||
|
||||
var topOffset int64
|
||||
var epoch string
|
||||
var historyTTL *time.Duration
|
||||
var historySize *int
|
||||
|
||||
err := pool.QueryRow(ctx, metaQuery, ch).Scan(&topOffset, &epoch, &historyTTL, &historySize)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
// Channel never published or meta expired — return empty, matching
|
||||
// map broker behavior. The centrifuge node treats empty epoch as
|
||||
// "don't check epoch" (epochOK := sinceEpoch == "" || sinceEpoch == streamTop.Epoch).
|
||||
return nil, centrifuge.StreamPosition{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, centrifuge.StreamPosition{}, fmt.Errorf("postgres stream broker: history meta: %w", err)
|
||||
}
|
||||
|
||||
streamPos := centrifuge.StreamPosition{
|
||||
Offset: uint64(topOffset),
|
||||
Epoch: epoch,
|
||||
}
|
||||
|
||||
// Filter.Limit == 0 → caller only wants the position.
|
||||
if opts.Filter.Limit == 0 {
|
||||
return nil, streamPos, nil
|
||||
}
|
||||
|
||||
// Compute effective bounds Go-side.
|
||||
windowStart := int64(1)
|
||||
if historySize != nil && *historySize > 0 {
|
||||
if topOffset >= int64(*historySize) {
|
||||
windowStart = topOffset - int64(*historySize) + 1
|
||||
}
|
||||
}
|
||||
sinceOffset := int64(0)
|
||||
if opts.Filter.Since != nil {
|
||||
sinceOffset = int64(opts.Filter.Since.Offset)
|
||||
}
|
||||
// Compute the offset window [effectiveStart, effectiveEnd]. Forward returns
|
||||
// publications after Since (ascending); reverse returns publications before
|
||||
// Since (descending) - the boundary sits on the opposite side of the cursor,
|
||||
// as in the memory/redis/pgmapbroker brokers. Using Since+1 for both
|
||||
// directions made reverse pagination return already-seen or empty pages and
|
||||
// never reach older messages.
|
||||
effectiveStart := windowStart
|
||||
effectiveEnd := topOffset
|
||||
if opts.Filter.Reverse {
|
||||
// Reverse returns publications strictly before Since (descending).
|
||||
// Guard sinceOffset == 0: centrifuge's Since.Offset is uint64, so its
|
||||
// `since.Offset - 1` underflows to MaxUint64 for offset 0, which means
|
||||
// "read the whole stream from the top". We keep effectiveEnd = topOffset
|
||||
// in that case instead of computing -1 (which would match no rows).
|
||||
if opts.Filter.Since != nil && sinceOffset > 0 {
|
||||
effectiveEnd = sinceOffset - 1
|
||||
}
|
||||
} else {
|
||||
effectiveStart = sinceOffset + 1
|
||||
if windowStart > effectiveStart {
|
||||
effectiveStart = windowStart
|
||||
}
|
||||
}
|
||||
|
||||
effectiveLimit := math.MaxInt32
|
||||
if opts.Filter.Limit > 0 && opts.Filter.Limit < effectiveLimit {
|
||||
effectiveLimit = opts.Filter.Limit
|
||||
}
|
||||
if historySize != nil && *historySize > 0 && *historySize < effectiveLimit {
|
||||
effectiveLimit = *historySize
|
||||
}
|
||||
|
||||
// Read-time TTL filter cutoff.
|
||||
var cutoff time.Time
|
||||
if historyTTL != nil && *historyTTL > 0 {
|
||||
cutoff = time.Now().Add(-*historyTTL)
|
||||
} else {
|
||||
cutoff = time.Unix(0, 0) // sentinel — no filter
|
||||
}
|
||||
|
||||
order := "ASC"
|
||||
if opts.Filter.Reverse {
|
||||
order = "DESC"
|
||||
}
|
||||
|
||||
// The `epoch = $5` predicate rejects rows from prior epochs that linger
|
||||
// between meta TTL expiry and the next partition retention drop. The
|
||||
// time-based `created_at > cutoff` filter usually catches them too, but
|
||||
// fails when history_ttl is increased between the dead epoch's last
|
||||
// publish and the read — the new wider cutoff lets old rows through.
|
||||
// The new (channel, epoch, channel_offset) WHERE kind=0 partial index
|
||||
// resolves the predicate without scanning dead-epoch rows.
|
||||
rowsQuery := fmt.Sprintf(`
|
||||
SELECT channel_offset, epoch, data, tags, client_id, user_id, conn_info, chan_info, prev_data
|
||||
FROM %s
|
||||
WHERE channel = $1
|
||||
AND kind = 0
|
||||
AND channel_offset >= $2
|
||||
AND channel_offset <= $3
|
||||
AND created_at > $4
|
||||
AND epoch = $5
|
||||
ORDER BY channel_offset %s
|
||||
LIMIT $6
|
||||
`, e.names.stream, order)
|
||||
|
||||
rows, err := pool.Query(ctx, rowsQuery, ch, effectiveStart, effectiveEnd, cutoff, epoch, effectiveLimit)
|
||||
if err != nil {
|
||||
return nil, centrifuge.StreamPosition{}, fmt.Errorf("postgres stream broker: history query: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
allocHint := effectiveLimit
|
||||
if allocHint > 1024 {
|
||||
allocHint = 1024
|
||||
}
|
||||
arena := byteArena{buf: make([]byte, 0, allocHint*64)}
|
||||
backing := make([]centrifuge.Publication, 0, allocHint)
|
||||
pubs := make([]*centrifuge.Publication, 0, allocHint)
|
||||
|
||||
var fmts pgColFormats
|
||||
for rows.Next() {
|
||||
if fmts == nil {
|
||||
fmts = pgColFormatsFromRows(rows)
|
||||
}
|
||||
raw := rows.RawValues()
|
||||
backing = append(backing, centrifuge.Publication{})
|
||||
p := &backing[len(backing)-1]
|
||||
p.Offset = pgRawUint64(raw[0], fmts[0])
|
||||
p.Epoch = pgRawString(&arena, raw[1])
|
||||
p.Data = e.rawDataBytes(&arena, raw[2], fmts[2])
|
||||
p.Tags = pgRawJSONBMap(raw[3])
|
||||
if raw[4] != nil {
|
||||
p.Info = ¢rifuge.ClientInfo{
|
||||
ClientID: pgRawString(&arena, raw[4]),
|
||||
UserID: pgRawString(&arena, raw[5]),
|
||||
ConnInfo: e.rawDataBytes(&arena, raw[6], fmts[6]),
|
||||
ChanInfo: e.rawDataBytes(&arena, raw[7], fmts[7]),
|
||||
}
|
||||
}
|
||||
_ = raw[8] // prev_data — used by outbox, not by History
|
||||
pubs = append(pubs, p)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, centrifuge.StreamPosition{}, fmt.Errorf("postgres stream broker: history rows: %w", err)
|
||||
}
|
||||
|
||||
return pubs, streamPos, nil
|
||||
}
|
||||
|
||||
// RemoveHistory wipes publications (kind=0) for a channel via the
|
||||
// __PREFIX__remove_history SQL function.
|
||||
func (e *PostgresStreamBroker) RemoveHistory(ch string) error {
|
||||
ctx := context.Background()
|
||||
query := fmt.Sprintf(`SELECT %s($1, $2)`, e.names.removeHistory)
|
||||
if _, err := e.pool.Exec(ctx, query, ch, e.conf.NumShards); err != nil {
|
||||
return fmt.Errorf("postgres stream broker: remove_history: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
511
internal/pgstreambroker/internal/sql/schema.sql
Normal file
511
internal/pgstreambroker/internal/sql/schema.sql
Normal file
@@ -0,0 +1,511 @@
|
||||
-- ============================================================================
|
||||
-- PostgreSQL StreamBroker Schema and Functions (template).
|
||||
-- Auto-created by EnsureSchema(). All statements are idempotent.
|
||||
-- Placeholders are replaced at runtime by PostgresStreamBroker.renderSchema:
|
||||
-- __DATA_TYPE__ → JSONB (default) or BYTEA (when BinaryData is true)
|
||||
-- __PREFIX__ → <TablePrefix>_stream_ or <TablePrefix>_binary_stream_
|
||||
-- where TablePrefix is user-configurable (default "cf").
|
||||
-- EnsureSchema renders this template once per variant (JSONB + binary) and
|
||||
-- executes both, so a broker always creates both sets of tables regardless
|
||||
-- of the BinaryData runtime flag.
|
||||
-- ============================================================================
|
||||
|
||||
-- ============================================================================
|
||||
-- Stream Table (Publications + Joins/Leaves)
|
||||
--
|
||||
-- The stream table is always partitioned by created_at (daily). The composite
|
||||
-- primary key (id, created_at) is required because Postgres mandates that the
|
||||
-- partition key be part of every unique constraint. Initial partitions
|
||||
-- (today + lookahead) are created via pgoutbox.Partitioner.EnsureLookaheadPartitions
|
||||
-- after this DDL runs; the partition retention worker drops old partitions.
|
||||
--
|
||||
-- The kind column distinguishes:
|
||||
-- 0 = publication (the common case; History() filters WHERE kind = 0)
|
||||
-- 1 = join event (PublishJoin)
|
||||
-- 2 = leave event (PublishLeave)
|
||||
-- Joins and leaves piggyback on partition retention — no special cleanup pass.
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS __STREAM_TABLE__ (
|
||||
id BIGSERIAL,
|
||||
channel TEXT NOT NULL,
|
||||
channel_offset BIGINT NOT NULL,
|
||||
epoch TEXT NOT NULL DEFAULT '',
|
||||
kind SMALLINT NOT NULL DEFAULT 0,
|
||||
data __DATA_TYPE__,
|
||||
tags JSONB,
|
||||
client_id TEXT,
|
||||
user_id TEXT,
|
||||
conn_info __DATA_TYPE__,
|
||||
chan_info __DATA_TYPE__,
|
||||
prev_data __DATA_TYPE__,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
shard_id SMALLINT NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (id, created_at)
|
||||
) PARTITION BY RANGE (created_at);
|
||||
|
||||
-- Publications-only partial index. History() reads filter WHERE kind = 0,
|
||||
-- and the delta prev_data lookup also filters WHERE kind = 0. A partial
|
||||
-- index keeps the index small and the planner happy without scanning over
|
||||
-- join/leave rows. Including epoch lets History() seek directly past
|
||||
-- dead-epoch rows that linger between meta TTL expiry and partition retention.
|
||||
CREATE INDEX IF NOT EXISTS __STREAM_TABLE___channel_epoch_offset_idx
|
||||
ON __STREAM_TABLE__ (channel, epoch, channel_offset)
|
||||
WHERE kind = 0;
|
||||
|
||||
-- Outbox worker batch fetch — polls all kinds (publications, joins, leaves)
|
||||
-- and switches in Go.
|
||||
CREATE INDEX IF NOT EXISTS __STREAM_TABLE___shard_id_idx
|
||||
ON __STREAM_TABLE__ (shard_id, id);
|
||||
|
||||
-- Time-based scans (cleanup, retention, audit). Applies to all kinds.
|
||||
CREATE INDEX IF NOT EXISTS __STREAM_TABLE___created_at_idx
|
||||
ON __STREAM_TABLE__ (created_at);
|
||||
|
||||
|
||||
-- ============================================================================
|
||||
-- Channel Metadata Table
|
||||
--
|
||||
-- Tracks per-channel state needed for publish ordering, recovery, and the
|
||||
-- read-time TTL filter:
|
||||
--
|
||||
-- top_offset — last assigned channel_offset (incremented on publish)
|
||||
-- epoch — monotonic epoch string; new on first publish, persists
|
||||
-- until the channel is forgotten via expires_at
|
||||
-- history_ttl — from PublishOptions.HistoryTTL; refreshed on each publish.
|
||||
-- Used by History() reads for the read-time TTL filter.
|
||||
-- history_size — from PublishOptions.HistorySize; refreshed on each publish.
|
||||
-- Used by History() reads to clamp the result window.
|
||||
-- top_version — highest PublishOptions.Version accepted under
|
||||
-- top_version_epoch. An incoming publish with
|
||||
-- version<=top_version under the same epoch is
|
||||
-- suppressed with SuppressReasonVersion. Zero means
|
||||
-- "no versioned publish has been seen yet", so the
|
||||
-- first versioned publish always establishes the
|
||||
-- baseline without being compared.
|
||||
-- top_version_epoch — epoch string associated with top_version. When the
|
||||
-- caller supplies a different VersionEpoch the version
|
||||
-- comparison is reset — the new epoch's first publish
|
||||
-- becomes the fresh baseline even if its numeric
|
||||
-- version is lower. A non-versioned publish never
|
||||
-- clobbers these two fields.
|
||||
-- expires_at — from PublishOptions.HistoryMetaTTL (with 3-tier
|
||||
-- fallback); always non-NULL; when it passes, the
|
||||
-- meta cleanup pass deletes the row and the channel
|
||||
-- is forgotten.
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS __PREFIX__meta (
|
||||
channel TEXT PRIMARY KEY,
|
||||
top_offset BIGINT NOT NULL DEFAULT 0,
|
||||
epoch TEXT NOT NULL DEFAULT '',
|
||||
history_ttl INTERVAL NOT NULL DEFAULT interval '1 minute',
|
||||
history_size INTEGER NOT NULL DEFAULT 100,
|
||||
top_version BIGINT NOT NULL DEFAULT 0,
|
||||
top_version_epoch TEXT NOT NULL DEFAULT '',
|
||||
expires_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + interval '7 days',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS __PREFIX__meta_expires_idx
|
||||
ON __PREFIX__meta (expires_at);
|
||||
|
||||
-- Per-table autovacuum tuning: meta sees two writes per publish (lock + increment)
|
||||
-- plus one write per History() read (TTL refresh). Default autovacuum thresholds
|
||||
-- can lag behind dead-tuple production for high-traffic channels.
|
||||
ALTER TABLE __PREFIX__meta SET (
|
||||
autovacuum_vacuum_scale_factor = 0.05,
|
||||
autovacuum_vacuum_threshold = 100,
|
||||
autovacuum_analyze_scale_factor = 0.05
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- Idempotency Table
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS __PREFIX__idempotency (
|
||||
channel TEXT NOT NULL,
|
||||
idempotency_key TEXT NOT NULL,
|
||||
result_offset BIGINT NOT NULL,
|
||||
-- result_epoch captures the meta epoch at the time the original publish
|
||||
-- committed. Returned on idempotent replay so callers reading history
|
||||
-- from the replied position can find it — otherwise, if the meta row had
|
||||
-- been TTL-evicted and recreated under a fresh random epoch by the time
|
||||
-- of the replay, the suppressed reply would return the live (new) epoch
|
||||
-- and the caller's recovery would fail to locate the publication.
|
||||
result_epoch TEXT NOT NULL,
|
||||
result_id BIGINT NOT NULL,
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
PRIMARY KEY (channel, idempotency_key)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS __PREFIX__idempotency_expires_idx
|
||||
ON __PREFIX__idempotency (expires_at);
|
||||
|
||||
ALTER TABLE __PREFIX__idempotency SET (
|
||||
autovacuum_vacuum_scale_factor = 0.05,
|
||||
autovacuum_vacuum_threshold = 100,
|
||||
autovacuum_analyze_scale_factor = 0.05
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- Shard Lock Table
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS __PREFIX__shard_lock (
|
||||
shard_id SMALLINT PRIMARY KEY
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- Schema Version
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS __PREFIX__schema_version (
|
||||
id INTEGER PRIMARY KEY,
|
||||
schema_version INTEGER NOT NULL
|
||||
);
|
||||
INSERT INTO __PREFIX__schema_version (id, schema_version) VALUES (1, 1)
|
||||
ON CONFLICT (id) DO NOTHING;
|
||||
|
||||
-- ============================================================================
|
||||
-- Functions
|
||||
-- ============================================================================
|
||||
|
||||
-- __PREFIX__publish: atomic publish with idempotency and TTL handling.
|
||||
--
|
||||
-- Lock order: shard → meta. Publishes on the same shard serialize via the
|
||||
-- shard_lock row, ensuring BIGSERIAL id assignment matches commit order
|
||||
-- within each shard (the invariant the outbox worker relies on).
|
||||
--
|
||||
-- Defensive clamp: effective_meta_ttl = GREATEST(p_meta_ttl, p_history_ttl)
|
||||
-- so the meta row always survives at least as long as the recoverable history,
|
||||
-- preventing the foot-gun of metaTTL < historyTTL.
|
||||
--
|
||||
-- Suppression: idempotency key match returns early without incrementing
|
||||
-- top_offset (the publish is treated as a no-op except for the returned
|
||||
-- position).
|
||||
--
|
||||
-- The function uses INSERT ON CONFLICT DO UPDATE with a no-op write to acquire
|
||||
-- the meta row lock atomically — the DO NOTHING + SELECT FOR UPDATE pattern
|
||||
-- has a race window where concurrent cleanup could delete the row between
|
||||
-- statements.
|
||||
CREATE OR REPLACE FUNCTION __PREFIX__publish(
|
||||
p_channel TEXT,
|
||||
p_data __DATA_TYPE__,
|
||||
p_tags JSONB DEFAULT NULL,
|
||||
p_history_ttl INTERVAL DEFAULT NULL,
|
||||
p_history_size INTEGER DEFAULT NULL,
|
||||
p_meta_ttl INTERVAL DEFAULT NULL,
|
||||
p_version BIGINT DEFAULT NULL,
|
||||
p_version_epoch TEXT DEFAULT NULL,
|
||||
p_idempotency_key TEXT DEFAULT NULL,
|
||||
p_idempotency_ttl INTERVAL DEFAULT NULL,
|
||||
p_use_delta BOOLEAN DEFAULT FALSE,
|
||||
p_client_id TEXT DEFAULT NULL,
|
||||
p_user_id TEXT DEFAULT NULL,
|
||||
p_conn_info __DATA_TYPE__ DEFAULT NULL,
|
||||
p_chan_info __DATA_TYPE__ DEFAULT NULL,
|
||||
p_num_shards INTEGER DEFAULT NULL
|
||||
) RETURNS TABLE(
|
||||
out_result_id BIGINT,
|
||||
out_channel_offset BIGINT,
|
||||
out_epoch TEXT,
|
||||
out_suppressed BOOLEAN,
|
||||
out_suppress_reason TEXT
|
||||
) AS $$
|
||||
DECLARE
|
||||
v_offset BIGINT;
|
||||
v_id BIGINT;
|
||||
v_epoch TEXT;
|
||||
v_prev_data __DATA_TYPE__;
|
||||
v_shard_id INTEGER;
|
||||
v_effective_meta_ttl INTERVAL;
|
||||
v_cached_offset BIGINT;
|
||||
v_cached_epoch TEXT;
|
||||
v_current_version BIGINT;
|
||||
v_current_version_epoch TEXT;
|
||||
BEGIN
|
||||
-- Auto-derive num_shards from shard_lock table when not provided.
|
||||
IF p_num_shards IS NULL THEN
|
||||
SELECT COUNT(*)::INTEGER INTO p_num_shards FROM __PREFIX__shard_lock;
|
||||
END IF;
|
||||
IF p_num_shards <= 0 THEN
|
||||
RAISE EXCEPTION 'shard_lock table is empty — run EnsureSchema first';
|
||||
END IF;
|
||||
|
||||
v_shard_id := abs(hashtext(p_channel)::bigint) % p_num_shards;
|
||||
|
||||
-- Per-shard serialization lock (lock order: shard → meta).
|
||||
PERFORM 1 FROM __PREFIX__shard_lock WHERE shard_id = v_shard_id FOR UPDATE;
|
||||
|
||||
-- Defensive clamp: effective_meta_ttl = GREATEST(p_meta_ttl, p_history_ttl).
|
||||
-- Falls back to 7 days when neither is provided, ensuring meta always expires.
|
||||
-- A negative p_meta_ttl (e.g. interval '-1 second') disables expiration.
|
||||
v_effective_meta_ttl := GREATEST(
|
||||
COALESCE(p_meta_ttl, '0'::interval),
|
||||
COALESCE(p_history_ttl, '0'::interval)
|
||||
);
|
||||
IF v_effective_meta_ttl = '0'::interval THEN
|
||||
v_effective_meta_ttl := interval '7 days';
|
||||
END IF;
|
||||
|
||||
-- Atomic insert-or-lock meta. ON CONFLICT DO UPDATE with a no-op write
|
||||
-- acquires the row lock and returns the pre-update state in a single
|
||||
-- statement, avoiding the DO NOTHING + SELECT FOR UPDATE race.
|
||||
INSERT INTO __PREFIX__meta AS m (channel, top_offset, epoch, updated_at)
|
||||
VALUES (p_channel, 0, substr(md5(random()::text || random()::text), 1, 8), NOW())
|
||||
ON CONFLICT (channel) DO UPDATE
|
||||
SET updated_at = m.updated_at
|
||||
RETURNING m.top_offset, m.epoch, m.top_version, m.top_version_epoch
|
||||
INTO v_offset, v_epoch, v_current_version, v_current_version_epoch;
|
||||
|
||||
-- Idempotency check. Return the epoch the publication was committed
|
||||
-- under (stored in the idempotency row), NOT the current meta epoch —
|
||||
-- those can differ if meta was TTL-evicted and recreated between the
|
||||
-- original publish and this replay. Mirror of the Redis broker's
|
||||
-- broker_history_add_stream.lua, which stores both epoch and offset
|
||||
-- on the result key.
|
||||
IF p_idempotency_key IS NOT NULL THEN
|
||||
SELECT result_offset, result_epoch INTO v_cached_offset, v_cached_epoch
|
||||
FROM __PREFIX__idempotency
|
||||
WHERE channel = p_channel AND idempotency_key = p_idempotency_key
|
||||
AND expires_at > NOW();
|
||||
IF FOUND THEN
|
||||
RETURN QUERY SELECT NULL::BIGINT, v_cached_offset, v_cached_epoch, TRUE, 'idempotency'::TEXT;
|
||||
RETURN;
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
-- Per-stream version dedup. Suppress when the caller supplied a version
|
||||
-- that isn't strictly greater than the one already stored under the same
|
||||
-- epoch. An empty p_version_epoch matches any stored epoch (callers that
|
||||
-- don't care about epoch boundaries); a non-empty p_version_epoch that
|
||||
-- differs from the stored one resets the comparison (new epoch = fresh
|
||||
-- baseline, not suppressed even with lower numeric version).
|
||||
IF p_version IS NOT NULL AND p_version > 0 AND v_current_version > 0 THEN
|
||||
IF (p_version_epoch IS NULL OR p_version_epoch = '' OR p_version_epoch = v_current_version_epoch)
|
||||
AND p_version <= v_current_version THEN
|
||||
RETURN QUERY SELECT NULL::BIGINT, v_offset, v_epoch, TRUE, 'version'::TEXT;
|
||||
RETURN;
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
-- Delta lookup: fetch the most recent publication's data for this channel.
|
||||
-- Filter by epoch so a stale dead-epoch row with a higher channel_offset
|
||||
-- (e.g. old epoch was at 1000, new epoch just started at 1) doesn't get
|
||||
-- picked as the "previous" publication. Order by channel_offset (not id)
|
||||
-- so the partial index can serve it.
|
||||
IF p_use_delta THEN
|
||||
SELECT data INTO v_prev_data
|
||||
FROM __STREAM_TABLE__
|
||||
WHERE channel = p_channel AND epoch = v_epoch AND kind = 0
|
||||
ORDER BY channel_offset DESC LIMIT 1;
|
||||
END IF;
|
||||
|
||||
-- Increment offset + refresh meta TTLs and HistorySize. The COALESCE
|
||||
-- pattern preserves existing values when the parameter is NULL. Version
|
||||
-- fields advance only when the caller supplied a version: a bare publish
|
||||
-- must not clobber the stored version baseline (otherwise a non-versioned
|
||||
-- publish after versioned ones would erase dedup protection).
|
||||
UPDATE __PREFIX__meta AS m
|
||||
SET top_offset = m.top_offset + 1,
|
||||
history_ttl = COALESCE(p_history_ttl, m.history_ttl),
|
||||
history_size = COALESCE(p_history_size, m.history_size),
|
||||
top_version = CASE
|
||||
WHEN p_version IS NOT NULL AND p_version > 0 THEN p_version
|
||||
ELSE m.top_version
|
||||
END,
|
||||
top_version_epoch = CASE
|
||||
WHEN p_version IS NOT NULL AND p_version > 0
|
||||
THEN COALESCE(p_version_epoch, m.top_version_epoch)
|
||||
ELSE m.top_version_epoch
|
||||
END,
|
||||
expires_at = CASE
|
||||
WHEN v_effective_meta_ttl < '0'::interval THEN 'infinity'::timestamptz
|
||||
ELSE NOW() + v_effective_meta_ttl
|
||||
END,
|
||||
updated_at = NOW()
|
||||
WHERE m.channel = p_channel
|
||||
RETURNING m.top_offset INTO v_offset;
|
||||
|
||||
-- Insert the stream row. created_at defaults to NOW() so partition
|
||||
-- routing picks today's partition automatically.
|
||||
INSERT INTO __STREAM_TABLE__ (
|
||||
channel, channel_offset, epoch, kind, data, tags,
|
||||
client_id, user_id, conn_info, chan_info, prev_data, shard_id
|
||||
) VALUES (
|
||||
p_channel, v_offset, v_epoch, 0, p_data, p_tags,
|
||||
p_client_id, p_user_id, p_conn_info, p_chan_info, v_prev_data, v_shard_id
|
||||
) RETURNING id INTO v_id;
|
||||
|
||||
-- Save idempotency result for replay. Persist the epoch alongside the
|
||||
-- offset so replays return the (offset, epoch) tuple this publish
|
||||
-- committed under, not whatever epoch happens to live in meta at replay
|
||||
-- time. See the idempotency check above for the failure mode this
|
||||
-- prevents.
|
||||
IF p_idempotency_key IS NOT NULL THEN
|
||||
INSERT INTO __PREFIX__idempotency (channel, idempotency_key, result_offset, result_epoch, result_id, expires_at)
|
||||
VALUES (p_channel, p_idempotency_key, v_offset, v_epoch, v_id, NOW() + COALESCE(p_idempotency_ttl, INTERVAL '5 minutes'))
|
||||
ON CONFLICT (channel, idempotency_key) DO NOTHING;
|
||||
END IF;
|
||||
|
||||
-- Wake up notification listeners.
|
||||
PERFORM pg_notify('__PREFIX__notify', '');
|
||||
|
||||
RETURN QUERY SELECT v_id, v_offset, v_epoch, FALSE, NULL::TEXT;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- __PREFIX__publish_strict: identical to publish but raises on suppression.
|
||||
-- Used by callers that want hard errors instead of suppressed-result handling.
|
||||
CREATE OR REPLACE FUNCTION __PREFIX__publish_strict(
|
||||
p_channel TEXT,
|
||||
p_data __DATA_TYPE__,
|
||||
p_tags JSONB DEFAULT NULL,
|
||||
p_history_ttl INTERVAL DEFAULT NULL,
|
||||
p_history_size INTEGER DEFAULT NULL,
|
||||
p_meta_ttl INTERVAL DEFAULT NULL,
|
||||
p_version BIGINT DEFAULT NULL,
|
||||
p_version_epoch TEXT DEFAULT NULL,
|
||||
p_idempotency_key TEXT DEFAULT NULL,
|
||||
p_idempotency_ttl INTERVAL DEFAULT NULL,
|
||||
p_use_delta BOOLEAN DEFAULT FALSE,
|
||||
p_client_id TEXT DEFAULT NULL,
|
||||
p_user_id TEXT DEFAULT NULL,
|
||||
p_conn_info __DATA_TYPE__ DEFAULT NULL,
|
||||
p_chan_info __DATA_TYPE__ DEFAULT NULL,
|
||||
p_num_shards INTEGER DEFAULT NULL
|
||||
) RETURNS TABLE(
|
||||
out_result_id BIGINT,
|
||||
out_channel_offset BIGINT,
|
||||
out_epoch TEXT
|
||||
) AS $$
|
||||
DECLARE
|
||||
v_result RECORD;
|
||||
BEGIN
|
||||
SELECT * INTO v_result FROM __PREFIX__publish(
|
||||
p_channel, p_data, p_tags,
|
||||
p_history_ttl, p_history_size, p_meta_ttl,
|
||||
p_version, p_version_epoch,
|
||||
p_idempotency_key, p_idempotency_ttl,
|
||||
p_use_delta, p_client_id, p_user_id, p_conn_info, p_chan_info,
|
||||
p_num_shards
|
||||
);
|
||||
IF v_result.out_suppressed THEN
|
||||
RAISE EXCEPTION 'publish suppressed: %', v_result.out_suppress_reason;
|
||||
END IF;
|
||||
RETURN QUERY SELECT v_result.out_result_id, v_result.out_channel_offset, v_result.out_epoch;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- __PREFIX__publish_join: insert a join (kind=1) row, no offset increment,
|
||||
-- no meta update. Acquires the shard lock for outbox id-ordering correctness.
|
||||
CREATE OR REPLACE FUNCTION __PREFIX__publish_join(
|
||||
p_channel TEXT,
|
||||
p_client_id TEXT,
|
||||
p_user_id TEXT,
|
||||
p_conn_info __DATA_TYPE__ DEFAULT NULL,
|
||||
p_chan_info __DATA_TYPE__ DEFAULT NULL,
|
||||
p_num_shards INTEGER DEFAULT NULL
|
||||
) RETURNS BIGINT AS $$
|
||||
DECLARE
|
||||
v_id BIGINT;
|
||||
v_shard_id INTEGER;
|
||||
BEGIN
|
||||
IF p_num_shards IS NULL THEN
|
||||
SELECT COUNT(*)::INTEGER INTO p_num_shards FROM __PREFIX__shard_lock;
|
||||
END IF;
|
||||
v_shard_id := abs(hashtext(p_channel)::bigint) % p_num_shards;
|
||||
|
||||
-- Acquire shard lock so this insert serializes with publications on the
|
||||
-- same shard. Without this, the join's id could be committed before an
|
||||
-- in-progress publication's id, causing the outbox worker to advance
|
||||
-- past the in-progress row.
|
||||
PERFORM 1 FROM __PREFIX__shard_lock WHERE shard_id = v_shard_id FOR UPDATE;
|
||||
|
||||
INSERT INTO __STREAM_TABLE__ (
|
||||
channel, channel_offset, kind, client_id, user_id, conn_info, chan_info, shard_id
|
||||
) VALUES (
|
||||
p_channel, 0, 1, p_client_id, p_user_id, p_conn_info, p_chan_info, v_shard_id
|
||||
) RETURNING id INTO v_id;
|
||||
|
||||
PERFORM pg_notify('__PREFIX__notify', '');
|
||||
RETURN v_id;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- __PREFIX__publish_leave: insert a leave (kind=2) row. Same shape as publish_join.
|
||||
CREATE OR REPLACE FUNCTION __PREFIX__publish_leave(
|
||||
p_channel TEXT,
|
||||
p_client_id TEXT,
|
||||
p_user_id TEXT,
|
||||
p_conn_info __DATA_TYPE__ DEFAULT NULL,
|
||||
p_chan_info __DATA_TYPE__ DEFAULT NULL,
|
||||
p_num_shards INTEGER DEFAULT NULL
|
||||
) RETURNS BIGINT AS $$
|
||||
DECLARE
|
||||
v_id BIGINT;
|
||||
v_shard_id INTEGER;
|
||||
BEGIN
|
||||
IF p_num_shards IS NULL THEN
|
||||
SELECT COUNT(*)::INTEGER INTO p_num_shards FROM __PREFIX__shard_lock;
|
||||
END IF;
|
||||
v_shard_id := abs(hashtext(p_channel)::bigint) % p_num_shards;
|
||||
|
||||
PERFORM 1 FROM __PREFIX__shard_lock WHERE shard_id = v_shard_id FOR UPDATE;
|
||||
|
||||
INSERT INTO __STREAM_TABLE__ (
|
||||
channel, channel_offset, kind, client_id, user_id, conn_info, chan_info, shard_id
|
||||
) VALUES (
|
||||
p_channel, 0, 2, p_client_id, p_user_id, p_conn_info, p_chan_info, v_shard_id
|
||||
) RETURNING id INTO v_id;
|
||||
|
||||
PERFORM pg_notify('__PREFIX__notify', '');
|
||||
RETURN v_id;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- __PREFIX__remove_history: wipes publications (kind=0) for a channel.
|
||||
-- Acquires the shard lock so that any in-progress publish either commits
|
||||
-- before the DELETE or runs after — no in-flight publication can survive
|
||||
-- the wipe semantically.
|
||||
CREATE OR REPLACE FUNCTION __PREFIX__remove_history(
|
||||
p_channel TEXT,
|
||||
p_num_shards INTEGER DEFAULT NULL
|
||||
) RETURNS VOID AS $$
|
||||
DECLARE
|
||||
v_shard_id INTEGER;
|
||||
BEGIN
|
||||
IF p_num_shards IS NULL THEN
|
||||
SELECT COUNT(*)::INTEGER INTO p_num_shards FROM __PREFIX__shard_lock;
|
||||
END IF;
|
||||
v_shard_id := abs(hashtext(p_channel)::bigint) % p_num_shards;
|
||||
|
||||
PERFORM 1 FROM __PREFIX__shard_lock WHERE shard_id = v_shard_id FOR UPDATE;
|
||||
|
||||
DELETE FROM __STREAM_TABLE__ WHERE channel = p_channel AND kind = 0;
|
||||
-- Note: meta epoch is NOT reset — matches Redis broker behavior. Joins
|
||||
-- and leaves (kind=1/2) are also kept and age out via partition retention.
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- __PREFIX__top_position: Get current stream top position (top_offset + epoch).
|
||||
-- Lightweight read for the external-state pattern: the application calls this
|
||||
-- inside the same transaction as reading its own data, capturing the stream
|
||||
-- position that covers any mutations during/after the state read. The client
|
||||
-- SDK then subscribes with this position and recovers any publications that
|
||||
-- arrived between the read and subscribe.
|
||||
CREATE OR REPLACE FUNCTION __PREFIX__top_position(
|
||||
p_channel TEXT
|
||||
) RETURNS TABLE(
|
||||
out_top_offset BIGINT,
|
||||
out_epoch TEXT
|
||||
) AS $$
|
||||
BEGIN
|
||||
RETURN QUERY
|
||||
SELECT COALESCE(m.top_offset, 0::BIGINT), COALESCE(m.epoch, ''::TEXT)
|
||||
FROM (SELECT 1) AS _dummy
|
||||
LEFT JOIN __PREFIX__meta m ON m.channel = p_channel;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
281
internal/pgstreambroker/outbox.go
Normal file
281
internal/pgstreambroker/outbox.go
Normal file
@@ -0,0 +1,281 @@
|
||||
package pgstreambroker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/centrifugal/centrifugo/v6/internal/pgoutbox"
|
||||
|
||||
"github.com/centrifugal/centrifuge"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// Kind values for the kind column on the history table.
|
||||
const (
|
||||
kindPublication int16 = 0
|
||||
kindJoin int16 = 1
|
||||
kindLeave int16 = 2
|
||||
)
|
||||
|
||||
// outboxWorkerConfig returns (pool, shardIDs) for outbox worker workerIdx.
|
||||
// One shard per worker — the standard pgmapbroker pattern. With replicas,
|
||||
// the worker reads from readPools[workerIdx % len(readPools)].
|
||||
func (e *PostgresStreamBroker) outboxWorkerConfig(workerIdx int) (*pgxpool.Pool, []int) {
|
||||
pool := e.pool
|
||||
if len(e.readPools) > 0 {
|
||||
pool = e.readPools[workerIdx%len(e.readPools)]
|
||||
}
|
||||
return pool, []int{workerIdx}
|
||||
}
|
||||
|
||||
// initOutboxCursor bootstraps the outbox worker cursor from MAX(id) of the
|
||||
// history table. Any rows committed before the worker started are skipped
|
||||
// (at-most-once live delivery; reachable via History recovery).
|
||||
func (e *PostgresStreamBroker) initOutboxCursor(ctx context.Context, pool *pgxpool.Pool) (int64, error) {
|
||||
var cursor int64
|
||||
err := pool.QueryRow(ctx, fmt.Sprintf(
|
||||
`SELECT COALESCE(MAX(id), 0) FROM %s`, e.names.stream)).Scan(&cursor)
|
||||
return cursor, err
|
||||
}
|
||||
|
||||
// runOutboxWorker is the per-shard outbox poller for the non-fanout case.
|
||||
// Wraps pgoutbox.Worker; the broker-specific row scanning + dispatch lives
|
||||
// in processOutboxBatch.
|
||||
func (e *PostgresStreamBroker) runOutboxWorker(workerIdx int, initialCursor int64) {
|
||||
pool, shards := e.outboxWorkerConfig(workerIdx)
|
||||
|
||||
w := &pgoutbox.Worker{
|
||||
Pool: pool,
|
||||
ShardIDs: shards,
|
||||
PollInterval: e.conf.Outbox.PollInterval,
|
||||
NotifyCh: e.notifyCh,
|
||||
InitCursor: func(ctx context.Context, p *pgxpool.Pool) (int64, error) {
|
||||
return initialCursor, nil
|
||||
},
|
||||
ProcessBatch: func(ctx context.Context, p *pgxpool.Pool, cursor int64, sids []int) (int, int64, error) {
|
||||
return e.processOutboxBatch(ctx, p, cursor, sids)
|
||||
},
|
||||
ErrorFn: e.logErrorMsg,
|
||||
}
|
||||
w.Run(e.cancelCtx, e.closeCh)
|
||||
}
|
||||
|
||||
// runOutboxWorkerWithLock is the per-shard outbox poller for fanout mode.
|
||||
// Wraps pgoutbox.LockWorker which uses a session-level advisory lock so only
|
||||
// one node per shard polls at a time.
|
||||
//
|
||||
// The initialCursor parameter is intentionally unused — pgoutbox.LockWorker
|
||||
// calls InitCursor on every lock acquisition (not just at startup) so the
|
||||
// cursor can catch up to MAX(id) after another node held the lock. Returning
|
||||
// a fixed startup value here is not a correctness bug (subscribers dedup by
|
||||
// offset; the position-check loop + history recovery close any live gap) but
|
||||
// it costs O(rows_since_startup) of re-fanout work on every re-acquisition.
|
||||
// Querying fresh MAX(id) each time keeps the steady-state cost bounded.
|
||||
func (e *PostgresStreamBroker) runOutboxWorkerWithLock(workerIdx int, _ int64) {
|
||||
pollPool, shards := e.outboxWorkerConfig(workerIdx)
|
||||
|
||||
lw := &pgoutbox.LockWorker{
|
||||
LockPool: e.pool,
|
||||
PollPool: pollPool,
|
||||
ShardIDs: shards,
|
||||
LockID: e.conf.Outbox.AdvisoryLockBaseID + int64(workerIdx),
|
||||
PollInterval: e.conf.Outbox.PollInterval,
|
||||
RetryInterval: e.conf.Outbox.AdvisoryLockRetryInterval,
|
||||
InitCursor: func(ctx context.Context, p *pgxpool.Pool) (int64, error) {
|
||||
cursor, err := e.initOutboxCursor(ctx, p)
|
||||
if err == nil {
|
||||
e.workersInitialized[workerIdx].Store(true)
|
||||
}
|
||||
return cursor, err
|
||||
},
|
||||
ProcessBatch: func(ctx context.Context, p *pgxpool.Pool, cursor int64, sids []int) (int, int64, error) {
|
||||
return e.processOutboxBatch(ctx, p, cursor, sids)
|
||||
},
|
||||
ErrorFn: func(msg string, err error) { e.logErrorMsg(msg, err) },
|
||||
InfoFn: func(msg string) { /* TODO: info logging */ },
|
||||
}
|
||||
lw.Run(e.cancelCtx, e.closeCh)
|
||||
}
|
||||
|
||||
// runNotificationListener listens for pg_notify on the broker's notify channel
|
||||
// and wakes the outbox worker(s). Thin wrapper around pgoutbox.NotificationListener.
|
||||
func (e *PostgresStreamBroker) runNotificationListener() {
|
||||
pool := e.pool
|
||||
if e.notifyPool != nil {
|
||||
pool = e.notifyPool
|
||||
}
|
||||
l := &pgoutbox.NotificationListener{
|
||||
Pool: pool,
|
||||
Channel: e.names.notifyChannel,
|
||||
NotifyCh: e.notifyCh,
|
||||
ErrorFn: e.logErrorMsg,
|
||||
OnReady: func() { e.notifyListenerReady.Store(true) },
|
||||
}
|
||||
l.Run(e.cancelCtx, e.closeCh)
|
||||
}
|
||||
|
||||
// processOutboxBatch reads up to BatchSize history rows past the cursor for
|
||||
// the given shard, scans them with zero-allocation pgx helpers, and dispatches
|
||||
// each one based on its kind:
|
||||
//
|
||||
// - kind=0 (publication): construct *Publication, call HandlePublication
|
||||
// (or fanout's inner Broker.Publish in fanout mode)
|
||||
// - kind=1 (join): construct *ClientInfo, call HandleJoin
|
||||
// - kind=2 (leave): construct *ClientInfo, call HandleLeave
|
||||
//
|
||||
// Returns (rowsProcessed, newCursor, error).
|
||||
func (e *PostgresStreamBroker) processOutboxBatch(
|
||||
ctx context.Context, pool *pgxpool.Pool,
|
||||
cursor int64, shardIDs []int,
|
||||
) (int, int64, error) {
|
||||
query := fmt.Sprintf(`
|
||||
SELECT id, shard_id, channel, channel_offset, epoch, kind, data, tags,
|
||||
client_id, user_id, conn_info, chan_info, prev_data
|
||||
FROM %s
|
||||
WHERE id > $1 AND shard_id = ANY($2)
|
||||
ORDER BY id
|
||||
LIMIT $3
|
||||
`, e.names.stream)
|
||||
|
||||
rows, err := pool.Query(ctx, query, cursor, shardIDs, e.conf.Outbox.BatchSize)
|
||||
if err != nil {
|
||||
return 0, cursor, fmt.Errorf("postgres stream broker: outbox query: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
allocHint := e.conf.Outbox.BatchSize
|
||||
if allocHint > 1001 {
|
||||
allocHint = 1001
|
||||
}
|
||||
arena := byteArena{buf: make([]byte, 0, allocHint*64)}
|
||||
pubBacking := make([]centrifuge.Publication, 0, allocHint)
|
||||
infoBacking := make([]centrifuge.ClientInfo, 0, allocHint)
|
||||
|
||||
type batchEntry struct {
|
||||
id int64
|
||||
channel string
|
||||
kind int16
|
||||
pub *centrifuge.Publication
|
||||
info *centrifuge.ClientInfo
|
||||
epoch string
|
||||
}
|
||||
entries := make([]batchEntry, 0, allocHint)
|
||||
|
||||
var fmts pgColFormats
|
||||
maxID := cursor
|
||||
for rows.Next() {
|
||||
if fmts == nil {
|
||||
fmts = pgColFormatsFromRows(rows)
|
||||
}
|
||||
raw := rows.RawValues()
|
||||
// Column order:
|
||||
// 0 id, 1 shard_id, 2 channel, 3 channel_offset, 4 epoch, 5 kind,
|
||||
// 6 data, 7 tags, 8 client_id, 9 user_id, 10 conn_info, 11 chan_info,
|
||||
// 12 prev_data
|
||||
id := pgRawInt64(raw[0], fmts[0])
|
||||
channel := pgRawString(&arena, raw[2])
|
||||
channelOffset := pgRawUint64(raw[3], fmts[3])
|
||||
epoch := pgRawString(&arena, raw[4])
|
||||
kind := pgRawInt16(raw[5], fmts[5])
|
||||
|
||||
entry := batchEntry{
|
||||
id: id,
|
||||
channel: channel,
|
||||
kind: kind,
|
||||
epoch: epoch,
|
||||
}
|
||||
|
||||
switch kind {
|
||||
case kindPublication:
|
||||
pubBacking = append(pubBacking, centrifuge.Publication{})
|
||||
p := &pubBacking[len(pubBacking)-1]
|
||||
p.Offset = channelOffset
|
||||
p.Epoch = epoch
|
||||
p.Data = e.rawDataBytes(&arena, raw[6], fmts[6])
|
||||
p.Tags = pgRawJSONBMap(raw[7])
|
||||
if raw[8] != nil {
|
||||
infoBacking = append(infoBacking, centrifuge.ClientInfo{
|
||||
ClientID: pgRawString(&arena, raw[8]),
|
||||
UserID: pgRawString(&arena, raw[9]),
|
||||
ConnInfo: e.rawDataBytes(&arena, raw[10], fmts[10]),
|
||||
ChanInfo: e.rawDataBytes(&arena, raw[11], fmts[11]),
|
||||
})
|
||||
p.Info = &infoBacking[len(infoBacking)-1]
|
||||
}
|
||||
entry.pub = p
|
||||
case kindJoin, kindLeave:
|
||||
if raw[8] != nil {
|
||||
infoBacking = append(infoBacking, centrifuge.ClientInfo{
|
||||
ClientID: pgRawString(&arena, raw[8]),
|
||||
UserID: pgRawString(&arena, raw[9]),
|
||||
ConnInfo: e.rawDataBytes(&arena, raw[10], fmts[10]),
|
||||
ChanInfo: e.rawDataBytes(&arena, raw[11], fmts[11]),
|
||||
})
|
||||
entry.info = &infoBacking[len(infoBacking)-1]
|
||||
}
|
||||
}
|
||||
|
||||
entries = append(entries, entry)
|
||||
if id > maxID {
|
||||
maxID = id
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, cursor, fmt.Errorf("postgres stream broker: outbox rows: %w", err)
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
// Dispatch entries.
|
||||
for _, en := range entries {
|
||||
switch en.kind {
|
||||
case kindPublication:
|
||||
sp := centrifuge.StreamPosition{Offset: en.pub.Offset, Epoch: en.epoch}
|
||||
if e.conf.Broker != nil {
|
||||
// Fanout mode: forward to inner broker.
|
||||
_, err := e.conf.Broker.Publish(en.channel, en.pub.Data, centrifuge.PublishOptions{
|
||||
ClientInfo: en.pub.Info,
|
||||
Tags: en.pub.Tags,
|
||||
Offset: en.pub.Offset,
|
||||
Epoch: en.epoch,
|
||||
})
|
||||
if err != nil {
|
||||
e.logErrorMsg("outbox forward to inner broker", err)
|
||||
}
|
||||
} else {
|
||||
if err := e.eventHandler.HandlePublication(en.channel, en.pub, sp, false, nil); err != nil {
|
||||
e.logErrorMsg("outbox HandlePublication", err)
|
||||
}
|
||||
}
|
||||
case kindJoin:
|
||||
if e.conf.Broker != nil {
|
||||
// Joins shouldn't reach here in fanout mode (PublishJoin
|
||||
// short-circuits to the inner broker), but handle defensively.
|
||||
if err := e.conf.Broker.PublishJoin(en.channel, en.info); err != nil {
|
||||
e.logErrorMsg("outbox forward join to inner broker", err)
|
||||
}
|
||||
} else if en.info != nil {
|
||||
if err := e.eventHandler.HandleJoin(en.channel, en.info); err != nil {
|
||||
e.logErrorMsg("outbox HandleJoin", err)
|
||||
}
|
||||
}
|
||||
case kindLeave:
|
||||
if e.conf.Broker != nil {
|
||||
if err := e.conf.Broker.PublishLeave(en.channel, en.info); err != nil {
|
||||
e.logErrorMsg("outbox forward leave to inner broker", err)
|
||||
}
|
||||
} else if en.info != nil {
|
||||
if err := e.eventHandler.HandleLeave(en.channel, en.info); err != nil {
|
||||
e.logErrorMsg("outbox HandleLeave", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update sampler cursor for the lag metric. With one shard per worker
|
||||
// the shardIDs slice has exactly one element.
|
||||
if e.sampler != nil && len(shardIDs) == 1 {
|
||||
e.sampler.storeCursor(shardIDs[0], maxID)
|
||||
}
|
||||
|
||||
return len(entries), maxID, nil
|
||||
}
|
||||
113
internal/pgstreambroker/partition.go
Normal file
113
internal/pgstreambroker/partition.go
Normal file
@@ -0,0 +1,113 @@
|
||||
package pgstreambroker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/centrifugal/centrifugo/v6/internal/metrics"
|
||||
"github.com/centrifugal/centrifugo/v6/internal/pgoutbox"
|
||||
)
|
||||
|
||||
// isShutdownErr reports whether err is a normal-shutdown signal — context
|
||||
// cancellation surfacing because the broker is being closed mid-tick. Those
|
||||
// aren't real failures and shouldn't be logged at error level, since they
|
||||
// would otherwise pollute CI logs and mask genuine cleanup errors.
|
||||
func isShutdownErr(err error) bool {
|
||||
return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)
|
||||
}
|
||||
|
||||
// newPartitioner constructs a pgoutbox.Partitioner configured for the
|
||||
// stream broker's history table.
|
||||
func (e *PostgresStreamBroker) newPartitioner() *pgoutbox.Partitioner {
|
||||
return &pgoutbox.Partitioner{
|
||||
Pool: e.pool,
|
||||
ParentTable: e.names.stream,
|
||||
CleanupInterval: e.conf.CleanupInterval,
|
||||
LookaheadDays: e.conf.PartitionLookaheadDays,
|
||||
RetentionDays: e.conf.PartitionRetentionDays,
|
||||
ErrorFn: e.logErrorMsg,
|
||||
}
|
||||
}
|
||||
|
||||
// runPartitionWorker manages partition lookahead creation and retention drops.
|
||||
func (e *PostgresStreamBroker) runPartitionWorker() {
|
||||
p := e.newPartitioner()
|
||||
p.Run(e.cancelCtx, e.closeCh)
|
||||
}
|
||||
|
||||
// runCleanupWorker runs the cleanup pass on CleanupInterval ticks.
|
||||
func (e *PostgresStreamBroker) runCleanupWorker() {
|
||||
ticker := time.NewTicker(e.conf.CleanupInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-e.cancelCtx.Done():
|
||||
return
|
||||
case <-e.closeCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
e.cleanupSupportTables(e.cancelCtx)
|
||||
if e.sampler != nil {
|
||||
e.sampler.sample(e.cancelCtx)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func addCleanupRows(broker, pass string, n int64) {
|
||||
if metrics.BrokerPostgresCleanupRemovedTotal != nil {
|
||||
metrics.BrokerPostgresCleanupRemovedTotal.WithLabelValues(broker, pass).Add(float64(n))
|
||||
}
|
||||
}
|
||||
|
||||
// cleanupSupportTables deletes expired meta and idempotency rows in bounded
|
||||
// batches to avoid long-running DELETE locks on busy tables. Each batch is
|
||||
// capped at cleanupBatchSize rows; passes repeat with a cleanupChunkPause gap
|
||||
// until fewer than a full batch is deleted. Context cancellation (broker
|
||||
// shutting down mid-tick) is treated as normal and not logged.
|
||||
func (e *PostgresStreamBroker) cleanupSupportTables(ctx context.Context) {
|
||||
e.cleanupExpiredBatched(ctx, "meta", fmt.Sprintf(
|
||||
`DELETE FROM %[1]s WHERE channel IN (
|
||||
SELECT channel FROM %[1]s
|
||||
WHERE expires_at IS NOT NULL AND expires_at < NOW()
|
||||
LIMIT $1
|
||||
)`, e.names.meta))
|
||||
e.cleanupExpiredBatched(ctx, "idempotency", fmt.Sprintf(
|
||||
`DELETE FROM %[1]s WHERE (channel, idempotency_key) IN (
|
||||
SELECT channel, idempotency_key FROM %[1]s
|
||||
WHERE expires_at < NOW()
|
||||
LIMIT $1
|
||||
)`, e.names.idempotency))
|
||||
}
|
||||
|
||||
const (
|
||||
cleanupBatchSize = 1000
|
||||
cleanupChunkPause = 100 * time.Millisecond
|
||||
)
|
||||
|
||||
// cleanupExpiredBatched runs a chunked DELETE loop for the given table.
|
||||
// Stops when a batch deletes fewer rows than cleanupBatchSize or ctx is done.
|
||||
func (e *PostgresStreamBroker) cleanupExpiredBatched(ctx context.Context, passName, query string) {
|
||||
for {
|
||||
res, err := e.pool.Exec(ctx, query, cleanupBatchSize)
|
||||
if err != nil {
|
||||
if !isShutdownErr(err) {
|
||||
e.logErrorMsg("error cleaning up expired "+passName, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
addCleanupRows(e.conf.Name, passName, res.RowsAffected())
|
||||
if res.RowsAffected() < cleanupBatchSize {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-e.closeCh:
|
||||
return
|
||||
case <-time.After(cleanupChunkPause):
|
||||
}
|
||||
}
|
||||
}
|
||||
660
internal/pgstreambroker/pgstreambroker.go
Normal file
660
internal/pgstreambroker/pgstreambroker.go
Normal file
@@ -0,0 +1,660 @@
|
||||
// Package pgstreambroker implements centrifuge.Broker using PostgreSQL as
|
||||
// the backing store for stream subscriptions. It provides:
|
||||
//
|
||||
// - Transactional publishing: applications can publish in the same SQL
|
||||
// transaction as their business writes, eliminating the dual-write problem.
|
||||
// - Per-channel HistoryTTL and HistorySize honored at read time via the
|
||||
// two-TTL meta model (matches Redis broker semantics).
|
||||
// - At-most-once live delivery via the polling outbox; reliable recovery
|
||||
// via History() reads.
|
||||
// - Always-partitioned schema with daily partition rolling for vacuum-free
|
||||
// bulk cleanup. PartitionRetentionDays defaults to 7 days; the partition
|
||||
// retention worker (pgoutbox.Partitioner) drops old partitions whole.
|
||||
//
|
||||
// PG version requirement: PostgreSQL 13 or later (for partitioned table
|
||||
// features and INSERT ... ON CONFLICT DO UPDATE on partitioned tables).
|
||||
//
|
||||
// See internal/pgstreambroker/internal/sql/schema.sql for the underlying
|
||||
// schema and SQL functions.
|
||||
package pgstreambroker
|
||||
|
||||
import (
|
||||
"context"
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/centrifugal/centrifugo/v6/internal/configtypes"
|
||||
"github.com/centrifugal/centrifugo/v6/internal/pgschema"
|
||||
|
||||
"github.com/centrifugal/centrifuge"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
//go:embed internal/sql/schema.sql
|
||||
var postgresSchemaTemplate string
|
||||
|
||||
// schemaVersion is the current schema version. Bump when adding migrations.
|
||||
var schemaVersion = 1
|
||||
|
||||
// schemaMigrations maps target version to a migration SQL TEMPLATE using the
|
||||
// same placeholders as `schema.sql` (`__PREFIX__`, `__DATA_TYPE__`,
|
||||
// `__STREAM_TABLE__`). EnsureSchema renders the template once per variant
|
||||
// (jsonb + binary) and runs both rendered SQLs in a single transaction with
|
||||
// atomic schema_version bumps across both prefix tables — see
|
||||
// pgmapbroker.schemaMigrations for the full contract. Empty for v1.
|
||||
var schemaMigrations = map[int]string{}
|
||||
|
||||
func init() {
|
||||
pgschema.ValidateMigrationMap("pgstreambroker", schemaVersion, schemaMigrations)
|
||||
}
|
||||
|
||||
// renderSchemaTemplate substitutes the placeholders this broker recognises
|
||||
// in any template string. Reused by renderSchema (for the embedded baseline)
|
||||
// and by migrationVariants (for each registered migration). Keeping this in
|
||||
// one place means migration SQL works for any user-configured TablePrefix
|
||||
// and for both jsonb/binary variants without the migration author writing
|
||||
// the same statement twice.
|
||||
//
|
||||
// __PREFIX__ → e.g. "cf_stream_" (includes trailing underscore)
|
||||
// __DATA_TYPE__ → "JSONB" or "BYTEA"
|
||||
// __STREAM_TABLE__ → e.g. "cf_stream" (prefix without trailing underscore)
|
||||
func renderSchemaTemplate(template, prefix string, binary bool) string {
|
||||
dataType := "JSONB"
|
||||
if binary {
|
||||
dataType = "BYTEA"
|
||||
}
|
||||
streamTable := strings.TrimRight(prefix, "_")
|
||||
return strings.NewReplacer(
|
||||
"__STREAM_TABLE__", streamTable,
|
||||
"__PREFIX__", prefix,
|
||||
"__DATA_TYPE__", dataType,
|
||||
).Replace(template)
|
||||
}
|
||||
|
||||
func renderSchema(prefix string, binary bool) string {
|
||||
return renderSchemaTemplate(postgresSchemaTemplate, prefix, binary)
|
||||
}
|
||||
|
||||
// splitSchemaSQL splits the schema SQL into DDL (tables+indexes) and function
|
||||
// definitions. They must be executed as separate transactions to avoid deadlocks
|
||||
// with concurrent function executions during rolling deploys.
|
||||
func splitSchemaSQL(sql string) (ddl, funcs string) {
|
||||
const marker = "CREATE OR REPLACE FUNCTION"
|
||||
i := strings.Index(sql, marker)
|
||||
if i < 0 {
|
||||
return sql, ""
|
||||
}
|
||||
return sql[:i], sql[i:]
|
||||
}
|
||||
|
||||
// SchemaObject identifies a database object referenced in a SchemaError.
|
||||
type SchemaObject struct {
|
||||
Type string // "table", "function", "index", "schema"
|
||||
Name string
|
||||
}
|
||||
|
||||
// SchemaError wraps a schema-related error with object and operation info.
|
||||
type SchemaError struct {
|
||||
Object SchemaObject
|
||||
Op string // "create", "verify", etc.
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *SchemaError) Error() string {
|
||||
return fmt.Sprintf("postgres stream broker: %s %s %q: %v", e.Op, e.Object.Type, e.Object.Name, e.Err)
|
||||
}
|
||||
|
||||
func (e *SchemaError) Unwrap() error {
|
||||
return e.Err
|
||||
}
|
||||
|
||||
// pgNames holds precomputed table/function/notify-channel names. The
|
||||
// jsonbPrefix and binaryPrefix are kept so EnsureSchema can iterate both
|
||||
// variants regardless of the active BinaryData mode.
|
||||
//
|
||||
// jsonbPrefix = userPrefix + "_stream_" (e.g. "cf_stream_")
|
||||
// binaryPrefix = userPrefix + "_binary_stream_" (e.g. "cf_binary_stream_")
|
||||
//
|
||||
// The history/meta/idempotency/... fields are computed from the *active*
|
||||
// prefix (jsonbPrefix when BinaryData is false, binaryPrefix when true).
|
||||
type pgNames struct {
|
||||
stream, meta, idempotency, shardLock, schemaVersion string // table names (active variant)
|
||||
publish, publishStrict, publishJoin, publishLeave string // function names (active variant)
|
||||
removeHistory string
|
||||
notifyChannel string
|
||||
|
||||
jsonbPrefix, binaryPrefix string
|
||||
}
|
||||
|
||||
func newPgNames(userPrefix string, binary bool) pgNames {
|
||||
jsonbPrefix := userPrefix + "_stream_"
|
||||
binaryPrefix := userPrefix + "_binary_stream_"
|
||||
|
||||
p := jsonbPrefix
|
||||
if binary {
|
||||
p = binaryPrefix
|
||||
}
|
||||
return pgNames{
|
||||
stream: strings.TrimRight(p, "_"), // cf_stream (not cf_stream_history)
|
||||
meta: p + "meta",
|
||||
idempotency: p + "idempotency",
|
||||
shardLock: p + "shard_lock",
|
||||
schemaVersion: p + "schema_version",
|
||||
publish: p + "publish",
|
||||
publishStrict: p + "publish_strict",
|
||||
publishJoin: p + "publish_join",
|
||||
publishLeave: p + "publish_leave",
|
||||
removeHistory: p + "remove_history",
|
||||
notifyChannel: p + "notify",
|
||||
jsonbPrefix: jsonbPrefix,
|
||||
binaryPrefix: binaryPrefix,
|
||||
}
|
||||
}
|
||||
|
||||
// durationToIntervalString converts a time.Duration to a PostgreSQL interval string.
|
||||
func durationToIntervalString(d time.Duration) string {
|
||||
ms := d.Milliseconds()
|
||||
if ms > 0 {
|
||||
return strconv.FormatInt(ms, 10) + " milliseconds"
|
||||
}
|
||||
return "1 milliseconds"
|
||||
}
|
||||
|
||||
// hashtext is a simple polynomial hash used only for in-process read-replica
|
||||
// routing. It only needs consistency within a process, not cross-process
|
||||
// compatibility with PostgreSQL's hashtext() (which uses Jenkins lookup3 and is
|
||||
// used separately inside SQL for shard assignment). It returns uint32 so the
|
||||
// result is always non-negative and can be taken modulo NumShards directly (no
|
||||
// abs, and no math.MinInt32 overflow edge case).
|
||||
func hashtext(s string) uint32 {
|
||||
var h uint32
|
||||
for i := 0; i < len(s); i++ {
|
||||
h = h*31 + uint32(s[i])
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// OutboxConfig configures outbox-based delivery.
|
||||
type OutboxConfig struct {
|
||||
// PollInterval is how often to poll for new history rows when idle.
|
||||
// Default: 100ms.
|
||||
PollInterval time.Duration
|
||||
|
||||
// BatchSize is the maximum number of rows to process per batch.
|
||||
// Default: 1000.
|
||||
BatchSize int
|
||||
|
||||
// AdvisoryLockBaseID is the base ID for PostgreSQL advisory locks used to
|
||||
// claim shards when Broker fan-out is enabled. Lock ID = AdvisoryLockBaseID + shardID.
|
||||
// Default: 726966531 (one above the map broker base to avoid collision).
|
||||
AdvisoryLockBaseID int64
|
||||
|
||||
// AdvisoryLockRetryInterval is how often to retry advisory lock acquisition
|
||||
// when Broker fan-out is enabled. Default: 5s.
|
||||
AdvisoryLockRetryInterval time.Duration
|
||||
}
|
||||
|
||||
// PostgresStreamBrokerConfig configures the PostgreSQL stream broker.
|
||||
type PostgresStreamBrokerConfig struct {
|
||||
// Name is an observability label included in metrics and logs.
|
||||
Name string
|
||||
|
||||
// DSN is the primary PostgreSQL connection string for writes.
|
||||
DSN string
|
||||
|
||||
// TLS is an optional TLS configuration applied to all pools (primary,
|
||||
// replicas, notify). Use instead of embedding TLS params in the DSN.
|
||||
TLS configtypes.TLSConfig
|
||||
|
||||
// PoolSize sets the maximum number of connections in the primary pool.
|
||||
// Default: 32.
|
||||
PoolSize int
|
||||
|
||||
// NumShards is the total number of shards for parallel delivery workers.
|
||||
// Channels are distributed via hash(channel) % NumShards.
|
||||
// Default: 8.
|
||||
NumShards int
|
||||
|
||||
// CleanupInterval is how often the cleanup worker and partition worker tick.
|
||||
// Default: 1m.
|
||||
CleanupInterval time.Duration
|
||||
|
||||
// IdempotentResultTTL is the default TTL for idempotency cache entries.
|
||||
// Default: 5m.
|
||||
IdempotentResultTTL time.Duration
|
||||
|
||||
// Outbox configures the outbox-based delivery mode.
|
||||
Outbox OutboxConfig
|
||||
|
||||
// BinaryData uses BYTEA columns instead of JSONB for data fields.
|
||||
// Set true if data payloads are not valid JSON (binary/protobuf).
|
||||
BinaryData bool
|
||||
|
||||
// TablePrefix is the user-facing namespace prefix. Default "cf".
|
||||
// The broker appends "_stream_" or "_binary_stream_" internally, so
|
||||
// the default produces:
|
||||
//
|
||||
// cf_stream_history, cf_stream_meta, cf_stream_idempotency, cf_stream_publish, ...
|
||||
//
|
||||
// and their cf_binary_stream_* counterparts for the binary variant.
|
||||
TablePrefix string
|
||||
|
||||
// StreamRetention is the safety floor for HistoryMetaTTL when neither
|
||||
// PublishOptions nor node config sets it. Default: 24h. Guarantees that
|
||||
// every channel meta row eventually expires (no NULL expires_at).
|
||||
StreamRetention time.Duration
|
||||
|
||||
// UseNotify enables LISTEN/NOTIFY for low-latency outbox wakeup.
|
||||
// When false (default), outbox worker uses PollInterval-based polling only.
|
||||
UseNotify bool
|
||||
|
||||
// NotifyDSN is an optional separate connection string used exclusively for
|
||||
// the LISTEN connection when UseNotify is true. Set this to a direct
|
||||
// PostgreSQL URL (bypassing PGBouncer) when DSN points at a PGBouncer
|
||||
// endpoint — PGBouncer transaction pooling mode is incompatible with
|
||||
// LISTEN/NOTIFY. If empty, the primary DSN pool is used (fine for direct
|
||||
// PostgreSQL connections).
|
||||
NotifyDSN string
|
||||
|
||||
// ReplicaDSN is a list of read-replica connection strings. When set,
|
||||
// outbox cursor bootstraps go to the replica routing instead of the
|
||||
// primary. Default: empty (all reads go to primary).
|
||||
ReplicaDSN []string
|
||||
|
||||
// ReplicaPoolSize sets max connections per replica pool.
|
||||
// Default: same as PoolSize.
|
||||
ReplicaPoolSize int
|
||||
|
||||
// Broker is an optional Broker (e.g. RedisBroker) for PUB/SUB fan-out.
|
||||
// When set, outbox workers use advisory locks (one worker per shard
|
||||
// across all nodes) and publish via the inner Broker instead of
|
||||
// HandlePublication directly. PublishJoin/PublishLeave bypass PG
|
||||
// entirely and go through the inner broker.
|
||||
Broker centrifuge.Broker
|
||||
|
||||
// PartitionLookaheadDays controls how many future daily partitions to
|
||||
// pre-create. Required > 0 so writes don't fail at the day rollover.
|
||||
// Default: 2 (gives a 48-hour safety window).
|
||||
PartitionLookaheadDays int
|
||||
|
||||
// PartitionRetentionDays controls how old a partition can be before
|
||||
// it gets dropped whole by the partition retention worker.
|
||||
//
|
||||
// Set to 0 for unlimited retention (the pgoutbox.Partitioner guard
|
||||
// treats RetentionDays <= 0 as "never drop"; old partitions accumulate).
|
||||
//
|
||||
// The OSS configtypes default is 7 (via the struct-tag default). Direct
|
||||
// Go-level construction (e.g. tests) must set this explicitly — there
|
||||
// is no implicit default in setDefaults so that 0 (unlimited) survives.
|
||||
PartitionRetentionDays int
|
||||
}
|
||||
|
||||
func (c *PostgresStreamBrokerConfig) setDefaults() {
|
||||
if c.PoolSize <= 0 {
|
||||
c.PoolSize = 16
|
||||
}
|
||||
if c.NumShards <= 0 {
|
||||
c.NumShards = 8
|
||||
}
|
||||
if c.CleanupInterval <= 0 {
|
||||
c.CleanupInterval = time.Minute
|
||||
}
|
||||
if c.IdempotentResultTTL <= 0 {
|
||||
c.IdempotentResultTTL = 5 * time.Minute
|
||||
}
|
||||
if c.StreamRetention <= 0 {
|
||||
c.StreamRetention = 24 * time.Hour
|
||||
}
|
||||
c.TablePrefix = strings.TrimRight(c.TablePrefix, "_")
|
||||
if c.TablePrefix == "" {
|
||||
c.TablePrefix = "cf"
|
||||
}
|
||||
|
||||
if c.Outbox.PollInterval <= 0 {
|
||||
c.Outbox.PollInterval = 100 * time.Millisecond
|
||||
}
|
||||
if c.Outbox.BatchSize <= 0 {
|
||||
c.Outbox.BatchSize = 1000
|
||||
}
|
||||
if c.Outbox.AdvisoryLockBaseID == 0 {
|
||||
// Stream broker occupies [5_067_067_000, 5_067_067_000+NumShards).
|
||||
// Map broker uses [4_067_067_000, 4_067_067_000+NumShards).
|
||||
// Non-round bases reduce the chance of collision with other software.
|
||||
c.Outbox.AdvisoryLockBaseID = 5_067_067_000
|
||||
}
|
||||
if c.Outbox.AdvisoryLockRetryInterval <= 0 {
|
||||
c.Outbox.AdvisoryLockRetryInterval = 5 * time.Second
|
||||
}
|
||||
if c.ReplicaPoolSize <= 0 {
|
||||
c.ReplicaPoolSize = c.PoolSize
|
||||
}
|
||||
if c.PartitionLookaheadDays <= 0 {
|
||||
c.PartitionLookaheadDays = 2
|
||||
}
|
||||
// PartitionRetentionDays is intentionally NOT defaulted here. The
|
||||
// configtypes layer (centrifugo config tag default:"7") handles the
|
||||
// production default. Direct Go-level construction must set this
|
||||
// explicitly: positive = days of retention, 0 = unlimited (the
|
||||
// pgoutbox.Partitioner guard treats 0 as a no-op DROP). Defaulting
|
||||
// to 7 here would make it impossible to express "unlimited retention"
|
||||
// without a sentinel like -1.
|
||||
}
|
||||
|
||||
// PostgresStreamBroker is a centrifuge.Broker implementation backed by PostgreSQL.
|
||||
type PostgresStreamBroker struct {
|
||||
node *centrifuge.Node
|
||||
conf PostgresStreamBrokerConfig
|
||||
names pgNames
|
||||
pool *pgxpool.Pool
|
||||
readPools []*pgxpool.Pool
|
||||
notifyPool *pgxpool.Pool // Dedicated single-conn pool for LISTEN; nil = use pool
|
||||
eventHandler centrifuge.BrokerEventHandler
|
||||
closeCh chan struct{}
|
||||
closeOnce sync.Once
|
||||
running atomic.Bool
|
||||
cancelCtx context.Context
|
||||
cancelFunc context.CancelFunc
|
||||
notifyCh chan struct{}
|
||||
sampler *metricsSampler
|
||||
|
||||
// notifyListenerReady flips true after the LISTEN command succeeds on
|
||||
// the notification listener's connection. Used by tests that depend on
|
||||
// NOTIFY-driven delivery (typically with a long PollInterval) so they
|
||||
// publish only after LISTEN is bound. Production tolerates the race
|
||||
// because PollInterval is the fallback.
|
||||
notifyListenerReady atomic.Bool
|
||||
|
||||
// workersInitialized[i] flips true once shard i's outbox LockWorker
|
||||
// completes its first InitCursor (i.e., has snapshotted MAX(id) and is
|
||||
// actively polling). Used by tests to publish only after the cursor
|
||||
// snapshot is taken — otherwise a publish committed between lock
|
||||
// acquisition and the snapshot would be observed by InitCursor and
|
||||
// skipped. Only the LockWorker path sets this; left as nil for the
|
||||
// non-fan-out Worker path (which uses a pre-init cursor synchronously
|
||||
// in RegisterBrokerEventHandler and so doesn't have this race).
|
||||
workersInitialized []atomic.Bool
|
||||
}
|
||||
|
||||
var _ centrifuge.Broker = (*PostgresStreamBroker)(nil)
|
||||
|
||||
// NewPostgresStreamBroker constructs a new PostgresStreamBroker. Call EnsureSchema
|
||||
// after construction and then RegisterBrokerEventHandler.
|
||||
func NewPostgresStreamBroker(n *centrifuge.Node, conf PostgresStreamBrokerConfig) (*PostgresStreamBroker, error) {
|
||||
conf.setDefaults()
|
||||
|
||||
if conf.DSN == "" {
|
||||
return nil, errors.New("postgres stream broker: DSN is required")
|
||||
}
|
||||
if err := pgschema.ValidateTablePrefix("postgres stream broker", conf.TablePrefix); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
poolConfig, err := pgxpool.ParseConfig(conf.DSN)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("postgres stream broker: parse config: %w", err)
|
||||
}
|
||||
poolConfig.MaxConns = int32(conf.PoolSize)
|
||||
if conf.TLS.Enabled {
|
||||
tlsCfg, err := conf.TLS.ToGoTLSConfig("postgres-stream-broker")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("postgres stream broker: TLS config: %w", err)
|
||||
}
|
||||
poolConfig.ConnConfig.TLSConfig = tlsCfg
|
||||
}
|
||||
|
||||
pool, err := pgxpool.NewWithConfig(ctx, poolConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("postgres stream broker: create pool: %w", err)
|
||||
}
|
||||
|
||||
var readPools []*pgxpool.Pool
|
||||
for i, dsn := range conf.ReplicaDSN {
|
||||
rcfg, err := pgxpool.ParseConfig(dsn)
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
for _, rp := range readPools {
|
||||
rp.Close()
|
||||
}
|
||||
return nil, fmt.Errorf("postgres stream broker: parse replica %d config: %w", i, err)
|
||||
}
|
||||
rcfg.MaxConns = int32(conf.ReplicaPoolSize)
|
||||
if conf.TLS.Enabled {
|
||||
tlsCfg, err := conf.TLS.ToGoTLSConfig("postgres-stream-broker")
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
for _, rp := range readPools {
|
||||
rp.Close()
|
||||
}
|
||||
return nil, fmt.Errorf("postgres stream broker: replica TLS config: %w", err)
|
||||
}
|
||||
rcfg.ConnConfig.TLSConfig = tlsCfg
|
||||
}
|
||||
rp, err := pgxpool.NewWithConfig(ctx, rcfg)
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
for _, p := range readPools {
|
||||
p.Close()
|
||||
}
|
||||
return nil, fmt.Errorf("postgres stream broker: create replica %d pool: %w", i, err)
|
||||
}
|
||||
readPools = append(readPools, rp)
|
||||
}
|
||||
|
||||
cancelCtx, cancelFunc := context.WithCancel(context.Background())
|
||||
|
||||
b := &PostgresStreamBroker{
|
||||
node: n,
|
||||
conf: conf,
|
||||
names: newPgNames(conf.TablePrefix, conf.BinaryData),
|
||||
pool: pool,
|
||||
readPools: readPools,
|
||||
closeCh: make(chan struct{}),
|
||||
cancelCtx: cancelCtx,
|
||||
cancelFunc: cancelFunc,
|
||||
workersInitialized: make([]atomic.Bool, conf.NumShards),
|
||||
}
|
||||
b.sampler = newMetricsSampler(b)
|
||||
if conf.UseNotify {
|
||||
b.notifyCh = make(chan struct{}, 1)
|
||||
if conf.NotifyDSN != "" {
|
||||
nCfg, err := pgxpool.ParseConfig(conf.NotifyDSN)
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
for _, rp := range readPools {
|
||||
rp.Close()
|
||||
}
|
||||
cancelFunc()
|
||||
return nil, fmt.Errorf("postgres stream broker: parse notify DSN: %w", err)
|
||||
}
|
||||
nCfg.MaxConns = 1
|
||||
if conf.TLS.Enabled {
|
||||
tlsCfg, err := conf.TLS.ToGoTLSConfig("postgres-stream-broker")
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
for _, rp := range readPools {
|
||||
rp.Close()
|
||||
}
|
||||
cancelFunc()
|
||||
return nil, fmt.Errorf("postgres stream broker: notify TLS config: %w", err)
|
||||
}
|
||||
nCfg.ConnConfig.TLSConfig = tlsCfg
|
||||
}
|
||||
nPool, err := pgxpool.NewWithConfig(ctx, nCfg)
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
for _, rp := range readPools {
|
||||
rp.Close()
|
||||
}
|
||||
cancelFunc()
|
||||
return nil, fmt.Errorf("postgres stream broker: connect notify: %w", err)
|
||||
}
|
||||
b.notifyPool = nPool
|
||||
}
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// ReliableDelivery reports whether this broker guarantees no-gaps delivery to
|
||||
// local subscribers, so the centrifuge node can skip periodic position sync
|
||||
// requests. True when polling the PG outbox directly (shard lock guarantees
|
||||
// commit order = id order, cursor-based polling reads every row). False when
|
||||
// broker fan-out is enabled — the Redis/Nats fan-out leg can drop messages.
|
||||
func (e *PostgresStreamBroker) ReliableDelivery() bool {
|
||||
return e.conf.Broker == nil
|
||||
}
|
||||
|
||||
// Close shuts down the broker.
|
||||
func (e *PostgresStreamBroker) Close(ctx context.Context) error {
|
||||
e.closeOnce.Do(func() {
|
||||
e.cancelFunc()
|
||||
close(e.closeCh)
|
||||
if e.conf.Broker != nil {
|
||||
if c, ok := e.conf.Broker.(centrifuge.Closer); ok {
|
||||
_ = c.Close(ctx)
|
||||
}
|
||||
}
|
||||
for _, rp := range e.readPools {
|
||||
rp.Close()
|
||||
}
|
||||
if e.notifyPool != nil {
|
||||
e.notifyPool.Close()
|
||||
}
|
||||
e.pool.Close()
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// RegisterBrokerEventHandler registers the event handler and starts background workers.
|
||||
func (e *PostgresStreamBroker) RegisterBrokerEventHandler(h centrifuge.BrokerEventHandler) error {
|
||||
e.eventHandler = h
|
||||
|
||||
if e.running.Swap(true) {
|
||||
return errors.New("postgres stream broker: already running")
|
||||
}
|
||||
|
||||
// Pre-initialize the outbox cursor before launching worker goroutines so
|
||||
// that any message published after RegisterBrokerEventHandler returns is
|
||||
// guaranteed to be delivered. Without this, a goroutine scheduled late
|
||||
// could call initOutboxCursor after a message was already inserted, see
|
||||
// that ID as MAX(id), and silently skip it.
|
||||
initialCursor, err := e.initOutboxCursor(e.cancelCtx, e.pool)
|
||||
if err != nil {
|
||||
e.logErrorMsg("pre-init outbox cursor", err)
|
||||
initialCursor = 0
|
||||
}
|
||||
|
||||
if e.conf.Broker != nil {
|
||||
if err := e.conf.Broker.RegisterBrokerEventHandler(h); err != nil {
|
||||
return fmt.Errorf("postgres stream broker: register inner broker: %w", err)
|
||||
}
|
||||
for i := 0; i < e.conf.NumShards; i++ {
|
||||
go e.runOutboxWorkerWithLock(i, initialCursor)
|
||||
}
|
||||
} else {
|
||||
if e.conf.UseNotify {
|
||||
go e.runNotificationListener()
|
||||
}
|
||||
for i := 0; i < e.conf.NumShards; i++ {
|
||||
go e.runOutboxWorker(i, initialCursor)
|
||||
}
|
||||
}
|
||||
|
||||
go e.runCleanupWorker()
|
||||
go e.runPartitionWorker() // unconditional — schema is always partitioned
|
||||
return nil
|
||||
}
|
||||
|
||||
// Subscribe is a no-op when running without an inner broker (every node polls
|
||||
// the outbox). With fanout enabled, delegates to the inner Broker.
|
||||
func (e *PostgresStreamBroker) Subscribe(channels ...string) error {
|
||||
if e.conf.Broker != nil {
|
||||
return e.conf.Broker.Subscribe(channels...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unsubscribe mirrors Subscribe.
|
||||
func (e *PostgresStreamBroker) Unsubscribe(channels ...string) error {
|
||||
if e.conf.Broker != nil {
|
||||
return e.conf.Broker.Unsubscribe(channels...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getReadPool returns the appropriate pool for read queries. If allowCached
|
||||
// is true and replicas are configured, routes by shard hash to a replica.
|
||||
func (e *PostgresStreamBroker) getReadPool(channel string, allowCached bool) *pgxpool.Pool {
|
||||
if !allowCached || len(e.readPools) == 0 {
|
||||
return e.pool
|
||||
}
|
||||
shardID := int(hashtext(channel) % uint32(e.conf.NumShards))
|
||||
replicaIdx := shardID % len(e.readPools)
|
||||
return e.readPools[replicaIdx]
|
||||
}
|
||||
|
||||
// dataParam wraps a []byte for use as a SQL parameter. With JSONB columns,
|
||||
// wrap as json.RawMessage so pgx encodes it as JSON text. With BYTEA, pass
|
||||
// as plain []byte.
|
||||
func (e *PostgresStreamBroker) dataParam(b []byte) any {
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
if !e.conf.BinaryData {
|
||||
return json.RawMessage(b)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// rawDataBytes reads a data column (JSONB or BYTEA depending on BinaryData)
|
||||
// using the correct format-aware parser.
|
||||
func (e *PostgresStreamBroker) rawDataBytes(a *byteArena, b []byte, format int16) []byte {
|
||||
if e.conf.BinaryData {
|
||||
return pgRawBytes(a, b, format)
|
||||
}
|
||||
return pgRawJSONBBytes(a, b, format)
|
||||
}
|
||||
|
||||
func (e *PostgresStreamBroker) logErrorMsg(msg string, err error) {
|
||||
log.Error().Err(err).Str("broker", e.conf.Name).Msg("postgres stream broker: " + msg)
|
||||
}
|
||||
|
||||
// pgColFormatsFromRows extracts per-column wire format codes from pgx rows.
|
||||
func pgColFormatsFromRows(rows pgx.Rows) pgColFormats {
|
||||
descs := rows.FieldDescriptions()
|
||||
fmts := make(pgColFormats, len(descs))
|
||||
for i, d := range descs {
|
||||
fmts[i] = d.Format
|
||||
}
|
||||
return fmts
|
||||
}
|
||||
|
||||
// execSchemaWithRetry executes idempotent schema SQL, retrying the batch on
|
||||
// the conflicts several nodes running the same DDL at once produce — see
|
||||
// pgschema.RetrySchemaExec for the policy and for the invariant the SQL has
|
||||
// to keep (one multi-statement Exec with no arguments, so the server runs it
|
||||
// in a single implicit transaction that rolls back completely on failure).
|
||||
func (e *PostgresStreamBroker) execSchemaWithRetry(ctx context.Context, sql string) error {
|
||||
err := pgschema.RetrySchemaExec(ctx, func(ctx context.Context) error {
|
||||
_, err := e.pool.Exec(ctx, sql)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return &SchemaError{
|
||||
Object: SchemaObject{Type: "schema", Name: ""},
|
||||
Op: "create",
|
||||
Err: err,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
207
internal/pgstreambroker/pgstreambroker_bench_test.go
Normal file
207
internal/pgstreambroker/pgstreambroker_bench_test.go
Normal file
@@ -0,0 +1,207 @@
|
||||
//go:build integration
|
||||
|
||||
package pgstreambroker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/centrifugal/centrifuge"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// newBenchPostgresStreamBroker is a slim broker constructor for benchmarks.
|
||||
// It uses BinaryData=true (so we don't pay JSON validation cost) and a
|
||||
// large outbox batch size for throughput tests.
|
||||
func newBenchPostgresStreamBroker(b *testing.B) (*PostgresStreamBroker, *centrifuge.Node, *testBrokerEventHandler) {
|
||||
node, err := centrifuge.New(centrifuge.Config{})
|
||||
require.NoError(b, err)
|
||||
|
||||
connString := getPostgresConnString(b)
|
||||
e, err := NewPostgresStreamBroker(node, PostgresStreamBrokerConfig{
|
||||
DSN: connString,
|
||||
BinaryData: true,
|
||||
CleanupInterval: 1 * time.Minute, // long — don't interfere with bench
|
||||
StreamRetention: 24 * time.Hour,
|
||||
PartitionLookaheadDays: 1,
|
||||
PartitionRetentionDays: 7,
|
||||
Outbox: OutboxConfig{
|
||||
BatchSize: 1000,
|
||||
// PollInterval uses default (50ms) — matches map broker bench
|
||||
// config so numbers are comparable. Lower poll intervals steal
|
||||
// PG capacity from the publish goroutines.
|
||||
},
|
||||
})
|
||||
require.NoError(b, err)
|
||||
|
||||
ctx := context.Background()
|
||||
hardResetTestSchema(b, e)
|
||||
require.NoError(b, e.EnsureSchema(ctx))
|
||||
cleanupTestTables(ctx, e)
|
||||
|
||||
handler := &testBrokerEventHandler{}
|
||||
require.NoError(b, e.RegisterBrokerEventHandler(handler))
|
||||
require.NoError(b, node.Run())
|
||||
|
||||
b.Cleanup(func() {
|
||||
_ = e.Close(context.Background())
|
||||
_ = node.Shutdown(context.Background())
|
||||
})
|
||||
return e, node, handler
|
||||
}
|
||||
|
||||
// BenchmarkPostgresStreamBroker_Publish measures the core publish path with
|
||||
// parallel goroutines across many channels (same structure as the map broker
|
||||
// Publish benchmark). This is the steady-state number to report.
|
||||
func BenchmarkPostgresStreamBroker_Publish(b *testing.B) {
|
||||
e, _, _ := newBenchPostgresStreamBroker(b)
|
||||
|
||||
data := []byte("hello world")
|
||||
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
var counter int64
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
i := atomic.AddInt64(&counter, 1)
|
||||
ch := fmt.Sprintf("bench_publish_%d", i%1000)
|
||||
_, err := e.Publish(ch, data, centrifuge.PublishOptions{
|
||||
HistoryTTL: 10 * time.Minute,
|
||||
HistorySize: 1000,
|
||||
})
|
||||
if err != nil {
|
||||
b.Fatalf("publish: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// BenchmarkPostgresStreamBroker_OutboxLatency measures publish→delivery
|
||||
// end-to-end latency: time from Publish() return to HandlePublication() call
|
||||
// in the outbox worker. Reports the average latency via b.ReportMetric.
|
||||
func BenchmarkPostgresStreamBroker_OutboxLatency(b *testing.B) {
|
||||
e, _, handler := newBenchPostgresStreamBroker(b)
|
||||
|
||||
channel := "bench_latency"
|
||||
data := []byte("hello world")
|
||||
|
||||
// Pre-allocate a slice indexed by offset (1-based). The publish loop
|
||||
// stores the publish time at position res.Offset, then unlocks. The
|
||||
// handler reads at pub.Offset. Mutex serializes the store/load against
|
||||
// the publish call so the handler always sees the time before the
|
||||
// outbox worker delivers the row.
|
||||
publishTimes := make([]time.Time, b.N+2)
|
||||
var mu sync.Mutex
|
||||
delivered := make(chan time.Duration, b.N)
|
||||
|
||||
handler.HandlePublicationFunc = func(ch string, pub *centrifuge.Publication, sp centrifuge.StreamPosition, delta bool, prevPub *centrifuge.Publication) error {
|
||||
if ch != channel {
|
||||
return nil
|
||||
}
|
||||
mu.Lock()
|
||||
var t time.Time
|
||||
if int(pub.Offset) < len(publishTimes) {
|
||||
t = publishTimes[pub.Offset]
|
||||
}
|
||||
mu.Unlock()
|
||||
if !t.IsZero() {
|
||||
delivered <- time.Since(t)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
// Hold the lock around publish + array store so the handler sees
|
||||
// the publish time before processing the outbox row.
|
||||
mu.Lock()
|
||||
now := time.Now()
|
||||
res, err := e.Publish(channel, data, centrifuge.PublishOptions{
|
||||
HistoryTTL: 10 * time.Minute,
|
||||
HistorySize: 10000,
|
||||
})
|
||||
if err != nil {
|
||||
mu.Unlock()
|
||||
b.Fatalf("publish: %v", err)
|
||||
}
|
||||
if int(res.Offset) < len(publishTimes) {
|
||||
publishTimes[res.Offset] = now
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
|
||||
// Wait for all deliveries.
|
||||
deadline := time.NewTimer(30 * time.Second)
|
||||
defer deadline.Stop()
|
||||
received := 0
|
||||
totalLatency := time.Duration(0)
|
||||
for received < b.N {
|
||||
select {
|
||||
case d := <-delivered:
|
||||
totalLatency += d
|
||||
received++
|
||||
case <-deadline.C:
|
||||
b.Fatalf("only received %d/%d deliveries within 30s", received, b.N)
|
||||
}
|
||||
}
|
||||
|
||||
b.StopTimer()
|
||||
if b.N > 0 {
|
||||
avg := totalLatency / time.Duration(b.N)
|
||||
b.ReportMetric(float64(avg.Microseconds()), "µs/delivery")
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkPostgresStreamBroker_RetentionDrop measures partition drop time.
|
||||
// Pre-populates b.N partitions each with rowsPerPartition rows, then times
|
||||
// only the DROP TABLE calls (the create + populate happens with the timer
|
||||
// stopped).
|
||||
func BenchmarkPostgresStreamBroker_RetentionDrop(b *testing.B) {
|
||||
e, _, _ := newBenchPostgresStreamBroker(b)
|
||||
ctx := context.Background()
|
||||
|
||||
const rowsPerPartition = 10000
|
||||
yesterday := time.Now().UTC().AddDate(0, 0, -1)
|
||||
partNames := make([]string, b.N)
|
||||
|
||||
// Stop the timer for the setup phase.
|
||||
b.StopTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
partName := fmt.Sprintf("%s_bench_%d_%d", e.names.stream, time.Now().UnixNano(), i)
|
||||
// Use date ranges many years in the past to avoid collisions with
|
||||
// the lookahead-managed partitions.
|
||||
yearsAgo := yesterday.AddDate(-100-i, 0, 0)
|
||||
yearsAgoNext := yearsAgo.AddDate(0, 0, 1)
|
||||
_, err := e.pool.Exec(ctx, fmt.Sprintf(
|
||||
`CREATE TABLE IF NOT EXISTS %s PARTITION OF %s FOR VALUES FROM ('%s') TO ('%s')`,
|
||||
partName, e.names.stream,
|
||||
yearsAgo.Format("2006-01-02 00:00:00+00"),
|
||||
yearsAgoNext.Format("2006-01-02 00:00:00+00"),
|
||||
))
|
||||
require.NoError(b, err)
|
||||
partNames[i] = partName
|
||||
|
||||
// Pre-populate the partition.
|
||||
for r := 0; r < rowsPerPartition; r++ {
|
||||
_, err := e.pool.Exec(ctx, fmt.Sprintf(
|
||||
`INSERT INTO %s (channel, channel_offset, kind, data, created_at, shard_id) VALUES ($1, $2, 0, '\x'::bytea, $3, 0)`,
|
||||
e.names.stream,
|
||||
), fmt.Sprintf("bench_drop_%d_%d", i, r), int64(r), yearsAgo.Add(time.Duration(r)*time.Microsecond))
|
||||
require.NoError(b, err)
|
||||
}
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
b.StartTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := e.pool.Exec(ctx, fmt.Sprintf("DROP TABLE %s", partNames[i]))
|
||||
require.NoError(b, err)
|
||||
}
|
||||
}
|
||||
|
||||
1659
internal/pgstreambroker/pgstreambroker_test.go
Normal file
1659
internal/pgstreambroker/pgstreambroker_test.go
Normal file
File diff suppressed because it is too large
Load Diff
149
internal/pgstreambroker/pgx_helpers.go
Normal file
149
internal/pgstreambroker/pgx_helpers.go
Normal file
@@ -0,0 +1,149 @@
|
||||
//go:build !appengine
|
||||
|
||||
package pgstreambroker
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
|
||||
"github.com/centrifugal/centrifugo/v6/internal/convert"
|
||||
)
|
||||
|
||||
// byteArena is a chunked arena allocator for byte data. It copies incoming
|
||||
// slices into a contiguous buffer and hands back sub-slices (or strings via
|
||||
// unsafe conversion). When the current chunk is full a new one is allocated;
|
||||
// old chunks stay alive via the slices/strings already referencing them.
|
||||
//
|
||||
// Copied from internal/pgmapbroker/pgx_helpers.go. Both packages need this
|
||||
// for zero-allocation row scanning. A future refactor could extract these
|
||||
// helpers into a shared internal package; for now they're duplicated to
|
||||
// keep the packages independent.
|
||||
type byteArena struct {
|
||||
buf []byte
|
||||
}
|
||||
|
||||
// copyBytes copies src into the arena and returns a sub-slice of the arena buffer.
|
||||
func (a *byteArena) copyBytes(src []byte) []byte {
|
||||
n := len(src)
|
||||
if n == 0 {
|
||||
return nil
|
||||
}
|
||||
if cap(a.buf)-len(a.buf) < n {
|
||||
newCap := 2 * cap(a.buf)
|
||||
if newCap < n {
|
||||
newCap = n
|
||||
}
|
||||
if newCap < 4096 {
|
||||
newCap = 4096
|
||||
}
|
||||
a.buf = make([]byte, 0, newCap)
|
||||
}
|
||||
start := len(a.buf)
|
||||
a.buf = append(a.buf, src...)
|
||||
return a.buf[start : start+n : start+n]
|
||||
}
|
||||
|
||||
// copyString copies src into the arena and returns a string backed by arena memory.
|
||||
func (a *byteArena) copyString(src []byte) string {
|
||||
if len(src) == 0 {
|
||||
return ""
|
||||
}
|
||||
b := a.copyBytes(src)
|
||||
return convert.BytesToString(b)
|
||||
}
|
||||
|
||||
// pgTextFormat is the pgx text wire format code.
|
||||
const pgTextFormat int16 = 0
|
||||
|
||||
// pgRawInt64 parses a pgx int8 value as int64.
|
||||
func pgRawInt64(b []byte, format int16) int64 {
|
||||
if b == nil {
|
||||
return 0
|
||||
}
|
||||
if format == pgTextFormat {
|
||||
v, _ := strconv.ParseInt(convert.BytesToString(b), 10, 64)
|
||||
return v
|
||||
}
|
||||
return int64(binary.BigEndian.Uint64(b))
|
||||
}
|
||||
|
||||
// pgRawUint64 parses a pgx int8 value as uint64.
|
||||
func pgRawUint64(b []byte, format int16) uint64 {
|
||||
if b == nil {
|
||||
return 0
|
||||
}
|
||||
if format == pgTextFormat {
|
||||
v, _ := strconv.ParseUint(convert.BytesToString(b), 10, 64)
|
||||
return v
|
||||
}
|
||||
return binary.BigEndian.Uint64(b)
|
||||
}
|
||||
|
||||
// pgRawInt16 parses a pgx int2 (SMALLINT) value as int16.
|
||||
// Handles both binary wire format (2-byte big-endian) and text (decimal string).
|
||||
func pgRawInt16(b []byte, format int16) int16 {
|
||||
if b == nil {
|
||||
return 0
|
||||
}
|
||||
if format == pgTextFormat {
|
||||
v, _ := strconv.ParseInt(convert.BytesToString(b), 10, 16)
|
||||
return int16(v)
|
||||
}
|
||||
return int16(binary.BigEndian.Uint16(b))
|
||||
}
|
||||
|
||||
// pgRawString copies a pgx text value into the arena and returns the resulting
|
||||
// string. TEXT/VARCHAR have identical representation in both binary and text wire
|
||||
// formats, so no format parameter is needed. Returns "" for nil input.
|
||||
func pgRawString(a *byteArena, b []byte) string {
|
||||
if b == nil {
|
||||
return ""
|
||||
}
|
||||
return a.copyString(b)
|
||||
}
|
||||
|
||||
// pgRawBytes copies a pgx BYTEA value into the arena.
|
||||
func pgRawBytes(a *byteArena, b []byte, format int16) []byte {
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
if format == pgTextFormat && len(b) >= 2 && b[0] == '\\' && b[1] == 'x' {
|
||||
decoded := make([]byte, hex.DecodedLen(len(b)-2))
|
||||
n, err := hex.Decode(decoded, b[2:])
|
||||
if err != nil {
|
||||
return a.copyBytes(b) // fallback: copy as-is
|
||||
}
|
||||
return a.copyBytes(decoded[:n])
|
||||
}
|
||||
return a.copyBytes(b)
|
||||
}
|
||||
|
||||
// pgRawJSONBBytes extracts the JSON body from a pgx JSONB value into the arena.
|
||||
func pgRawJSONBBytes(a *byteArena, b []byte, format int16) []byte {
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
if format != pgTextFormat && len(b) > 1 {
|
||||
return a.copyBytes(b[1:]) // strip version byte
|
||||
}
|
||||
return a.copyBytes(b)
|
||||
}
|
||||
|
||||
// pgColFormats holds per-column wire format codes extracted from pgx rows.
|
||||
type pgColFormats []int16
|
||||
|
||||
// pgRawJSONBMap parses a JSONB value into a map[string]string.
|
||||
func pgRawJSONBMap(b []byte) map[string]string {
|
||||
if len(b) == 0 {
|
||||
return nil
|
||||
}
|
||||
data := b
|
||||
if len(b) > 1 && b[0] != '{' && b[0] != '[' && b[0] != 'n' {
|
||||
data = b[1:] // skip version byte
|
||||
}
|
||||
var m map[string]string
|
||||
_ = json.Unmarshal(data, &m)
|
||||
return m
|
||||
}
|
||||
217
internal/pgstreambroker/publish.go
Normal file
217
internal/pgstreambroker/publish.go
Normal file
@@ -0,0 +1,217 @@
|
||||
package pgstreambroker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/centrifugal/centrifuge"
|
||||
)
|
||||
|
||||
// Publish writes a publication to the history table and assigns a stream
|
||||
// position. The publish SQL function does the heavy lifting (atomic shard
|
||||
// lock + insert-or-lock meta + version/idempotency checks + offset increment
|
||||
// + history INSERT + NOTIFY) — this Go-side method just marshals options,
|
||||
// converts Go zero values to NULL pointers where the SQL function uses
|
||||
// COALESCE, and reshapes the result.
|
||||
// Publish is the centrifuge.Broker Publish implementation. Publish count and
|
||||
// latency metrics are provided by the centrifuge library at the Node level —
|
||||
// the PG broker only tracks PG-specific operational metrics (cursor lag,
|
||||
// partition count, cleanup rows, orphan rows).
|
||||
func (e *PostgresStreamBroker) Publish(ch string, data []byte, opts centrifuge.PublishOptions) (centrifuge.PublishResult, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Marshal Tags to JSONB.
|
||||
var tagsJSON any
|
||||
if len(opts.Tags) > 0 {
|
||||
b, err := json.Marshal(opts.Tags)
|
||||
if err != nil {
|
||||
return centrifuge.PublishResult{}, fmt.Errorf("postgres stream broker: marshal tags: %w", err)
|
||||
}
|
||||
tagsJSON = json.RawMessage(b)
|
||||
}
|
||||
|
||||
// HistoryTTL → INTERVAL pointer (nil if 0).
|
||||
var historyTTL *string
|
||||
if opts.HistoryTTL > 0 {
|
||||
s := durationToIntervalString(opts.HistoryTTL)
|
||||
historyTTL = &s
|
||||
}
|
||||
|
||||
// HistorySize → INTEGER pointer (nil if 0).
|
||||
var historySize *int
|
||||
if opts.HistorySize > 0 {
|
||||
n := opts.HistorySize
|
||||
historySize = &n
|
||||
}
|
||||
|
||||
// HistoryMetaTTL with 3-tier fallback: opts → node config → StreamRetention.
|
||||
// The third tier guarantees expires_at is always set, so the meta
|
||||
// cleanup pass eventually fires for every channel.
|
||||
historyMetaTTL := opts.HistoryMetaTTL
|
||||
if historyMetaTTL == 0 {
|
||||
historyMetaTTL = e.node.Config().HistoryMetaTTL
|
||||
}
|
||||
if historyMetaTTL == 0 {
|
||||
historyMetaTTL = e.conf.StreamRetention
|
||||
}
|
||||
metaTTLStr := durationToIntervalString(historyMetaTTL)
|
||||
metaTTL := &metaTTLStr
|
||||
|
||||
// Idempotency key — convert empty string to nil.
|
||||
var idempotencyKey *string
|
||||
var idempotencyTTL *string
|
||||
if opts.IdempotencyKey != "" {
|
||||
idempotencyKey = &opts.IdempotencyKey
|
||||
ttl := opts.IdempotentResultTTL
|
||||
if ttl == 0 {
|
||||
ttl = e.conf.IdempotentResultTTL
|
||||
}
|
||||
s := durationToIntervalString(ttl)
|
||||
idempotencyTTL = &s
|
||||
}
|
||||
|
||||
// ClientInfo handling.
|
||||
var clientID, userID *string
|
||||
var connInfo, chanInfo []byte
|
||||
if opts.ClientInfo != nil {
|
||||
if opts.ClientInfo.ClientID != "" {
|
||||
clientID = &opts.ClientInfo.ClientID
|
||||
}
|
||||
if opts.ClientInfo.UserID != "" {
|
||||
userID = &opts.ClientInfo.UserID
|
||||
}
|
||||
connInfo = opts.ClientInfo.ConnInfo
|
||||
chanInfo = opts.ClientInfo.ChanInfo
|
||||
}
|
||||
|
||||
numShards := e.conf.NumShards
|
||||
|
||||
// Version / VersionEpoch: pointers so the SQL function sees NULL when the
|
||||
// caller didn't request versioned dedup. Zero Version also maps to NULL
|
||||
// so a bare publish can't accidentally clobber the stored baseline.
|
||||
var versionArg *int64
|
||||
var versionEpochArg *string
|
||||
if opts.Version > 0 {
|
||||
v := int64(opts.Version)
|
||||
versionArg = &v
|
||||
if opts.VersionEpoch != "" {
|
||||
versionEpochArg = &opts.VersionEpoch
|
||||
}
|
||||
}
|
||||
|
||||
// Call __PREFIX__publish. Parameter order mirrors pgmapbroker's publish:
|
||||
// channel, data, tags, TTLs, version/epoch, idempotency, delta flag,
|
||||
// client info, num_shards.
|
||||
query := fmt.Sprintf(`
|
||||
SELECT out_result_id, out_channel_offset, out_epoch, out_suppressed, out_suppress_reason
|
||||
FROM %s($1, $2, $3, $4::interval, $5, $6::interval, $7, $8, $9, $10::interval, $11, $12, $13, $14, $15, $16)
|
||||
`, e.names.publish)
|
||||
|
||||
var resultID *int64
|
||||
var channelOffset int64
|
||||
var epoch string
|
||||
var suppressed bool
|
||||
var suppressReason *string
|
||||
|
||||
err := e.pool.QueryRow(ctx, query,
|
||||
ch,
|
||||
e.dataParam(data),
|
||||
tagsJSON,
|
||||
historyTTL,
|
||||
historySize,
|
||||
metaTTL,
|
||||
versionArg,
|
||||
versionEpochArg,
|
||||
idempotencyKey,
|
||||
idempotencyTTL,
|
||||
opts.UseDelta,
|
||||
clientID,
|
||||
userID,
|
||||
e.dataParam(connInfo),
|
||||
e.dataParam(chanInfo),
|
||||
numShards,
|
||||
).Scan(&resultID, &channelOffset, &epoch, &suppressed, &suppressReason)
|
||||
|
||||
if err != nil {
|
||||
return centrifuge.PublishResult{}, fmt.Errorf("postgres stream broker: publish: %w", err)
|
||||
}
|
||||
|
||||
result := centrifuge.PublishResult{
|
||||
StreamPosition: centrifuge.StreamPosition{
|
||||
Offset: uint64(channelOffset),
|
||||
Epoch: epoch,
|
||||
},
|
||||
}
|
||||
if suppressed {
|
||||
result.Suppressed = true
|
||||
if suppressReason != nil {
|
||||
result.SuppressReason = centrifuge.SuppressReason(*suppressReason)
|
||||
}
|
||||
}
|
||||
|
||||
// Contract quirk: for HistorySize<=0 || HistoryTTL<=0, the caller expects
|
||||
// an empty StreamPosition. The SQL function still wrote a row (so it's
|
||||
// delivered via the outbox) but we return zero per the Broker contract.
|
||||
if opts.HistorySize <= 0 || opts.HistoryTTL <= 0 {
|
||||
result.StreamPosition = centrifuge.StreamPosition{}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// PublishJoin publishes a join event. In fanout mode, short-circuits to the
|
||||
// inner Broker (bypassing PG entirely). In non-fanout mode, inserts a kind=1
|
||||
// row via the publish_join SQL function (which acquires the shard lock).
|
||||
func (e *PostgresStreamBroker) PublishJoin(ch string, info *centrifuge.ClientInfo) error {
|
||||
if e.conf.Broker != nil {
|
||||
return e.conf.Broker.PublishJoin(ch, info)
|
||||
}
|
||||
return e.publishJoinViaHistory(ch, info)
|
||||
}
|
||||
|
||||
// PublishLeave mirrors PublishJoin for leave events (kind=2).
|
||||
func (e *PostgresStreamBroker) PublishLeave(ch string, info *centrifuge.ClientInfo) error {
|
||||
if e.conf.Broker != nil {
|
||||
return e.conf.Broker.PublishLeave(ch, info)
|
||||
}
|
||||
return e.publishLeaveViaHistory(ch, info)
|
||||
}
|
||||
|
||||
func (e *PostgresStreamBroker) publishJoinViaHistory(ch string, info *centrifuge.ClientInfo) error {
|
||||
return e.publishJoinLeaveViaHistory(e.names.publishJoin, ch, info)
|
||||
}
|
||||
|
||||
func (e *PostgresStreamBroker) publishLeaveViaHistory(ch string, info *centrifuge.ClientInfo) error {
|
||||
return e.publishJoinLeaveViaHistory(e.names.publishLeave, ch, info)
|
||||
}
|
||||
|
||||
func (e *PostgresStreamBroker) publishJoinLeaveViaHistory(funcName, ch string, info *centrifuge.ClientInfo) error {
|
||||
ctx := context.Background()
|
||||
|
||||
var clientID, userID string
|
||||
var connInfo, chanInfo []byte
|
||||
if info != nil {
|
||||
clientID = info.ClientID
|
||||
userID = info.UserID
|
||||
connInfo = info.ConnInfo
|
||||
chanInfo = info.ChanInfo
|
||||
}
|
||||
|
||||
query := fmt.Sprintf(
|
||||
`SELECT %s($1, $2, $3, $4, $5, $6)`,
|
||||
funcName,
|
||||
)
|
||||
var id int64
|
||||
if err := e.pool.QueryRow(ctx, query,
|
||||
ch,
|
||||
clientID,
|
||||
userID,
|
||||
e.dataParam(connInfo),
|
||||
e.dataParam(chanInfo),
|
||||
e.conf.NumShards,
|
||||
).Scan(&id); err != nil {
|
||||
return fmt.Errorf("postgres stream broker: %s: %w", funcName, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
74
internal/pgstreambroker/sampler.go
Normal file
74
internal/pgstreambroker/sampler.go
Normal file
@@ -0,0 +1,74 @@
|
||||
package pgstreambroker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/centrifugal/centrifugo/v6/internal/metrics"
|
||||
)
|
||||
|
||||
// metricsSampler periodically samples PG-specific gauge metrics: outbox cursor
|
||||
// lag, partition count, orphan rows. Uses the centralized metrics from
|
||||
// internal/metrics package.
|
||||
type metricsSampler struct {
|
||||
broker *PostgresStreamBroker
|
||||
mu sync.Mutex
|
||||
cursorByShard map[int]int64
|
||||
}
|
||||
|
||||
func newMetricsSampler(b *PostgresStreamBroker) *metricsSampler {
|
||||
return &metricsSampler{
|
||||
broker: b,
|
||||
cursorByShard: make(map[int]int64),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *metricsSampler) storeCursor(shard int, cursor int64) {
|
||||
s.mu.Lock()
|
||||
s.cursorByShard[shard] = cursor
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *metricsSampler) sample(ctx context.Context) {
|
||||
if metrics.BrokerPostgresPartitions == nil {
|
||||
return
|
||||
}
|
||||
brokerName := s.broker.conf.Name
|
||||
|
||||
// Sample partition count.
|
||||
var partitionCount int
|
||||
err := s.broker.pool.QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM pg_inherits i
|
||||
JOIN pg_class c ON c.oid = i.inhparent
|
||||
WHERE c.relname = $1
|
||||
`, s.broker.names.stream).Scan(&partitionCount)
|
||||
if err == nil {
|
||||
metrics.BrokerPostgresPartitions.WithLabelValues(brokerName).Set(float64(partitionCount))
|
||||
}
|
||||
|
||||
// Sample cursor lag per shard.
|
||||
s.mu.Lock()
|
||||
cursors := make(map[int]int64, len(s.cursorByShard))
|
||||
for k, v := range s.cursorByShard {
|
||||
cursors[k] = v
|
||||
}
|
||||
s.mu.Unlock()
|
||||
for shard, cursor := range cursors {
|
||||
var createdAt time.Time
|
||||
err := s.broker.pool.QueryRow(ctx, fmt.Sprintf(
|
||||
`SELECT created_at FROM %s WHERE id = $1 LIMIT 1`,
|
||||
s.broker.names.stream,
|
||||
), cursor).Scan(&createdAt)
|
||||
if err != nil {
|
||||
metrics.BrokerPostgresOutboxCursorLagSeconds.WithLabelValues(brokerName, fmt.Sprintf("%d", shard)).Set(0)
|
||||
continue
|
||||
}
|
||||
lag := time.Since(createdAt).Seconds()
|
||||
if lag < 0 {
|
||||
lag = 0
|
||||
}
|
||||
metrics.BrokerPostgresOutboxCursorLagSeconds.WithLabelValues(brokerName, fmt.Sprintf("%d", shard)).Set(lag)
|
||||
}
|
||||
}
|
||||
40
internal/pgstreambroker/schema_template_test.go
Normal file
40
internal/pgstreambroker/schema_template_test.go
Normal file
@@ -0,0 +1,40 @@
|
||||
package pgstreambroker
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// execSchemaWithRetry re-runs the whole batch on a concurrent-DDL conflict,
|
||||
// which is only safe while the batch stays one implicit transaction: no
|
||||
// statement of its own opening or closing a transaction, and nothing that
|
||||
// cannot run inside a transaction block at all (CREATE INDEX CONCURRENTLY
|
||||
// would fail outright with 25001, and a retry could not undo the half of the
|
||||
// batch that already committed). The invariant lives in the SQL template, so
|
||||
// pin it here rather than in a comment.
|
||||
func TestSchemaTemplate_StaysOneImplicitTransaction(t *testing.T) {
|
||||
concurrently := regexp.MustCompile(`(?i)\bCONCURRENTLY\b`)
|
||||
txControl := regexp.MustCompile(`(?im)^\s*(BEGIN|COMMIT|ROLLBACK|START\s+TRANSACTION)\b`)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
prefix string
|
||||
binary bool
|
||||
}{
|
||||
{"jsonb", "cf_stream_", false},
|
||||
{"binary", "cf_binary_stream_", true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
sql := renderSchema(tc.prefix, tc.binary)
|
||||
if loc := concurrently.FindString(sql); loc != "" {
|
||||
t.Errorf("schema template uses %q — it cannot run inside the implicit transaction execSchemaWithRetry relies on", loc)
|
||||
}
|
||||
// Only the DDL half is checked for transaction control: the funcs
|
||||
// half is dollar-quoted PL/pgSQL, where BEGIN opens a block.
|
||||
ddl, _ := splitSchemaSQL(sql)
|
||||
if loc := txControl.FindString(ddl); loc != "" {
|
||||
t.Errorf("schema DDL contains explicit transaction control %q — the batch must be left to the server's implicit transaction", loc)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user