snapshot of https://github.com/centrifugal/centrifugo.git (master, 2026-09-16; blobs > 131072B omitted)
This commit is contained in:
31
internal/controllers/config_test.go
Normal file
31
internal/controllers/config_test.go
Normal file
@@ -0,0 +1,31 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/centrifugal/centrifuge"
|
||||
)
|
||||
|
||||
// A table prefix reaches DDL unquoted, so one PostgreSQL can't parse has to be
|
||||
// caught at construction — otherwise it surfaces much later as a syntax error
|
||||
// naming a generated table rather than the setting behind it.
|
||||
func TestNewPostgresController_RejectsBadTablePrefix(t *testing.T) {
|
||||
node, err := centrifuge.New(centrifuge.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("centrifuge.New: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = node.Shutdown(t.Context()) })
|
||||
|
||||
// Rejected before any connection is attempted, so this needs no Postgres.
|
||||
_, err = NewPostgresController(node, PostgresControllerConfig{
|
||||
DSN: "postgres://test:test@localhost:5432/test",
|
||||
TablePrefix: "cf-prod",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for a prefix PostgreSQL cannot parse unquoted")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "table_prefix") {
|
||||
t.Fatalf("error should name the setting at fault, got: %v", err)
|
||||
}
|
||||
}
|
||||
531
internal/controllers/controller_postgres.go
Normal file
531
internal/controllers/controller_postgres.go
Normal file
@@ -0,0 +1,531 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"context"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/centrifugal/centrifugo/v6/internal/configtypes"
|
||||
"github.com/centrifugal/centrifugo/v6/internal/pgoutbox"
|
||||
|
||||
"github.com/centrifugal/centrifuge"
|
||||
"github.com/centrifugal/centrifugo/v6/internal/pgschema"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
//go:embed controller_postgres_schema.sql
|
||||
var postgresControllerSchemaTemplate string
|
||||
|
||||
// renderControllerTemplate substitutes the placeholders the controller schema
|
||||
// recognises. Used by both the DDL render path (baseline) and (when the first
|
||||
// migration ships) the migration render path — so migrations work with any
|
||||
// user-configured TablePrefix without manual string-juggling per call site.
|
||||
func renderControllerTemplate(template, prefix string) string {
|
||||
return strings.NewReplacer("__PREFIX__", prefix).Replace(template)
|
||||
}
|
||||
|
||||
// tablePrefix returns the normalised `<user-prefix>_` used in this controller's
|
||||
// table and function names.
|
||||
func (c *PostgresController) tablePrefix() string {
|
||||
return strings.TrimRight(c.conf.TablePrefix, "_") + "_"
|
||||
}
|
||||
|
||||
var controllerSchemaVersion = 1
|
||||
|
||||
// PostgresControllerConfig configures the PostgreSQL controller.
|
||||
type PostgresControllerConfig struct {
|
||||
// DSN is the primary PostgreSQL connection string.
|
||||
DSN string
|
||||
|
||||
// TLS is an optional TLS configuration applied to all pools.
|
||||
TLS configtypes.TLSConfig
|
||||
// PoolSize sets the maximum number of connections in the primary pool.
|
||||
// Default: 8.
|
||||
PoolSize int
|
||||
// NumShards is the number of shards for serialized publishing. Each shard
|
||||
// has a lock row; the publish function acquires FOR UPDATE before INSERT,
|
||||
// ensuring BIGSERIAL ids are assigned in commit order within a shard.
|
||||
// Default: 1 (single shard — sufficient for controller traffic).
|
||||
NumShards int
|
||||
// TablePrefix is the user-facing namespace prefix. Default "cf".
|
||||
// Produces table names like cf_controller_messages.
|
||||
TablePrefix string
|
||||
// PollInterval is how often to poll for new control messages when idle.
|
||||
// Default: 50ms.
|
||||
PollInterval time.Duration
|
||||
// UseNotify enables LISTEN/NOTIFY for low-latency wakeup.
|
||||
// Default: true.
|
||||
UseNotify bool
|
||||
// NotifyDSN is an optional separate connection string used exclusively for
|
||||
// the LISTEN connection when UseNotify is true. Set this to a direct
|
||||
// PostgreSQL URL (bypassing PGBouncer) when DSN points at a PGBouncer
|
||||
// endpoint — PGBouncer transaction pooling mode is incompatible with
|
||||
// LISTEN/NOTIFY. If empty, the primary DSN pool is used.
|
||||
NotifyDSN string
|
||||
// PartitionRetentionDays controls how old a partition must be before
|
||||
// it is dropped. Default: 1 (control messages are ephemeral).
|
||||
PartitionRetentionDays int
|
||||
// PartitionLookaheadDays controls how many future daily partitions
|
||||
// to pre-create. Default: 2.
|
||||
PartitionLookaheadDays int
|
||||
// PartitionCleanupInterval is the ticker interval for the partitioner.
|
||||
// Default: 1m.
|
||||
PartitionCleanupInterval time.Duration
|
||||
// BatchSize is the maximum number of rows to process per batch.
|
||||
// Default: 1000.
|
||||
BatchSize int
|
||||
}
|
||||
|
||||
func (c *PostgresControllerConfig) setDefaults() {
|
||||
if c.PoolSize <= 0 {
|
||||
c.PoolSize = 8
|
||||
}
|
||||
if c.NumShards <= 0 {
|
||||
c.NumShards = 1
|
||||
}
|
||||
if c.TablePrefix == "" {
|
||||
c.TablePrefix = "cf"
|
||||
}
|
||||
if c.PollInterval <= 0 {
|
||||
c.PollInterval = 50 * time.Millisecond
|
||||
}
|
||||
if c.PartitionLookaheadDays <= 0 {
|
||||
c.PartitionLookaheadDays = 2
|
||||
}
|
||||
if c.PartitionRetentionDays <= 0 {
|
||||
c.PartitionRetentionDays = 1
|
||||
}
|
||||
if c.PartitionCleanupInterval <= 0 {
|
||||
c.PartitionCleanupInterval = time.Minute
|
||||
}
|
||||
if c.BatchSize <= 0 {
|
||||
c.BatchSize = 1000
|
||||
}
|
||||
}
|
||||
|
||||
// controllerNames holds precomputed table/function/channel names.
|
||||
type controllerNames struct {
|
||||
messages string // e.g. cf_controller_messages
|
||||
shardLock string // e.g. cf_controller_shard_lock
|
||||
schemaVersion string // e.g. cf_controller_schema_version
|
||||
publishFunc string // e.g. cf_controller_publish
|
||||
notifyChannel string // e.g. cf_controller_notify
|
||||
}
|
||||
|
||||
func newControllerNames(prefix string) controllerNames {
|
||||
// Normalize: ensure prefix ends with "_".
|
||||
p := strings.TrimRight(prefix, "_") + "_"
|
||||
return controllerNames{
|
||||
messages: p + "controller_messages",
|
||||
shardLock: p + "controller_shard_lock",
|
||||
schemaVersion: p + "controller_schema_version",
|
||||
publishFunc: p + "controller_publish",
|
||||
notifyChannel: p + "controller_notify",
|
||||
}
|
||||
}
|
||||
|
||||
// PostgresController implements centrifuge.Controller using PostgreSQL.
|
||||
type PostgresController struct {
|
||||
node *centrifuge.Node
|
||||
conf PostgresControllerConfig
|
||||
pool *pgxpool.Pool
|
||||
notifyPool *pgxpool.Pool // Dedicated single-conn pool for LISTEN; nil = use pool
|
||||
names controllerNames
|
||||
eventHandler centrifuge.ControlEventHandler
|
||||
myNodeID string
|
||||
closeCh chan struct{}
|
||||
closeOnce sync.Once
|
||||
cancelCtx context.Context
|
||||
cancelFunc context.CancelFunc
|
||||
notifyCh chan struct{}
|
||||
|
||||
// notifyListenerReady flips true after the LISTEN command succeeds. Used
|
||||
// by tests that assert NOTIFY-driven latency — a publish whose NOTIFY
|
||||
// fires before LISTEN runs would be dropped and the test would observe
|
||||
// PollInterval instead. Production tolerates this race because
|
||||
// PollInterval is the fallback.
|
||||
notifyListenerReady atomic.Bool
|
||||
}
|
||||
|
||||
var _ centrifuge.Controller = (*PostgresController)(nil)
|
||||
|
||||
func NewPostgresController(node *centrifuge.Node, conf PostgresControllerConfig) (*PostgresController, error) {
|
||||
conf.setDefaults()
|
||||
|
||||
if err := pgschema.ValidateTablePrefix("postgres controller", conf.TablePrefix); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
poolConfig, err := pgxpool.ParseConfig(conf.DSN)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing postgres DSN: %w", err)
|
||||
}
|
||||
poolConfig.MaxConns = int32(conf.PoolSize)
|
||||
if conf.TLS.Enabled {
|
||||
tlsCfg, err := conf.TLS.ToGoTLSConfig("postgres-controller")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("postgres controller: TLS config: %w", err)
|
||||
}
|
||||
poolConfig.ConnConfig.TLSConfig = tlsCfg
|
||||
}
|
||||
pool, err := pgxpool.NewWithConfig(ctx, poolConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating postgres pool: %w", err)
|
||||
}
|
||||
|
||||
cancelCtx, cancelFunc := context.WithCancel(context.Background())
|
||||
|
||||
c := &PostgresController{
|
||||
node: node,
|
||||
conf: conf,
|
||||
pool: pool,
|
||||
names: newControllerNames(conf.TablePrefix),
|
||||
closeCh: make(chan struct{}),
|
||||
cancelCtx: cancelCtx,
|
||||
cancelFunc: cancelFunc,
|
||||
notifyCh: make(chan struct{}, 1),
|
||||
}
|
||||
|
||||
if conf.NotifyDSN != "" {
|
||||
nCfg, err := pgxpool.ParseConfig(conf.NotifyDSN)
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
cancelFunc()
|
||||
return nil, fmt.Errorf("error parsing notify DSN: %w", err)
|
||||
}
|
||||
nCfg.MaxConns = 1
|
||||
if conf.TLS.Enabled {
|
||||
tlsCfg, err := conf.TLS.ToGoTLSConfig("postgres-controller")
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
cancelFunc()
|
||||
return nil, fmt.Errorf("postgres controller: notify TLS config: %w", err)
|
||||
}
|
||||
nCfg.ConnConfig.TLSConfig = tlsCfg
|
||||
}
|
||||
nPool, err := pgxpool.NewWithConfig(ctx, nCfg)
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
cancelFunc()
|
||||
return nil, fmt.Errorf("error creating notify pool: %w", err)
|
||||
}
|
||||
c.notifyPool = nPool
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (c *PostgresController) logError(msg string, err error) {
|
||||
log.Error().Err(err).Str("controller", "postgres").Msg(msg)
|
||||
}
|
||||
|
||||
// EnsureSchema creates the required database objects idempotently.
|
||||
// reconcileShardLock ensures the shard_lock table has exactly one row for
|
||||
// each shard in [0, NumShards). Called on every EnsureSchema including the
|
||||
// fast path so a NumShards change between runs is picked up. A stale
|
||||
// shard_lock is load-bearing, not cosmetic: a missing row makes
|
||||
// `PERFORM 1 FROM shard_lock WHERE shard_id = X FOR UPDATE` lock nothing,
|
||||
// breaking per-shard publish serialization and letting the outbox cursor
|
||||
// skip rows whose transactions committed out of BIGSERIAL id order.
|
||||
func (c *PostgresController) reconcileShardLock(ctx context.Context) error {
|
||||
if _, err := c.pool.Exec(ctx, fmt.Sprintf(
|
||||
`INSERT INTO %s (shard_id) SELECT generate_series(0, $1 - 1) ON CONFLICT DO NOTHING`,
|
||||
c.names.shardLock), c.conf.NumShards); err != nil {
|
||||
return fmt.Errorf("postgres controller: populate shard_lock: %w", err)
|
||||
}
|
||||
if _, err := c.pool.Exec(ctx, fmt.Sprintf(
|
||||
`DELETE FROM %s WHERE shard_id >= $1`,
|
||||
c.names.shardLock), c.conf.NumShards); err != nil {
|
||||
return fmt.Errorf("postgres controller: trim shard_lock: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *PostgresController) EnsureSchema(ctx context.Context) error {
|
||||
const label = "postgres-controller"
|
||||
|
||||
// Read schema_version with error discrimination — see pgschema.ReadSchemaVersion.
|
||||
dbVersion, isFresh, err := pgschema.ReadSchemaVersion(ctx, c.pool, c.names.schemaVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Fast path: version current AND messages table exists AND partition shape ok.
|
||||
if !isFresh && dbVersion == controllerSchemaVersion {
|
||||
if _, probeErr := c.pool.Exec(ctx, fmt.Sprintf(
|
||||
`SELECT 1 FROM %s LIMIT 0`, c.names.messages)); probeErr == nil {
|
||||
if err := c.verifyPartitionedShape(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.reconcileShardLock(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return c.ensureInitialPartitions(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// Reject downgrade.
|
||||
if !isFresh {
|
||||
if err := pgschema.CheckDowngrade(label, dbVersion, controllerSchemaVersion); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// The controller has no migrations registered today (controllerSchemaVersion=1).
|
||||
// When the first one ships it goes here, mirroring pgmapbroker/pgstreambroker
|
||||
// — pgschema.ApplyMigrationInTx per version with a single MigrationVariant
|
||||
// (controller has just one prefix), before the DDL render below. Migration
|
||||
// SQL should use the same __PREFIX__ placeholder as the schema template
|
||||
// so it works with any user-configured TablePrefix.
|
||||
|
||||
// Render schema template.
|
||||
prefix := c.tablePrefix()
|
||||
schemaSQL := renderControllerTemplate(postgresControllerSchemaTemplate, prefix)
|
||||
|
||||
// Split DDL from functions.
|
||||
ddl, funcs := splitControllerSchemaSQL(schemaSQL)
|
||||
for _, sql := range []string{ddl, funcs} {
|
||||
if sql == "" {
|
||||
continue
|
||||
}
|
||||
if err := c.execSchemaWithRetry(ctx, sql); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := c.reconcileShardLock(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.verifyPartitionedShape(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.ensureInitialPartitions(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Final schema_version write — fatal on failure (see pgmapbroker for why).
|
||||
return pgschema.SetSchemaVersion(ctx, c.pool, label, controllerSchemaVersion, []string{c.names.schemaVersion})
|
||||
}
|
||||
|
||||
// splitControllerSchemaSQL separates DDL from function definitions.
|
||||
func splitControllerSchemaSQL(sql string) (ddl, funcs string) {
|
||||
const marker = "CREATE OR REPLACE FUNCTION"
|
||||
i := strings.Index(sql, marker)
|
||||
if i < 0 {
|
||||
return sql, ""
|
||||
}
|
||||
return sql[:i], sql[i:]
|
||||
}
|
||||
|
||||
// execSchemaWithRetry executes idempotent schema SQL, retrying the batch on
|
||||
// the conflicts several nodes running the same DDL at once produce — see
|
||||
// pgschema.RetrySchemaExec for the policy and for the invariant the SQL has
|
||||
// to keep (one multi-statement Exec with no arguments, so the server runs it
|
||||
// in a single implicit transaction that rolls back completely on failure).
|
||||
func (c *PostgresController) execSchemaWithRetry(ctx context.Context, sql string) error {
|
||||
err := pgschema.RetrySchemaExec(ctx, func(ctx context.Context) error {
|
||||
_, err := c.pool.Exec(ctx, sql)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("postgres controller: schema exec: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *PostgresController) verifyPartitionedShape(ctx context.Context) error {
|
||||
var isPartitioned bool
|
||||
err := c.pool.QueryRow(ctx, `
|
||||
SELECT EXISTS(
|
||||
SELECT 1 FROM pg_partitioned_table
|
||||
WHERE partrelid = $1::regclass
|
||||
)
|
||||
`, c.names.messages).Scan(&isPartitioned)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if !isPartitioned {
|
||||
return fmt.Errorf(
|
||||
"postgres controller: %s exists but is not partitioned; "+
|
||||
"drop the existing tables manually: DROP TABLE IF EXISTS %s, %s, %s CASCADE",
|
||||
c.names.messages, c.names.messages, c.names.shardLock, c.names.schemaVersion,
|
||||
)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *PostgresController) ensureInitialPartitions(ctx context.Context) error {
|
||||
p := c.newPartitioner()
|
||||
return p.EnsureLookaheadPartitions(ctx)
|
||||
}
|
||||
|
||||
func (c *PostgresController) newPartitioner() *pgoutbox.Partitioner {
|
||||
return &pgoutbox.Partitioner{
|
||||
Pool: c.pool,
|
||||
ParentTable: c.names.messages,
|
||||
CleanupInterval: c.conf.PartitionCleanupInterval,
|
||||
LookaheadDays: c.conf.PartitionLookaheadDays,
|
||||
RetentionDays: c.conf.PartitionRetentionDays,
|
||||
ErrorFn: c.logError,
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterControlEventHandler is called once during node.Run(). It stores
|
||||
// the event handler and starts the background workers (outbox poller,
|
||||
// notification listener, partitioner).
|
||||
func (c *PostgresController) RegisterControlEventHandler(h centrifuge.ControlEventHandler) error {
|
||||
c.eventHandler = h
|
||||
c.myNodeID = c.node.ID()
|
||||
|
||||
// Pre-initialize the outbox cursor before launching worker goroutines so
|
||||
// that any message published after RegisterControlEventHandler returns is
|
||||
// guaranteed to be delivered. Without this, a goroutine scheduled late
|
||||
// could call initCursor after a message was already inserted, see that ID
|
||||
// as MAX(id), and silently skip it (observed as flaky broadcast tests in
|
||||
// CI). Same pattern as pgmapbroker/pgstreambroker.
|
||||
initialCursor, err := c.initCursor(c.cancelCtx, c.pool)
|
||||
if err != nil {
|
||||
c.logError("pre-init outbox cursor", err)
|
||||
initialCursor = 0
|
||||
}
|
||||
|
||||
// Start one outbox worker per shard. With NumShards=1 (default),
|
||||
// a single worker polls all messages.
|
||||
for i := 0; i < c.conf.NumShards; i++ {
|
||||
go c.runOutboxWorker(i, initialCursor)
|
||||
}
|
||||
|
||||
// Start notification listener if enabled.
|
||||
if c.conf.UseNotify {
|
||||
go c.runNotificationListener()
|
||||
}
|
||||
|
||||
// Start partition maintenance.
|
||||
go c.newPartitioner().Run(c.cancelCtx, c.closeCh)
|
||||
|
||||
log.Info().Str("controller", "postgres").Msg("postgres controller running")
|
||||
return nil
|
||||
}
|
||||
|
||||
// PublishControl sends a control message. If nodeID is empty, the message
|
||||
// is broadcast to all nodes. If nodeID is set, only that node will process it.
|
||||
func (c *PostgresController) PublishControl(data []byte, nodeID, _ string) error {
|
||||
_, err := c.pool.Exec(c.cancelCtx,
|
||||
fmt.Sprintf(`SELECT %s($1, $2, $3)`, c.names.publishFunc),
|
||||
data, nodeID, c.conf.NumShards,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("postgres controller: publish: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Run blocks until ctx is cancelled, then closes pools.
|
||||
func (c *PostgresController) Run(ctx context.Context) error {
|
||||
<-ctx.Done()
|
||||
c.closeOnce.Do(func() {
|
||||
close(c.closeCh)
|
||||
c.cancelFunc()
|
||||
if c.notifyPool != nil {
|
||||
c.notifyPool.Close()
|
||||
}
|
||||
c.pool.Close()
|
||||
})
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// initCursor bootstraps the cursor from MAX(id) of the messages table.
|
||||
func (c *PostgresController) initCursor(ctx context.Context, pool *pgxpool.Pool) (int64, error) {
|
||||
var cursor int64
|
||||
err := pool.QueryRow(ctx, fmt.Sprintf(
|
||||
`SELECT COALESCE(MAX(id), 0) FROM %s`, c.names.messages)).Scan(&cursor)
|
||||
return cursor, err
|
||||
}
|
||||
|
||||
// runOutboxWorker polls the messages table for new control messages
|
||||
// belonging to the given shard. initialCursor is captured by the InitCursor
|
||||
// closure so the worker does not re-query MAX(id) inside its goroutine —
|
||||
// see RegisterControlEventHandler for the race this avoids.
|
||||
func (c *PostgresController) runOutboxWorker(shardIdx int, initialCursor int64) {
|
||||
w := &pgoutbox.Worker{
|
||||
Pool: c.pool,
|
||||
ShardIDs: []int{shardIdx},
|
||||
PollInterval: c.conf.PollInterval,
|
||||
NotifyCh: c.notifyCh,
|
||||
InitCursor: func(ctx context.Context, p *pgxpool.Pool) (int64, error) {
|
||||
return initialCursor, nil
|
||||
},
|
||||
ProcessBatch: c.processControlBatch,
|
||||
ErrorFn: c.logError,
|
||||
}
|
||||
w.Run(c.cancelCtx, c.closeCh)
|
||||
}
|
||||
|
||||
// runNotificationListener listens for NOTIFY on the controller channel.
|
||||
func (c *PostgresController) runNotificationListener() {
|
||||
pool := c.pool
|
||||
if c.notifyPool != nil {
|
||||
pool = c.notifyPool
|
||||
}
|
||||
l := &pgoutbox.NotificationListener{
|
||||
Pool: pool,
|
||||
Channel: c.names.notifyChannel,
|
||||
NotifyCh: c.notifyCh,
|
||||
ErrorFn: c.logError,
|
||||
OnReady: func() { c.notifyListenerReady.Store(true) },
|
||||
}
|
||||
l.Run(c.cancelCtx, c.closeCh)
|
||||
}
|
||||
|
||||
// processControlBatch reads a batch of control messages from the outbox
|
||||
// and dispatches them to the event handler.
|
||||
func (c *PostgresController) processControlBatch(
|
||||
ctx context.Context, pool *pgxpool.Pool, cursor int64, shardIDs []int,
|
||||
) (int, int64, error) {
|
||||
rows, err := pool.Query(ctx, fmt.Sprintf(`
|
||||
SELECT id, node_id, data
|
||||
FROM %s
|
||||
WHERE id > $1 AND shard_id = ANY($3)
|
||||
ORDER BY id
|
||||
LIMIT $2
|
||||
`, c.names.messages), cursor, c.conf.BatchSize, shardIDs)
|
||||
if err != nil {
|
||||
return 0, cursor, fmt.Errorf("postgres controller: outbox query: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
count := 0
|
||||
maxID := cursor
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var nodeID string
|
||||
var data []byte
|
||||
if err := rows.Scan(&id, &nodeID, &data); err != nil {
|
||||
return count, maxID, fmt.Errorf("postgres controller: outbox scan: %w", err)
|
||||
}
|
||||
if id > maxID {
|
||||
maxID = id
|
||||
}
|
||||
count++
|
||||
|
||||
// Filter targeted messages.
|
||||
if nodeID != "" && nodeID != c.myNodeID {
|
||||
continue
|
||||
}
|
||||
if err := c.eventHandler.HandleControl(data); err != nil {
|
||||
c.logError("handle control message", err)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return count, maxID, fmt.Errorf("postgres controller: outbox rows: %w", err)
|
||||
}
|
||||
|
||||
return count, maxID, nil
|
||||
}
|
||||
78
internal/controllers/controller_postgres_schema.sql
Normal file
78
internal/controllers/controller_postgres_schema.sql
Normal file
@@ -0,0 +1,78 @@
|
||||
-- PostgreSQL Controller schema template.
|
||||
-- Placeholders: __PREFIX__ → e.g. "cf_" (includes trailing underscore).
|
||||
--
|
||||
-- Tables:
|
||||
-- __PREFIX__controller_messages — control message outbox (partitioned by created_at, daily)
|
||||
-- __PREFIX__controller_shard_lock — per-shard serialization lock rows
|
||||
-- __PREFIX__controller_schema_version — schema version tracking
|
||||
--
|
||||
-- Functions:
|
||||
-- __PREFIX__controller_publish(BYTEA, TEXT, INTEGER) — shard lock + INSERT + NOTIFY
|
||||
|
||||
-- Messages outbox table — always partitioned by created_at (daily).
|
||||
CREATE TABLE IF NOT EXISTS __PREFIX__controller_messages (
|
||||
id BIGSERIAL,
|
||||
shard_id SMALLINT NOT NULL DEFAULT 0,
|
||||
node_id TEXT NOT NULL DEFAULT '',
|
||||
data BYTEA NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (id, created_at)
|
||||
) PARTITION BY RANGE (created_at);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS __PREFIX__controller_messages_shard_id_idx
|
||||
ON __PREFIX__controller_messages (shard_id, id);
|
||||
|
||||
-- Per-shard serialization lock. Rows populated by EnsureSchema (0..NumShards-1).
|
||||
-- The publish function acquires a FOR UPDATE lock on the relevant row before
|
||||
-- INSERT, ensuring BIGSERIAL id assignment matches commit order within a shard.
|
||||
CREATE TABLE IF NOT EXISTS __PREFIX__controller_shard_lock (
|
||||
shard_id SMALLINT PRIMARY KEY
|
||||
);
|
||||
|
||||
-- Schema version tracking.
|
||||
CREATE TABLE IF NOT EXISTS __PREFIX__controller_schema_version (
|
||||
id INTEGER PRIMARY KEY,
|
||||
schema_version INTEGER NOT NULL
|
||||
);
|
||||
INSERT INTO __PREFIX__controller_schema_version (id, schema_version) VALUES (1, 1)
|
||||
ON CONFLICT (id) DO NOTHING;
|
||||
|
||||
-- Publish function: shard lock + INSERT + NOTIFY.
|
||||
-- Acquires a FOR UPDATE lock on the shard_lock row for the computed shard,
|
||||
-- serializing concurrent publishes within the same shard. This ensures the
|
||||
-- BIGSERIAL id order matches commit order — no gaps for cursor-based polling.
|
||||
CREATE OR REPLACE FUNCTION __PREFIX__controller_publish(
|
||||
p_data BYTEA,
|
||||
p_node_id TEXT DEFAULT '',
|
||||
p_num_shards INTEGER DEFAULT NULL
|
||||
) RETURNS BIGINT AS $$
|
||||
DECLARE
|
||||
v_id BIGINT;
|
||||
v_shard_id INTEGER;
|
||||
BEGIN
|
||||
-- Auto-derive num_shards from shard_lock table when not provided.
|
||||
IF p_num_shards IS NULL OR p_num_shards <= 0 THEN
|
||||
SELECT COUNT(*)::INTEGER INTO p_num_shards FROM __PREFIX__controller_shard_lock;
|
||||
END IF;
|
||||
|
||||
-- All messages go to shard 0 when num_shards=1. With multiple shards,
|
||||
-- distribute by hash of node_id (targeted) or round-robin via txid (broadcast).
|
||||
IF p_num_shards <= 1 THEN
|
||||
v_shard_id := 0;
|
||||
ELSIF p_node_id != '' THEN
|
||||
v_shard_id := abs(hashtext(p_node_id)::bigint) % p_num_shards;
|
||||
ELSE
|
||||
v_shard_id := (txid_current() % p_num_shards)::INTEGER;
|
||||
END IF;
|
||||
|
||||
-- Serialize within shard.
|
||||
PERFORM 1 FROM __PREFIX__controller_shard_lock WHERE shard_id = v_shard_id FOR UPDATE;
|
||||
|
||||
INSERT INTO __PREFIX__controller_messages (shard_id, node_id, data)
|
||||
VALUES (v_shard_id, p_node_id, p_data)
|
||||
RETURNING id INTO v_id;
|
||||
|
||||
PERFORM pg_notify('__PREFIX__controller_notify', '');
|
||||
RETURN v_id;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
792
internal/controllers/controller_postgres_test.go
Normal file
792
internal/controllers/controller_postgres_test.go
Normal file
@@ -0,0 +1,792 @@
|
||||
//go:build integration
|
||||
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/centrifugal/centrifuge"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func getPostgresConnString(tb testing.TB) string {
|
||||
connString := os.Getenv("CENTRIFUGE_POSTGRES_URL")
|
||||
if connString == "" {
|
||||
connString = "postgres://test:test@localhost:5432/test?sslmode=disable"
|
||||
}
|
||||
return connString
|
||||
}
|
||||
|
||||
// testControlEventHandler implements centrifuge.ControlEventHandler for tests.
|
||||
type testControlEventHandler struct {
|
||||
HandleControlFunc func(data []byte) error
|
||||
}
|
||||
|
||||
func (h *testControlEventHandler) HandleControl(data []byte) error {
|
||||
if h.HandleControlFunc != nil {
|
||||
return h.HandleControlFunc(data)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// initializedPrefixes tracks table prefixes already dropped+created in this
|
||||
// test process. Tests that share a prefix between two controllers (e.g.
|
||||
// broadcast-style tests) must not have the second helper drop tables that
|
||||
// the first controller's outbox worker is already polling — doing so causes
|
||||
// transient `relation does not exist` errors and lost messages.
|
||||
var (
|
||||
initializedPrefixesMu sync.Mutex
|
||||
initializedPrefixes = map[string]bool{}
|
||||
)
|
||||
|
||||
// newTestPostgresController creates a controller with a unique table prefix
|
||||
// for test isolation, runs EnsureSchema, and registers the handler to start
|
||||
// background workers. Does NOT call node.Run() to avoid the node's internal
|
||||
// heartbeat/control message traffic polluting test assertions.
|
||||
func newTestPostgresController(tb testing.TB, conf PostgresControllerConfig, handler *testControlEventHandler) *PostgresController {
|
||||
tb.Helper()
|
||||
connString := getPostgresConnString(tb)
|
||||
|
||||
if conf.DSN == "" {
|
||||
conf.DSN = connString
|
||||
}
|
||||
if conf.TablePrefix == "" {
|
||||
conf.TablePrefix = fmt.Sprintf("test_%d", time.Now().UnixNano()%100000)
|
||||
}
|
||||
if conf.PollInterval <= 0 {
|
||||
conf.PollInterval = 10 * time.Millisecond
|
||||
}
|
||||
conf.UseNotify = true
|
||||
|
||||
node, err := centrifuge.New(centrifuge.Config{})
|
||||
require.NoError(tb, err)
|
||||
|
||||
c, err := NewPostgresController(node, conf)
|
||||
require.NoError(tb, err)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Drop schema only the first time we see this prefix in the test process.
|
||||
// Subsequent controllers for the same prefix attach to the existing schema
|
||||
// via EnsureSchema (idempotent fast-path) so we don't yank tables out from
|
||||
// under another controller's running workers.
|
||||
initializedPrefixesMu.Lock()
|
||||
firstForPrefix := !initializedPrefixes[conf.TablePrefix]
|
||||
if firstForPrefix {
|
||||
initializedPrefixes[conf.TablePrefix] = true
|
||||
}
|
||||
initializedPrefixesMu.Unlock()
|
||||
if firstForPrefix {
|
||||
dropTestControllerSchema(tb, c)
|
||||
}
|
||||
require.NoError(tb, c.EnsureSchema(ctx))
|
||||
cleanupTestControllerMessages(ctx, c)
|
||||
|
||||
// Register the handler directly (starts outbox worker, listener, partitioner).
|
||||
if handler != nil {
|
||||
require.NoError(tb, c.RegisterControlEventHandler(handler))
|
||||
}
|
||||
|
||||
tb.Cleanup(func() {
|
||||
c.closeOnce.Do(func() {
|
||||
close(c.closeCh)
|
||||
c.cancelFunc()
|
||||
c.pool.Close()
|
||||
})
|
||||
_ = node.Shutdown(context.Background())
|
||||
})
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func dropTestControllerSchema(tb testing.TB, c *PostgresController) {
|
||||
tb.Helper()
|
||||
ctx := context.Background()
|
||||
for _, tbl := range []string{c.names.messages, c.names.shardLock, c.names.schemaVersion} {
|
||||
_, _ = c.pool.Exec(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE", tbl))
|
||||
}
|
||||
_, _ = c.pool.Exec(ctx, fmt.Sprintf("DROP FUNCTION IF EXISTS %s", c.names.publishFunc))
|
||||
}
|
||||
|
||||
func cleanupTestControllerMessages(ctx context.Context, c *PostgresController) {
|
||||
rows, err := c.pool.Query(ctx, `
|
||||
SELECT c.relname
|
||||
FROM pg_inherits i
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
JOIN pg_class pp ON pp.oid = i.inhparent
|
||||
WHERE pp.relname = $1
|
||||
`, c.names.messages)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
var parts []string
|
||||
for rows.Next() {
|
||||
var name string
|
||||
if err := rows.Scan(&name); err == nil {
|
||||
parts = append(parts, name)
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
for _, part := range parts {
|
||||
_, _ = c.pool.Exec(ctx, fmt.Sprintf("TRUNCATE %s", part))
|
||||
}
|
||||
}
|
||||
|
||||
func waitForMessages(t *testing.T, counter *atomic.Int64, expectedCount int64, timeout time.Duration) {
|
||||
t.Helper()
|
||||
deadline := time.After(timeout)
|
||||
for {
|
||||
if counter.Load() >= expectedCount {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatalf("timeout: expected %d messages, got %d", expectedCount, counter.Load())
|
||||
case <-time.After(5 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPostgresController_EnsureSchema_ConcurrentNodes reproduces a cluster
|
||||
// whose nodes all start against the same empty database — the shape of a
|
||||
// fresh install rolled out to a whole deployment at once. CREATE TABLE/INDEX
|
||||
// IF NOT EXISTS probes the catalog before it takes any lock, so every node
|
||||
// but the winner loses the race and gets a hard error (42P07, 42710 on the
|
||||
// table's row type, or 23505 on pg_type/pg_class when it reached its own
|
||||
// catalog inserts first) instead of
|
||||
// the NOTICE a sequential re-run produces. EnsureSchema returning that error
|
||||
// means the node refuses to start, so every node here must come up clean.
|
||||
//
|
||||
// The race is timing-dependent and won't fire on every run — the test is a
|
||||
// regression guard, not a reproducer: it must never fail.
|
||||
func TestPostgresController_EnsureSchema_ConcurrentNodes(t *testing.T) {
|
||||
const (
|
||||
nodes = 8
|
||||
rounds = 3
|
||||
)
|
||||
ctx := context.Background()
|
||||
prefix := fmt.Sprintf("test_concurrent_%d", time.Now().UnixNano()%100000)
|
||||
|
||||
// Separate controllers with separate pools — one per simulated node.
|
||||
controllers := make([]*PostgresController, 0, nodes)
|
||||
for range nodes {
|
||||
node, err := centrifuge.New(centrifuge.Config{})
|
||||
require.NoError(t, err)
|
||||
c, err := NewPostgresController(node, PostgresControllerConfig{
|
||||
DSN: getPostgresConnString(t),
|
||||
TablePrefix: prefix,
|
||||
PollInterval: time.Hour,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
c.closeOnce.Do(func() {
|
||||
close(c.closeCh)
|
||||
c.cancelFunc()
|
||||
c.pool.Close()
|
||||
})
|
||||
_ = node.Shutdown(context.Background())
|
||||
})
|
||||
controllers = append(controllers, c)
|
||||
}
|
||||
|
||||
for round := range rounds {
|
||||
dropTestControllerSchema(t, controllers[0])
|
||||
|
||||
start := make(chan struct{})
|
||||
errs := make([]error, nodes)
|
||||
var wg sync.WaitGroup
|
||||
for i, c := range controllers {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
errs[i] = c.EnsureSchema(ctx)
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
|
||||
for i, err := range errs {
|
||||
require.NoErrorf(t, err, "round %d: node %d failed to bring up the schema", round, i)
|
||||
}
|
||||
for _, table := range []string{
|
||||
controllers[0].names.messages, controllers[0].names.shardLock, controllers[0].names.schemaVersion,
|
||||
} {
|
||||
var reg *string
|
||||
require.NoError(t, controllers[0].pool.QueryRow(ctx, "SELECT to_regclass($1)::text", table).Scan(®))
|
||||
require.NotNilf(t, reg, "round %d: table %s missing after concurrent EnsureSchema", round, table)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresController_PublishAndReceive(t *testing.T) {
|
||||
var received atomic.Int64
|
||||
var receivedData []byte
|
||||
var mu sync.Mutex
|
||||
doneCh := make(chan struct{})
|
||||
|
||||
handler := &testControlEventHandler{
|
||||
HandleControlFunc: func(data []byte) error {
|
||||
mu.Lock()
|
||||
receivedData = append([]byte{}, data...)
|
||||
mu.Unlock()
|
||||
if received.Add(1) == 1 {
|
||||
close(doneCh)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
c := newTestPostgresController(t, PostgresControllerConfig{}, handler)
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
testData := []byte("hello-control-message")
|
||||
require.NoError(t, c.PublishControl(testData, "", ""))
|
||||
|
||||
select {
|
||||
case <-doneCh:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timeout waiting for control message")
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
require.Equal(t, testData, receivedData)
|
||||
mu.Unlock()
|
||||
}
|
||||
|
||||
func TestPostgresController_BroadcastReachesAll(t *testing.T) {
|
||||
connString := getPostgresConnString(t)
|
||||
prefix := fmt.Sprintf("test_%d", time.Now().UnixNano()%100000)
|
||||
|
||||
var count1, count2 atomic.Int64
|
||||
done1 := make(chan struct{})
|
||||
done2 := make(chan struct{})
|
||||
|
||||
h1 := &testControlEventHandler{
|
||||
HandleControlFunc: func(data []byte) error {
|
||||
if count1.Add(1) == 1 {
|
||||
close(done1)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
h2 := &testControlEventHandler{
|
||||
HandleControlFunc: func(data []byte) error {
|
||||
if count2.Add(1) == 1 {
|
||||
close(done2)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
c1 := newTestPostgresController(t, PostgresControllerConfig{
|
||||
DSN: connString,
|
||||
TablePrefix: prefix,
|
||||
}, h1)
|
||||
_ = newTestPostgresController(t, PostgresControllerConfig{
|
||||
DSN: connString,
|
||||
TablePrefix: prefix,
|
||||
}, h2)
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
require.NoError(t, c1.PublishControl([]byte("broadcast"), "", ""))
|
||||
|
||||
select {
|
||||
case <-done1:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timeout: c1 didn't receive broadcast")
|
||||
}
|
||||
select {
|
||||
case <-done2:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timeout: c2 didn't receive broadcast")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresController_TargetedMessage(t *testing.T) {
|
||||
connString := getPostgresConnString(t)
|
||||
prefix := fmt.Sprintf("test_%d", time.Now().UnixNano()%100000)
|
||||
|
||||
var count1, count2 atomic.Int64
|
||||
|
||||
h1 := &testControlEventHandler{
|
||||
HandleControlFunc: func(data []byte) error {
|
||||
count1.Add(1)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
h2 := &testControlEventHandler{
|
||||
HandleControlFunc: func(data []byte) error {
|
||||
count2.Add(1)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
c1 := newTestPostgresController(t, PostgresControllerConfig{
|
||||
DSN: connString,
|
||||
TablePrefix: prefix,
|
||||
}, h1)
|
||||
c2 := newTestPostgresController(t, PostgresControllerConfig{
|
||||
DSN: connString,
|
||||
TablePrefix: prefix,
|
||||
}, h2)
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
// Send targeted message to c2 only.
|
||||
require.NoError(t, c1.PublishControl([]byte("targeted"), c2.myNodeID, ""))
|
||||
|
||||
waitForMessages(t, &count2, 1, 5*time.Second)
|
||||
|
||||
// Give c1 time to NOT receive it.
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
require.Equal(t, int64(0), count1.Load(), "c1 should not receive targeted message for c2")
|
||||
require.Equal(t, int64(1), count2.Load())
|
||||
}
|
||||
|
||||
func TestPostgresController_UseNotify_LowLatency(t *testing.T) {
|
||||
receivedCh := make(chan time.Time, 1)
|
||||
handler := &testControlEventHandler{
|
||||
HandleControlFunc: func(data []byte) error {
|
||||
select {
|
||||
case receivedCh <- time.Now():
|
||||
default:
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
c := newTestPostgresController(t, PostgresControllerConfig{
|
||||
PollInterval: 500 * time.Millisecond,
|
||||
UseNotify: true,
|
||||
}, handler)
|
||||
|
||||
// Wait for the LISTEN to be bound — a publish whose NOTIFY fires before
|
||||
// LISTEN runs would be dropped and the test would observe PollInterval
|
||||
// (500ms) instead of the sub-200ms NOTIFY latency it asserts on.
|
||||
require.Eventually(t, c.notifyListenerReady.Load,
|
||||
5*time.Second, 25*time.Millisecond,
|
||||
"notification listener did not bind LISTEN")
|
||||
|
||||
start := time.Now()
|
||||
require.NoError(t, c.PublishControl([]byte("fast"), "", ""))
|
||||
|
||||
select {
|
||||
case received := <-receivedCh:
|
||||
latency := received.Sub(start)
|
||||
t.Logf("NOTIFY latency: %v", latency)
|
||||
require.Less(t, latency, 200*time.Millisecond, "NOTIFY should deliver well under PollInterval")
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timeout waiting for NOTIFY-driven delivery")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresController_SchemaCreation(t *testing.T) {
|
||||
connString := getPostgresConnString(t)
|
||||
prefix := fmt.Sprintf("test_%d", time.Now().UnixNano()%100000)
|
||||
|
||||
c := newTestPostgresController(t, PostgresControllerConfig{
|
||||
DSN: connString,
|
||||
TablePrefix: prefix,
|
||||
}, nil)
|
||||
|
||||
ctx := context.Background()
|
||||
names := c.names
|
||||
|
||||
// Verify messages table exists and is partitioned.
|
||||
var isPartitioned bool
|
||||
err := c.pool.QueryRow(ctx, `
|
||||
SELECT EXISTS(
|
||||
SELECT 1 FROM pg_partitioned_table
|
||||
WHERE partrelid = $1::regclass
|
||||
)
|
||||
`, names.messages).Scan(&isPartitioned)
|
||||
require.NoError(t, err)
|
||||
require.True(t, isPartitioned, "messages table should be partitioned")
|
||||
|
||||
// Verify schema_version table.
|
||||
var version int
|
||||
err = c.pool.QueryRow(ctx, fmt.Sprintf(
|
||||
`SELECT schema_version FROM %s WHERE id = 1`, names.schemaVersion)).Scan(&version)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, controllerSchemaVersion, version)
|
||||
|
||||
// Verify shard_lock table exists with correct row count.
|
||||
var shardCount int
|
||||
err = c.pool.QueryRow(ctx, fmt.Sprintf(
|
||||
`SELECT count(*) FROM %s`, names.shardLock)).Scan(&shardCount)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, shardCount, "shard_lock should have 1 row (NumShards=1)")
|
||||
|
||||
// Verify publish function exists.
|
||||
var funcExists bool
|
||||
err = c.pool.QueryRow(ctx, `
|
||||
SELECT EXISTS(SELECT 1 FROM pg_proc WHERE proname = $1)
|
||||
`, names.publishFunc).Scan(&funcExists)
|
||||
require.NoError(t, err)
|
||||
require.True(t, funcExists, "publish function should exist")
|
||||
|
||||
// Verify partitions were pre-created.
|
||||
var partCount int
|
||||
err = c.pool.QueryRow(ctx, `
|
||||
SELECT count(*)
|
||||
FROM pg_inherits i
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
JOIN pg_class pp ON pp.oid = i.inhparent
|
||||
WHERE pp.relname = $1
|
||||
`, names.messages).Scan(&partCount)
|
||||
require.NoError(t, err)
|
||||
require.GreaterOrEqual(t, partCount, 1, "at least today's partition should exist")
|
||||
}
|
||||
|
||||
func TestPostgresController_TablePrefix(t *testing.T) {
|
||||
connString := getPostgresConnString(t)
|
||||
prefix := fmt.Sprintf("custom_%d", time.Now().UnixNano()%100000)
|
||||
|
||||
var received atomic.Int64
|
||||
doneCh := make(chan struct{})
|
||||
handler := &testControlEventHandler{
|
||||
HandleControlFunc: func(data []byte) error {
|
||||
if received.Add(1) == 1 {
|
||||
close(doneCh)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
c := newTestPostgresController(t, PostgresControllerConfig{
|
||||
DSN: connString,
|
||||
TablePrefix: prefix,
|
||||
}, handler)
|
||||
|
||||
// Verify the names use the custom prefix.
|
||||
expectedPrefix := prefix + "_"
|
||||
require.Contains(t, c.names.messages, expectedPrefix+"controller_messages")
|
||||
require.Contains(t, c.names.publishFunc, expectedPrefix+"controller_publish")
|
||||
require.Contains(t, c.names.notifyChannel, expectedPrefix+"controller_notify")
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
require.NoError(t, c.PublishControl([]byte("prefix-test"), "", ""))
|
||||
|
||||
select {
|
||||
case <-doneCh:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timeout: custom prefix publish/receive failed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresController_HighVolume(t *testing.T) {
|
||||
const msgCount = 1000
|
||||
var received atomic.Int64
|
||||
doneCh := make(chan struct{})
|
||||
handler := &testControlEventHandler{
|
||||
HandleControlFunc: func(data []byte) error {
|
||||
if received.Add(1) == int64(msgCount) {
|
||||
close(doneCh)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
c := newTestPostgresController(t, PostgresControllerConfig{}, handler)
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
for i := 0; i < msgCount; i++ {
|
||||
require.NoError(t, c.PublishControl([]byte(fmt.Sprintf("msg-%d", i)), "", ""))
|
||||
}
|
||||
|
||||
select {
|
||||
case <-doneCh:
|
||||
t.Logf("all %d messages received", msgCount)
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatalf("timeout: received %d/%d messages", received.Load(), msgCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresController_ReconnectResumesFromCursor(t *testing.T) {
|
||||
connString := getPostgresConnString(t)
|
||||
prefix := fmt.Sprintf("test_%d", time.Now().UnixNano()%100000)
|
||||
|
||||
var count1 atomic.Int64
|
||||
h1 := &testControlEventHandler{
|
||||
HandleControlFunc: func(data []byte) error {
|
||||
count1.Add(1)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
c1 := newTestPostgresController(t, PostgresControllerConfig{
|
||||
DSN: connString,
|
||||
TablePrefix: prefix,
|
||||
}, h1)
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
// Publish messages that c1 will consume.
|
||||
for i := 0; i < 5; i++ {
|
||||
require.NoError(t, c1.PublishControl([]byte(fmt.Sprintf("old-%d", i)), "", ""))
|
||||
}
|
||||
waitForMessages(t, &count1, 5, 5*time.Second)
|
||||
|
||||
// Create a second controller with the same prefix — should init cursor
|
||||
// from MAX(id) and NOT re-deliver old messages.
|
||||
var count2 atomic.Int64
|
||||
h2 := &testControlEventHandler{
|
||||
HandleControlFunc: func(data []byte) error {
|
||||
count2.Add(1)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
_ = newTestPostgresController(t, PostgresControllerConfig{
|
||||
DSN: connString,
|
||||
TablePrefix: prefix,
|
||||
}, h2)
|
||||
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
require.Equal(t, int64(0), count2.Load(), "new controller should not re-deliver old messages")
|
||||
|
||||
// Publish a new message — c2 should get it.
|
||||
require.NoError(t, c1.PublishControl([]byte("new-msg"), "", ""))
|
||||
waitForMessages(t, &count2, 1, 5*time.Second)
|
||||
}
|
||||
|
||||
func TestPostgresController_EnsureSchema_Idempotent(t *testing.T) {
|
||||
connString := getPostgresConnString(t)
|
||||
prefix := fmt.Sprintf("test_%d", time.Now().UnixNano()%100000)
|
||||
|
||||
c := newTestPostgresController(t, PostgresControllerConfig{
|
||||
DSN: connString,
|
||||
TablePrefix: prefix,
|
||||
}, nil)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, c.EnsureSchema(ctx))
|
||||
require.NoError(t, c.EnsureSchema(ctx))
|
||||
|
||||
var version int
|
||||
err := c.pool.QueryRow(ctx, fmt.Sprintf(
|
||||
`SELECT schema_version FROM %s WHERE id = 1`, c.names.schemaVersion)).Scan(&version)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, controllerSchemaVersion, version)
|
||||
}
|
||||
|
||||
func TestPostgresController_PartitionRetention(t *testing.T) {
|
||||
connString := getPostgresConnString(t)
|
||||
prefix := fmt.Sprintf("test_%d", time.Now().UnixNano()%100000)
|
||||
|
||||
c := newTestPostgresController(t, PostgresControllerConfig{
|
||||
DSN: connString,
|
||||
TablePrefix: prefix,
|
||||
PartitionRetentionDays: 1,
|
||||
PartitionLookaheadDays: 2,
|
||||
}, nil)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create an old partition manually (7 days ago).
|
||||
oldDate := time.Now().UTC().AddDate(0, 0, -7)
|
||||
nextDay := oldDate.AddDate(0, 0, 1)
|
||||
oldPartName := fmt.Sprintf("%s_%s", c.names.messages, oldDate.Format("2006_01_02"))
|
||||
_, err := c.pool.Exec(ctx, fmt.Sprintf(
|
||||
`CREATE TABLE IF NOT EXISTS %s PARTITION OF %s FOR VALUES FROM ('%s') TO ('%s')`,
|
||||
oldPartName, c.names.messages,
|
||||
oldDate.Format("2006-01-02"), nextDay.Format("2006-01-02"),
|
||||
))
|
||||
require.NoError(t, err)
|
||||
|
||||
var exists bool
|
||||
err = c.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_class WHERE relname = $1)`, oldPartName).Scan(&exists)
|
||||
require.NoError(t, err)
|
||||
require.True(t, exists, "old partition should exist before cleanup")
|
||||
|
||||
p := c.newPartitioner()
|
||||
p.DropOldPartitions(ctx)
|
||||
|
||||
err = c.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_class WHERE relname = $1)`, oldPartName).Scan(&exists)
|
||||
require.NoError(t, err)
|
||||
require.False(t, exists, "old partition should be dropped by retention cleanup")
|
||||
}
|
||||
|
||||
func TestPostgresController_ConcurrentPublish(t *testing.T) {
|
||||
// With shard_lock serialization, concurrent publishes are serialized
|
||||
// within each shard — no BIGSERIAL gaps, 100% delivery guaranteed.
|
||||
const goroutines = 10
|
||||
const msgsPerGoroutine = 50
|
||||
totalMsgs := int64(goroutines * msgsPerGoroutine)
|
||||
|
||||
var received atomic.Int64
|
||||
doneCh := make(chan struct{})
|
||||
handler := &testControlEventHandler{
|
||||
HandleControlFunc: func(data []byte) error {
|
||||
if received.Add(1) == totalMsgs {
|
||||
close(doneCh)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
c := newTestPostgresController(t, PostgresControllerConfig{}, handler)
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for g := 0; g < goroutines; g++ {
|
||||
wg.Add(1)
|
||||
go func(gid int) {
|
||||
defer wg.Done()
|
||||
for i := 0; i < msgsPerGoroutine; i++ {
|
||||
err := c.PublishControl([]byte(fmt.Sprintf("g%d-m%d", gid, i)), "", "")
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}(g)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
select {
|
||||
case <-doneCh:
|
||||
t.Logf("all %d concurrent messages received", totalMsgs)
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatalf("timeout: received %d/%d concurrent messages", received.Load(), totalMsgs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresController_DirectPoolPublish(t *testing.T) {
|
||||
var received atomic.Int64
|
||||
doneCh := make(chan struct{})
|
||||
handler := &testControlEventHandler{
|
||||
HandleControlFunc: func(data []byte) error {
|
||||
if received.Add(1) == 1 {
|
||||
close(doneCh)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
c := newTestPostgresController(t, PostgresControllerConfig{}, handler)
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
// Direct INSERT + NOTIFY (simulating what the SQL function does).
|
||||
ctx := context.Background()
|
||||
_, err := c.pool.Exec(ctx, fmt.Sprintf(
|
||||
`INSERT INTO %s (node_id, data) VALUES ('', $1)`, c.names.messages),
|
||||
[]byte("direct-insert"),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
_, err = c.pool.Exec(ctx, fmt.Sprintf("NOTIFY %s", c.names.notifyChannel))
|
||||
require.NoError(t, err)
|
||||
|
||||
select {
|
||||
case <-doneCh:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timeout: direct insert not picked up by outbox worker")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresController_PoolSizeDefaults(t *testing.T) {
|
||||
var conf PostgresControllerConfig
|
||||
conf.setDefaults()
|
||||
|
||||
require.Equal(t, 8, conf.PoolSize)
|
||||
require.Equal(t, 1, conf.NumShards)
|
||||
require.Equal(t, "cf", conf.TablePrefix)
|
||||
require.Equal(t, 50*time.Millisecond, conf.PollInterval)
|
||||
require.Equal(t, 2, conf.PartitionLookaheadDays)
|
||||
require.Equal(t, 1, conf.PartitionRetentionDays)
|
||||
require.Equal(t, time.Minute, conf.PartitionCleanupInterval)
|
||||
require.Equal(t, 1000, conf.BatchSize)
|
||||
}
|
||||
|
||||
func TestPostgresController_NewControllerNames(t *testing.T) {
|
||||
names := newControllerNames("cf")
|
||||
require.Equal(t, "cf_controller_messages", names.messages)
|
||||
require.Equal(t, "cf_controller_shard_lock", names.shardLock)
|
||||
require.Equal(t, "cf_controller_schema_version", names.schemaVersion)
|
||||
require.Equal(t, "cf_controller_publish", names.publishFunc)
|
||||
require.Equal(t, "cf_controller_notify", names.notifyChannel)
|
||||
|
||||
// Trailing underscore is trimmed.
|
||||
names2 := newControllerNames("cf_")
|
||||
require.Equal(t, names, names2)
|
||||
|
||||
// Custom prefix.
|
||||
names3 := newControllerNames("prod_us")
|
||||
require.Equal(t, "prod_us_controller_messages", names3.messages)
|
||||
require.Equal(t, "prod_us_controller_shard_lock", names3.shardLock)
|
||||
|
||||
// Bare prefix without trailing underscore.
|
||||
names4 := newControllerNames("myapp")
|
||||
require.Equal(t, "myapp_controller_messages", names4.messages)
|
||||
}
|
||||
|
||||
func TestPostgresController_VerifyInterface(t *testing.T) {
|
||||
// Compile-time check that PostgresController implements centrifuge.Controller.
|
||||
var _ centrifuge.Controller = (*PostgresController)(nil)
|
||||
|
||||
// Also verify it doesn't need the node to have a broker set.
|
||||
connString := getPostgresConnString(t)
|
||||
node, _ := centrifuge.New(centrifuge.Config{})
|
||||
c, err := NewPostgresController(node, PostgresControllerConfig{
|
||||
DSN: connString,
|
||||
TablePrefix: fmt.Sprintf("test_%d", time.Now().UnixNano()%100000),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
c.pool.Close()
|
||||
})
|
||||
require.NotNil(t, c)
|
||||
}
|
||||
|
||||
func TestPostgresController_PublishControlWithPool(t *testing.T) {
|
||||
// Test that PublishControl works independently (no outbox worker needed).
|
||||
connString := getPostgresConnString(t)
|
||||
prefix := fmt.Sprintf("test_%d", time.Now().UnixNano()%100000)
|
||||
|
||||
node, _ := centrifuge.New(centrifuge.Config{})
|
||||
c, err := NewPostgresController(node, PostgresControllerConfig{
|
||||
DSN: connString,
|
||||
TablePrefix: prefix,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx := context.Background()
|
||||
dropTestControllerSchema(t, c)
|
||||
require.NoError(t, c.EnsureSchema(ctx))
|
||||
|
||||
t.Cleanup(func() {
|
||||
c.pool.Close()
|
||||
})
|
||||
|
||||
// Publish without starting workers — the row should be inserted.
|
||||
require.NoError(t, c.PublishControl([]byte("write-only"), "", ""))
|
||||
require.NoError(t, c.PublishControl([]byte("targeted"), "some-node", ""))
|
||||
|
||||
// Verify rows in the table.
|
||||
var count int
|
||||
err = c.pool.QueryRow(ctx, fmt.Sprintf(
|
||||
`SELECT count(*) FROM %s`, c.names.messages)).Scan(&count)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, count)
|
||||
|
||||
// Verify the targeted message has the right node_id.
|
||||
var nodeID string
|
||||
err = c.pool.QueryRow(ctx, fmt.Sprintf(
|
||||
`SELECT node_id FROM %s ORDER BY id DESC LIMIT 1`, c.names.messages)).Scan(&nodeID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "some-node", nodeID)
|
||||
}
|
||||
|
||||
48
internal/controllers/controllers.go
Normal file
48
internal/controllers/controllers.go
Normal file
@@ -0,0 +1,48 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/centrifugal/centrifugo/v6/internal/configtypes"
|
||||
|
||||
"github.com/centrifugal/centrifuge"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// New creates and initializes a controller from the given config.
|
||||
// Returns nil, nil when config.Enabled is false.
|
||||
func New(node *centrifuge.Node, config configtypes.Controller) (centrifuge.Controller, error) {
|
||||
if !config.Enabled {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if config.Type == "postgres" {
|
||||
controller, err := NewPostgresController(node, PostgresControllerConfig{
|
||||
DSN: config.Postgres.DSN,
|
||||
TLS: config.Postgres.TLS,
|
||||
PoolSize: config.Postgres.PoolSize,
|
||||
NumShards: config.Postgres.NumShards,
|
||||
TablePrefix: config.Postgres.TablePrefix,
|
||||
PollInterval: config.Postgres.PollInterval.ToDuration(),
|
||||
UseNotify: config.Postgres.UseNotify,
|
||||
NotifyDSN: config.Postgres.NotifyDSN,
|
||||
PartitionRetentionDays: config.Postgres.PartitionRetentionDays,
|
||||
PartitionLookaheadDays: config.Postgres.PartitionLookaheadDays,
|
||||
PartitionCleanupInterval: config.Postgres.PartitionCleanupInterval.ToDuration(),
|
||||
BatchSize: config.Postgres.BatchSize,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error initializing Postgres controller: %w", err)
|
||||
}
|
||||
if !config.Postgres.SkipSchemaInit {
|
||||
if err := controller.EnsureSchema(context.Background()); err != nil {
|
||||
return nil, fmt.Errorf("error initializing Postgres controller schema: %w", err)
|
||||
}
|
||||
}
|
||||
log.Info().Str("controller_type", config.Type).Msg("controller is enabled")
|
||||
return controller, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unknown controller type: %s", config.Type)
|
||||
}
|
||||
29
internal/controllers/schema_template_test.go
Normal file
29
internal/controllers/schema_template_test.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// execSchemaWithRetry re-runs the whole batch on a concurrent-DDL conflict,
|
||||
// which is only safe while the batch stays one implicit transaction: no
|
||||
// statement of its own opening or closing a transaction, and nothing that
|
||||
// cannot run inside a transaction block at all (CREATE INDEX CONCURRENTLY
|
||||
// would fail outright with 25001, and a retry could not undo the half of the
|
||||
// batch that already committed). The invariant lives in the SQL template, so
|
||||
// pin it here rather than in a comment.
|
||||
func TestControllerSchemaTemplate_StaysOneImplicitTransaction(t *testing.T) {
|
||||
concurrently := regexp.MustCompile(`(?i)\bCONCURRENTLY\b`)
|
||||
txControl := regexp.MustCompile(`(?im)^\s*(BEGIN|COMMIT|ROLLBACK|START\s+TRANSACTION)\b`)
|
||||
|
||||
sql := renderControllerTemplate(postgresControllerSchemaTemplate, "cf_")
|
||||
if loc := concurrently.FindString(sql); loc != "" {
|
||||
t.Errorf("schema template uses %q — it cannot run inside the implicit transaction execSchemaWithRetry relies on", loc)
|
||||
}
|
||||
// Only the DDL half is checked for transaction control: the funcs half is
|
||||
// dollar-quoted PL/pgSQL, where BEGIN opens a block.
|
||||
ddl, _ := splitControllerSchemaSQL(sql)
|
||||
if loc := txControl.FindString(ddl); loc != "" {
|
||||
t.Errorf("schema DDL contains explicit transaction control %q — the batch must be left to the server's implicit transaction", loc)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user