//go:build integration package pgmapbroker import ( "context" "fmt" "os" "strconv" "sync" "testing" "time" "github.com/centrifugal/centrifuge" "github.com/jackc/pgx/v5/pgxpool" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" ) // testBrokerEventHandler is a local copy of centrifuge's unexported test helper. type testBrokerEventHandler struct { HandlePublicationFunc func(ch string, pub *centrifuge.Publication, sp centrifuge.StreamPosition, delta bool, prevPub *centrifuge.Publication) error HandleJoinFunc func(ch string, info *centrifuge.ClientInfo) error HandleLeaveFunc func(ch string, info *centrifuge.ClientInfo) error HandleControlFunc func([]byte) error } func (b *testBrokerEventHandler) HandlePublication(ch string, pub *centrifuge.Publication, sp centrifuge.StreamPosition, delta bool, prevPub *centrifuge.Publication) error { if b.HandlePublicationFunc != nil { return b.HandlePublicationFunc(ch, pub, sp, delta, prevPub) } return nil } func (b *testBrokerEventHandler) HandleJoin(ch string, info *centrifuge.ClientInfo) error { if b.HandleJoinFunc != nil { return b.HandleJoinFunc(ch, info) } return nil } func (b *testBrokerEventHandler) HandleLeave(ch string, info *centrifuge.ClientInfo) error { if b.HandleLeaveFunc != nil { return b.HandleLeaveFunc(ch, info) } return nil } func (b *testBrokerEventHandler) HandleControl(data []byte) error { if b.HandleControlFunc != nil { return b.HandleControlFunc(data) } return nil } func getPostgresConnString(tb testing.TB) string { connString := os.Getenv("CENTRIFUGE_POSTGRES_URL") if connString == "" { connString = "postgres://test:test@localhost:5432/test?sslmode=disable" } return connString } // newTestPostgresMapBroker creates a test broker with default outbox mode. func newTestPostgresMapBroker(tb testing.TB, n *centrifuge.Node) *PostgresMapBroker { return newTestPostgresMapBrokerWithOutbox(tb, n) } // newTestPostgresMapBrokerWithOutbox creates a test broker with outbox mode (default). func newTestPostgresMapBrokerWithOutbox(tb testing.TB, n *centrifuge.Node) *PostgresMapBroker { connString := getPostgresConnString(tb) e, err := NewPostgresMapBroker(n, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, // Fewer shards for faster tests BinaryData: true, Outbox: OutboxConfig{ PollInterval: 10 * time.Millisecond, BatchSize: 100, }, }) require.NoError(tb, err) ctx := context.Background() require.NoError(tb, e.EnsureSchema(ctx)) // Clean up tables before test cleanupTestTables(ctx, e) err = e.RegisterEventHandler(nil) require.NoError(tb, err) tb.Cleanup(func() { _ = e.Close(context.Background()) _ = n.Shutdown(context.Background()) }) return e } func cleanupTestTables(ctx context.Context, e *PostgresMapBroker) { _, _ = e.pool.Exec(ctx, fmt.Sprintf("DELETE FROM %s WHERE channel LIKE 'test_%%'", e.names.stream)) _, _ = e.pool.Exec(ctx, fmt.Sprintf("DELETE FROM %s WHERE channel LIKE 'test_%%'", e.names.state)) _, _ = e.pool.Exec(ctx, fmt.Sprintf("DELETE FROM %s WHERE channel LIKE 'test_%%'", e.names.meta)) _, _ = e.pool.Exec(ctx, fmt.Sprintf("DELETE FROM %s WHERE channel LIKE 'test_%%'", e.names.idempotency)) } // stateToMapPostgres converts []Publication to map for easier testing. func stateToMapPostgres(pubs []*centrifuge.Publication) map[string][]byte { result := make(map[string][]byte, len(pubs)) for _, pub := range pubs { result[pub.Key] = pub.Data } return result } // TestPostgresMapBroker_StatefulChannel tests stateful channel with keyed state and revisions. func TestPostgresMapBroker_StatefulChannel(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() channel := "test_stateful" // Publish some keyed state updates _, err := broker.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("data1"), }) require.NoError(t, err) _, err = broker.Publish(ctx, channel, "key2", centrifuge.MapPublishOptions{ Data: []byte("data2"), }) require.NoError(t, err) _, err = broker.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("data1_updated"), }) require.NoError(t, err) // Read state stateRes, err := broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{ Limit: 100, }) entries, streamPos, _ := stateRes.Publications, stateRes.Position, stateRes.Cursor require.NoError(t, err) require.NotEmpty(t, streamPos.Epoch) require.Greater(t, streamPos.Offset, uint64(0)) // Verify state contains latest values state := stateToMapPostgres(entries) require.Len(t, state, 2) require.Equal(t, []byte("data1_updated"), state["key1"]) require.Equal(t, []byte("data2"), state["key2"]) // Read stream to verify all publications are in history streamResult, err := broker.ReadStream(ctx, channel, centrifuge.MapReadStreamOptions{ Filter: centrifuge.StreamFilter{ Limit: -1, // Get all }, }) require.NoError(t, err) require.Len(t, streamResult.Publications, 3) // All 3 publications in stream } // TestPostgresMapBroker_StateRevision tests that state values include revisions. func TestPostgresMapBroker_StateRevision(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() channel := "test_revision" // Publish a keyed state update res1, err := broker.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("data1"), }) require.NoError(t, err) require.Equal(t, uint64(1), res1.Position.Offset) // Publish another update res2, err := broker.Publish(ctx, channel, "key2", centrifuge.MapPublishOptions{ Data: []byte("data2"), }) require.NoError(t, err) require.Equal(t, uint64(2), res2.Position.Offset) require.Equal(t, res1.Position.Epoch, res2.Position.Epoch) // Same epoch // Read state - entries now include per-entry revisions stateRes, err := broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{ Limit: 100, }) entries, streamPos, _ := stateRes.Publications, stateRes.Position, stateRes.Cursor require.NoError(t, err) require.Equal(t, res2.Position.Offset, streamPos.Offset) require.Equal(t, res2.Position.Epoch, streamPos.Epoch) // Verify payloads state := stateToMapPostgres(entries) require.Equal(t, []byte("data1"), state["key1"]) require.Equal(t, []byte("data2"), state["key2"]) // Verify per-entry offsets require.NotEmpty(t, streamPos.Epoch) for _, pub := range entries { require.Greater(t, pub.Offset, uint64(0)) } } // TestPostgresMapBroker_StatePagination tests cursor-based state pagination. func TestPostgresMapBroker_StatePagination(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() channel := "test_pagination" // Publish 10 keyed entries for i := 0; i < 10; i++ { _, err := broker.Publish(ctx, channel, fmt.Sprintf("key%d", i), centrifuge.MapPublishOptions{ Data: []byte(fmt.Sprintf("data%d", i)), }) require.NoError(t, err) } // Read state with limit stateRes, err := broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{ Limit: 3, Cursor: "", }) page1, pos1, cursor := stateRes.Publications, stateRes.Position, stateRes.Cursor require.NoError(t, err) require.NotEmpty(t, page1) // Collect all keys across pages allKeys := make(map[string]bool) for _, entry := range page1 { allKeys[entry.Key] = true } // Continue reading until cursor is empty for cursor != "" { stateRes, err := broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{ Limit: 3, Cursor: cursor, }) page, pos, newCursor := stateRes.Publications, stateRes.Position, stateRes.Cursor require.NoError(t, err) require.Equal(t, pos1.Epoch, pos.Epoch) // Same epoch across pages for _, entry := range page { // Keys should not repeat across pages require.NotContains(t, allKeys, entry.Key, "key should not repeat: %s", entry.Key) allKeys[entry.Key] = true } cursor = newCursor } // Should have read all 10 entries require.Len(t, allKeys, 10) } // TestPostgresMapBroker_StreamRecovery tests stream recovery. func TestPostgresMapBroker_StreamRecovery(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() channel := "test_recovery" // Publish some updates for i := 1; i <= 5; i++ { _, err := broker.Publish(ctx, channel, fmt.Sprintf("key%d", i), centrifuge.MapPublishOptions{ Data: []byte(fmt.Sprintf("data%d", i)), }) require.NoError(t, err) } // Get position after first 2 messages sincePos := centrifuge.StreamPosition{Offset: 2} // Read stream since position 2 streamResult, err := broker.ReadStream(ctx, channel, centrifuge.MapReadStreamOptions{ Filter: centrifuge.StreamFilter{ Since: &sincePos, Limit: -1, }, }) require.NoError(t, err) require.Equal(t, uint64(5), streamResult.Position.Offset) require.Len(t, streamResult.Publications, 3) // Should get messages 3, 4, 5 } // TestPostgresMapBroker_Idempotency tests idempotent publishing. func TestPostgresMapBroker_Idempotency(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() channel := "test_idempotency" // Publish with idempotency key res1, err := broker.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("data1"), IdempotencyKey: "unique-id-1", IdempotentResultTTL: 60 * time.Second, }) require.NoError(t, err) require.False(t, res1.Suppressed) require.Equal(t, uint64(1), res1.Position.Offset) // Publish again with same idempotency key res2, err := broker.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("data1_different"), IdempotencyKey: "unique-id-1", IdempotentResultTTL: 60 * time.Second, }) require.NoError(t, err) require.True(t, res2.Suppressed) // Suppressed due to idempotency require.Equal(t, centrifuge.SuppressReasonIdempotency, res2.SuppressReason) // The idempotency check returns the original offset require.Equal(t, res1.Position.Offset, res2.Position.Offset) // Same offset // State should still have original data stateRes, err := broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{ Limit: 100, }) entries, _, _ := stateRes.Publications, stateRes.Position, stateRes.Cursor require.NoError(t, err) state := stateToMapPostgres(entries) require.Len(t, state, 1) require.Equal(t, []byte("data1"), state["key1"]) } // TestPostgresMapBroker_KeyMode tests KeyMode (IfNew, IfExists). func TestPostgresMapBroker_KeyMode(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() channel := "test_keymode" // First publish with centrifuge.KeyModeIfNew should succeed res1, err := broker.Publish(ctx, channel, "slot1", centrifuge.MapPublishOptions{ Data: []byte("player1"), KeyMode: centrifuge.KeyModeIfNew, }) require.NoError(t, err) require.False(t, res1.Suppressed) require.Equal(t, uint64(1), res1.Position.Offset) // Second publish with centrifuge.KeyModeIfNew should be suppressed res2, err := broker.Publish(ctx, channel, "slot1", centrifuge.MapPublishOptions{ Data: []byte("player2"), KeyMode: centrifuge.KeyModeIfNew, }) require.NoError(t, err) require.True(t, res2.Suppressed) require.Equal(t, centrifuge.SuppressReasonKeyExists, res2.SuppressReason) // Verify state still has original data stateRes, err := broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{ Limit: 100, }) entries, _, _ := stateRes.Publications, stateRes.Position, stateRes.Cursor require.NoError(t, err) require.Len(t, entries, 1) require.Equal(t, []byte("player1"), entries[0].Data) // centrifuge.KeyModeIfExists should be suppressed for non-existent key res3, err := broker.Publish(ctx, channel, "nonexistent", centrifuge.MapPublishOptions{ Data: []byte("data"), KeyMode: centrifuge.KeyModeIfExists, }) require.NoError(t, err) require.True(t, res3.Suppressed) require.Equal(t, centrifuge.SuppressReasonKeyNotFound, res3.SuppressReason) // centrifuge.KeyModeIfExists should succeed for existing key res4, err := broker.Publish(ctx, channel, "slot1", centrifuge.MapPublishOptions{ Data: []byte("updated"), KeyMode: centrifuge.KeyModeIfExists, }) require.NoError(t, err) require.False(t, res4.Suppressed) } // TestPostgresMapBroker_CAS tests Compare-And-Swap operations. func TestPostgresMapBroker_CAS(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() channel := "test_cas" // Initial publish res1, err := broker.Publish(ctx, channel, "item1", centrifuge.MapPublishOptions{ Data: []byte(`{"stock": 10}`), }) require.NoError(t, err) require.False(t, res1.Suppressed) // Read current state stateRes, err := broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{ Key: "item1", }) entries, pos, _ := stateRes.Publications, stateRes.Position, stateRes.Cursor require.NoError(t, err) require.Len(t, entries, 1) // CAS update with correct position expectedPos := centrifuge.StreamPosition{Offset: entries[0].Offset, Epoch: pos.Epoch} res2, err := broker.Publish(ctx, channel, "item1", centrifuge.MapPublishOptions{ Data: []byte(`{"stock": 9}`), ExpectedPosition: &expectedPos, }) require.NoError(t, err) require.False(t, res2.Suppressed) // CAS update with stale position should fail res3, err := broker.Publish(ctx, channel, "item1", centrifuge.MapPublishOptions{ Data: []byte(`{"stock": 8}`), ExpectedPosition: &expectedPos, // Using old position }) require.NoError(t, err) require.True(t, res3.Suppressed) require.Equal(t, centrifuge.SuppressReasonPositionMismatch, res3.SuppressReason) require.NotNil(t, res3.CurrentEntry) require.Equal(t, []byte(`{"stock": 9}`), res3.CurrentEntry.Data) } // TestPostgresMapBroker_CAS_StaleEpoch verifies that a CAS attempt using a // position from a dead epoch is rejected even when offsets coincidentally // match. Clear flips the channel epoch and resets the offset counter; a // republish reaches the same offset under a new epoch. A naive offset-only // CAS check would let the stale-epoch caller through. Memory and Redis // brokers already compare epoch — this test pins PostgreSQL to the same // behavior. func TestPostgresMapBroker_CAS_StaleEpoch(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() channel := fmt.Sprintf("test_cas_epoch_%d", time.Now().UnixNano()) // Initial publish under epoch E1. res1, err := broker.Publish(ctx, channel, "k", centrifuge.MapPublishOptions{ Data: []byte(`{"v": 1}`), }) require.NoError(t, err) stalePos := res1.Position // {Offset: 1, Epoch: E1} // Clear the channel — this drops state and flips epoch on next publish. require.NoError(t, broker.Clear(ctx, channel, centrifuge.MapClearOptions{})) // Republish under a fresh epoch E2. Offset starts back at 1, so a // pure-offset CAS using stalePos would falsely succeed. res2, err := broker.Publish(ctx, channel, "k", centrifuge.MapPublishOptions{ Data: []byte(`{"v": 2}`), }) require.NoError(t, err) require.NotEqual(t, stalePos.Epoch, res2.Position.Epoch, "epoch should have flipped after Clear") // CAS using the old (E1, offset=1) position must be rejected — even though // the current offset is also 1, the epoch differs. stale := stalePos res3, err := broker.Publish(ctx, channel, "k", centrifuge.MapPublishOptions{ Data: []byte(`{"v": 99}`), ExpectedPosition: &stale, }) require.NoError(t, err) require.True(t, res3.Suppressed, "CAS with stale epoch must be rejected") require.Equal(t, centrifuge.SuppressReasonPositionMismatch, res3.SuppressReason) } func TestPostgresMapBroker_CleanupMetrics(t *testing.T) { if testing.Short() { t.Skip("skipping cleanup metrics test in short mode") } registry := prometheus.NewRegistry() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 2 * time.Second, } }, }, Metrics: centrifuge.MetricsConfig{ RegistererGatherer: registry, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() channel := fmt.Sprintf("test_cleanup_metrics_%d", time.Now().UnixNano()) // Publish two entries with short TTL. _, err := broker.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("data1"), }) require.NoError(t, err) _, err = broker.Publish(ctx, channel, "key2", centrifuge.MapPublishOptions{ Data: []byte("data2"), }) require.NoError(t, err) // Wait for TTL to expire. time.Sleep(3 * time.Second) // Poll until the cleanup metric reflects both removals. Both the // background TTL worker (every TTLCheckInterval=1s) and our explicit // expireKeys call contribute to the counter; Eventually handles timing // variations on slow CI where: // - the second publish's expires_at is barely past NOW() when a // cleanup pass runs, so only the first key is captured; // - or the manual call races with a background pass that already // removed one of the keys, leaving a single in-flight delete // visible to the next pass. // In any timing, the counter must converge to >= 2. require.Eventually(t, func() bool { broker.expireKeys(ctx) families, err := registry.Gather() if err != nil { return false } var removedCount float64 for _, f := range families { if f.GetName() == "centrifuge_map_broker_cleanup_removed_count" { for _, m := range f.GetMetric() { removedCount += m.GetCounter().GetValue() } } } return removedCount >= 2 }, 10*time.Second, 200*time.Millisecond, "cleanup_removed_count should reach >= 2") } // TestPostgresMapBroker_KeyTTL tests key TTL (this is a slower test). func TestPostgresMapBroker_KeyTTL(t *testing.T) { if testing.Short() { t.Skip("skipping TTL test in short mode") } node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 2 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() channel := fmt.Sprintf("test_key_ttl_%d", time.Now().UnixNano()) // Publish with short TTL (2s from channel config) _, err := broker.Publish(ctx, channel, "ephemeral", centrifuge.MapPublishOptions{ Data: []byte("temporary"), }) require.NoError(t, err) // Verify key exists stateRes, err := broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{ Key: "ephemeral", }) entries, _, _ := stateRes.Publications, stateRes.Position, stateRes.Cursor require.NoError(t, err) require.Len(t, entries, 1) // Wait for TTL to expire time.Sleep(3 * time.Second) // Trigger TTL check broker.expireKeys(ctx) // Key should be gone stateRes, err = broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{ Key: "ephemeral", }) entries, _, _ = stateRes.Publications, stateRes.Position, stateRes.Cursor require.NoError(t, err) require.Empty(t, entries) } // TestPostgresMapBroker_Version tests version-based ordering. func TestPostgresMapBroker_Version(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() channel := "test_version" // Publish with version 2 res1, err := broker.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("data_v2"), Version: 2, }) require.NoError(t, err) require.False(t, res1.Suppressed) // Try to publish older version (should be suppressed) res2, err := broker.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("data_v1"), Version: 1, }) require.NoError(t, err) require.True(t, res2.Suppressed) require.Equal(t, centrifuge.SuppressReasonVersion, res2.SuppressReason) // Publish newer version res3, err := broker.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("data_v3"), Version: 3, }) require.NoError(t, err) require.False(t, res3.Suppressed) // Verify state has v3 data stateRes, err := broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{ Key: "key1", }) entries, _, _ := stateRes.Publications, stateRes.Position, stateRes.Cursor require.NoError(t, err) require.Equal(t, []byte("data_v3"), entries[0].Data) } // TestPostgresMapBroker_PerKeyVersion tests that version tracking is per-key independent. func TestPostgresMapBroker_PerKeyVersion(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() channel := "test_per_key_version" // key1 with version=10 → accepted res1, err := broker.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("key1_v10"), Version: 10, }) require.NoError(t, err) require.False(t, res1.Suppressed) // key2 with version=5 → accepted (independent, was broken before per-key version) res2, err := broker.Publish(ctx, channel, "key2", centrifuge.MapPublishOptions{ Data: []byte("key2_v5"), Version: 5, }) require.NoError(t, err) require.False(t, res2.Suppressed, "key2 should not be suppressed by key1's version") // key1 with version=5 → suppressed (same key, lower version) res3, err := broker.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("key1_v5"), Version: 5, }) require.NoError(t, err) require.True(t, res3.Suppressed) require.Equal(t, centrifuge.SuppressReasonVersion, res3.SuppressReason) // Remove key1 _, err = broker.Remove(ctx, channel, "key1", centrifuge.MapRemoveOptions{}) require.NoError(t, err) // Publish key1 with version=1 → accepted (version cleared by remove) res4, err := broker.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("key1_v1"), Version: 1, }) require.NoError(t, err) require.False(t, res4.Suppressed, "key1 version should be cleared after remove") // Verify final state stateRes, err := broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{ Limit: 100, }) require.NoError(t, err) state := stateToMapPostgres(stateRes.Publications) require.Equal(t, []byte("key1_v1"), state["key1"]) require.Equal(t, []byte("key2_v5"), state["key2"]) } // TestPostgresMapBroker_Remove tests removing keys. func TestPostgresMapBroker_Remove(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() channel := "test_unpublish" // Publish some keys _, err := broker.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("data1"), }) require.NoError(t, err) _, err = broker.Publish(ctx, channel, "key2", centrifuge.MapPublishOptions{ Data: []byte("data2"), }) require.NoError(t, err) // Verify state has 2 keys stateRes, err := broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{ Limit: 100, }) entries, _, _ := stateRes.Publications, stateRes.Position, stateRes.Cursor require.NoError(t, err) require.Len(t, entries, 2) // Remove key1 res, err := broker.Remove(ctx, channel, "key1", centrifuge.MapRemoveOptions{}) require.NoError(t, err) require.False(t, res.Suppressed) // Verify state has 1 key stateRes, err = broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{ Limit: 100, }) entries, _, _ = stateRes.Publications, stateRes.Position, stateRes.Cursor require.NoError(t, err) require.Len(t, entries, 1) require.Equal(t, "key2", entries[0].Key) // Remove non-existent key should be suppressed res, err = broker.Remove(ctx, channel, "nonexistent", centrifuge.MapRemoveOptions{}) require.NoError(t, err) require.True(t, res.Suppressed) require.Equal(t, centrifuge.SuppressReasonKeyNotFound, res.SuppressReason) // Verify stream has removal event streamResult, err := broker.ReadStream(ctx, channel, centrifuge.MapReadStreamOptions{ Filter: centrifuge.StreamFilter{ Limit: -1, }, }) require.NoError(t, err) require.Len(t, streamResult.Publications, 3) // key1, key2, remove(key1) require.True(t, streamResult.Publications[2].Removed) require.Equal(t, "key1", streamResult.Publications[2].Key) } // TestPostgresMapBroker_Stats tests state statistics. func TestPostgresMapBroker_Stats(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() channel := "test_stats" // Initially empty stats, err := broker.Stats(ctx, channel) require.NoError(t, err) require.Equal(t, 0, stats.NumKeys) // Publish some keys for i := 0; i < 5; i++ { _, err := broker.Publish(ctx, channel, fmt.Sprintf("key%d", i), centrifuge.MapPublishOptions{ Data: []byte(fmt.Sprintf("data%d", i)), }) require.NoError(t, err) } // Should have 5 keys stats, err = broker.Stats(ctx, channel) require.NoError(t, err) require.Equal(t, 5, stats.NumKeys) } // TestPostgresMapBroker_EpochMismatch tests epoch validation. func TestPostgresMapBroker_EpochMismatch(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() channel := "test_epoch_mismatch" // Client tries to read with old epoch (channel doesn't exist yet) _, err := broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{ Revision: ¢rifuge.StreamPosition{ Epoch: "old_epoch", Offset: 100, }, Limit: 100, }) require.ErrorIs(t, err, centrifuge.ErrorUnrecoverablePosition) // Create channel _, err = broker.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("data1"), }) require.NoError(t, err) // Read actual epoch stateRes, err := broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{ Limit: 100, }) _, pos, _ := stateRes.Publications, stateRes.Position, stateRes.Cursor require.NoError(t, err) require.NotEmpty(t, pos.Epoch) // Read with wrong epoch should fail _, err = broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{ Revision: ¢rifuge.StreamPosition{ Epoch: "wrong_epoch", Offset: 1, }, Limit: 100, }) require.ErrorIs(t, err, centrifuge.ErrorUnrecoverablePosition) // Read with correct epoch should succeed stateRes, err = broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{ Revision: &pos, Limit: 100, }) entries, _, _ := stateRes.Publications, stateRes.Position, stateRes.Cursor require.NoError(t, err) require.Len(t, entries, 1) } // TestPostgresMapBroker_ConcurrentPublishOrdering tests that concurrent publishes maintain per-channel ordering. func TestPostgresMapBroker_ConcurrentPublishOrdering(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() channel := "test_concurrent_order" const numGoroutines = 5 const publishesPerGoroutine = 20 var wg sync.WaitGroup wg.Add(numGoroutines) // Concurrent publishes for g := 0; g < numGoroutines; g++ { go func(goroutineID int) { defer wg.Done() for i := 0; i < publishesPerGoroutine; i++ { key := fmt.Sprintf("g%d_k%d", goroutineID, i) data := []byte(fmt.Sprintf("data_%d_%d", goroutineID, i)) _, err := broker.Publish(ctx, channel, key, centrifuge.MapPublishOptions{ Data: data, }) require.NoError(t, err) } }(g) } wg.Wait() // Read stream and verify offsets are sequential with no gaps streamResult, err := broker.ReadStream(ctx, channel, centrifuge.MapReadStreamOptions{ Filter: centrifuge.StreamFilter{ Limit: -1, }, }) require.NoError(t, err) expectedCount := numGoroutines * publishesPerGoroutine require.Len(t, streamResult.Publications, expectedCount) require.Equal(t, uint64(expectedCount), streamResult.Position.Offset) // Verify offsets are 1, 2, 3, ... with no gaps for i, pub := range streamResult.Publications { require.Equal(t, uint64(i+1), pub.Offset, "offset at index %d should be %d, got %d", i, i+1, pub.Offset) } } // ============================================================================ // Outbox Mode Tests // ============================================================================ // TestPostgresMapBroker_OutboxOrdering tests that publications are delivered in channel_offset order. func TestPostgresMapBroker_OutboxOrdering(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, err := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) require.NoError(t, err) require.NoError(t, node.Run()) // Create broker with outbox mode (default) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, BinaryData: true, NumShards: 4, Outbox: OutboxConfig{ PollInterval: 10 * time.Millisecond, BatchSize: 100, }, }) require.NoError(t, err) require.NoError(t, broker.EnsureSchema(ctx)) // Use unique channel name per test run channel := fmt.Sprintf("test_outbox_order_%d", time.Now().UnixNano()) // Clean up tables cleanupTestTables(ctx, broker) t.Cleanup(func() { _ = broker.Close(context.Background()) _ = node.Shutdown(context.Background()) }) var received []*centrifuge.Publication var receivedMu sync.Mutex doneCh := make(chan struct{}) const numMessages = 10 err = broker.RegisterEventHandler(&testBrokerEventHandler{ HandlePublicationFunc: func(ch string, pub *centrifuge.Publication, sp centrifuge.StreamPosition, delta bool, prevPub *centrifuge.Publication) error { if ch != channel { return nil } receivedMu.Lock() received = append(received, pub) if len(received) >= numMessages { select { case <-doneCh: default: close(doneCh) } } receivedMu.Unlock() return nil }, }) require.NoError(t, err) // Give outbox worker a moment to start polling. time.Sleep(50 * time.Millisecond) // Publish multiple messages in sequence for i := 0; i < numMessages; i++ { _, err = broker.Publish(ctx, channel, fmt.Sprintf("key%d", i), centrifuge.MapPublishOptions{ Data: []byte(fmt.Sprintf("data%d", i)), }) require.NoError(t, err) } // Wait for all messages select { case <-doneCh: case <-time.After(10 * time.Second): t.Fatal("timeout waiting for all publications") } // Verify ordering by offset receivedMu.Lock() defer receivedMu.Unlock() require.Len(t, received, numMessages) // Offsets should be sequential (1, 2, 3, ...) for i, pub := range received { require.Equal(t, uint64(i+1), pub.Offset, "offset at index %d should be %d", i, i+1) } } // TestPostgresMapBroker_OutboxConcurrentPublish tests concurrent publishes maintain ordering. func TestPostgresMapBroker_OutboxConcurrentPublish(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, err := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) require.NoError(t, err) require.NoError(t, node.Run()) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, BinaryData: true, NumShards: 4, Outbox: OutboxConfig{ PollInterval: 10 * time.Millisecond, }, }) require.NoError(t, err) require.NoError(t, broker.EnsureSchema(ctx)) channel := fmt.Sprintf("test_outbox_concurrent_%d", time.Now().UnixNano()) cleanupTestTables(ctx, broker) t.Cleanup(func() { _ = broker.Close(context.Background()) _ = node.Shutdown(context.Background()) }) const numGoroutines = 5 const publishesPerGoroutine = 10 totalPublishes := numGoroutines * publishesPerGoroutine var received []*centrifuge.Publication var receivedMu sync.Mutex doneCh := make(chan struct{}) err = broker.RegisterEventHandler(&testBrokerEventHandler{ HandlePublicationFunc: func(ch string, pub *centrifuge.Publication, sp centrifuge.StreamPosition, delta bool, prevPub *centrifuge.Publication) error { if ch != channel { return nil } receivedMu.Lock() received = append(received, pub) if len(received) >= totalPublishes { select { case <-doneCh: default: close(doneCh) } } receivedMu.Unlock() return nil }, }) require.NoError(t, err) // Give outbox worker a moment to start polling. time.Sleep(50 * time.Millisecond) // Concurrent publishes var wg sync.WaitGroup wg.Add(numGoroutines) for g := 0; g < numGoroutines; g++ { go func(goroutineID int) { defer wg.Done() for i := 0; i < publishesPerGoroutine; i++ { key := fmt.Sprintf("g%d_k%d", goroutineID, i) data := []byte(fmt.Sprintf("data_%d_%d", goroutineID, i)) _, err := broker.Publish(ctx, channel, key, centrifuge.MapPublishOptions{ Data: data, }) require.NoError(t, err) } }(g) } wg.Wait() // Wait for all messages select { case <-doneCh: case <-time.After(15 * time.Second): receivedMu.Lock() t.Fatalf("timeout waiting for publications, received %d/%d", len(received), totalPublishes) receivedMu.Unlock() } receivedMu.Lock() defer receivedMu.Unlock() require.Len(t, received, totalPublishes) // Collect offsets and verify no gaps offsets := make(map[uint64]bool) for _, pub := range received { offsets[pub.Offset] = true } for i := 1; i <= totalPublishes; i++ { require.True(t, offsets[uint64(i)], "offset %d should exist", i) } } // TestPostgresMapBroker_Delta_Outbox tests key-based delta delivery via outbox workers. func TestPostgresMapBroker_Delta_Outbox(t *testing.T) { connString := getPostgresConnString(t) node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) type pubEvent struct { ch string pub *centrifuge.Publication delta bool prevPub *centrifuge.Publication } eventCh := make(chan pubEvent, 10) handler := &testBrokerEventHandler{ HandlePublicationFunc: func(ch string, pub *centrifuge.Publication, sp centrifuge.StreamPosition, delta bool, prevPub *centrifuge.Publication) error { eventCh <- pubEvent{ch: ch, pub: pub, delta: delta, prevPub: prevPub} return nil }, } e, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, BinaryData: true, NumShards: 4, Outbox: OutboxConfig{ PollInterval: 10 * time.Millisecond, BatchSize: 100, }, }) require.NoError(t, err) ctx := context.Background() require.NoError(t, e.EnsureSchema(ctx)) cleanupTestTables(ctx, e) err = e.RegisterEventHandler(handler) require.NoError(t, err) t.Cleanup(func() { _ = e.Close(context.Background()) _ = node.Shutdown(context.Background()) }) // Give outbox worker a moment to start polling. time.Sleep(50 * time.Millisecond) channel := fmt.Sprintf("test_delta_%d", time.Now().UnixNano()) waitEvent := func(t *testing.T) pubEvent { t.Helper() for { select { case ev := <-eventCh: if ev.ch == channel { return ev } // Skip events from other channels (stale outbox entries). case <-time.After(10 * time.Second): t.Fatal("timeout waiting for publication event") return pubEvent{} } } } // 1. First publish with UseDelta - no previous state. _, err = e.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("data1"), UseDelta: true, }) require.NoError(t, err) ev := waitEvent(t) require.False(t, ev.delta, "no previous data means useDelta is false in outbox delivery") require.Nil(t, ev.prevPub, "no previous state for first publish") require.Equal(t, []byte("data1"), ev.pub.Data) // 2. Second publish same key - should get prevPub with first data. _, err = e.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("data1_updated"), UseDelta: true, }) require.NoError(t, err) ev = waitEvent(t) require.True(t, ev.delta) require.NotNil(t, ev.prevPub) require.Equal(t, []byte("data1"), ev.prevPub.Data) require.Equal(t, []byte("data1_updated"), ev.pub.Data) // 3. Different key - no previous state for this key. _, err = e.Publish(ctx, channel, "key2", centrifuge.MapPublishOptions{ Data: []byte("data2"), UseDelta: true, }) require.NoError(t, err) ev = waitEvent(t) require.False(t, ev.delta, "no previous data for key2") require.Nil(t, ev.prevPub, "no previous state for key2") require.Equal(t, []byte("data2"), ev.pub.Data) // 4. UseDelta=false - no delta. _, err = e.Publish(ctx, channel, "key2", centrifuge.MapPublishOptions{ Data: []byte("data2_updated"), UseDelta: false, }) require.NoError(t, err) ev = waitEvent(t) require.False(t, ev.delta) require.Nil(t, ev.prevPub, "UseDelta=false means no delta") // 6. Third publish to key1 - prevPub should have previous data. _, err = e.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("data1_v3"), UseDelta: true, }) require.NoError(t, err) ev = waitEvent(t) require.True(t, ev.delta) require.NotNil(t, ev.prevPub) require.Equal(t, []byte("data1_updated"), ev.prevPub.Data, "prevPub should have current state data for key1") } func TestPostgresMapBroker_Clear(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() channel := "test_clear" // Publish some keyed state and stream entries. for i := 0; i < 3; i++ { _, err := broker.Publish(ctx, channel, fmt.Sprintf("key%d", i), centrifuge.MapPublishOptions{ Data: []byte(fmt.Sprintf("data%d", i)), }) require.NoError(t, err) } // Verify data exists. stateRes, err := broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{Limit: 100}) entries, _, _ := stateRes.Publications, stateRes.Position, stateRes.Cursor require.NoError(t, err) require.Len(t, entries, 3) streamResult, err := broker.ReadStream(ctx, channel, centrifuge.MapReadStreamOptions{Filter: centrifuge.StreamFilter{Limit: -1}}) require.NoError(t, err) require.Len(t, streamResult.Publications, 3) stats, err := broker.Stats(ctx, channel) require.NoError(t, err) require.Equal(t, 3, stats.NumKeys) // Clear the channel. err = broker.Clear(ctx, channel, centrifuge.MapClearOptions{}) require.NoError(t, err) // State should be empty. stateRes, err = broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{Limit: 100}) entries, _, _ = stateRes.Publications, stateRes.Position, stateRes.Cursor require.NoError(t, err) require.Empty(t, entries) // Stream should be empty. streamResult, err = broker.ReadStream(ctx, channel, centrifuge.MapReadStreamOptions{Filter: centrifuge.StreamFilter{Limit: -1}}) require.NoError(t, err) require.Empty(t, streamResult.Publications) // Stats should show zero keys. stats, err = broker.Stats(ctx, channel) require.NoError(t, err) require.Equal(t, 0, stats.NumKeys) } func TestPostgresMapBroker_ClearDoesNotAffectOtherChannels(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() // Populate two channels. for _, ch := range []string{"test_clear_iso_ch1", "test_clear_iso_ch2"} { _, err := broker.Publish(ctx, ch, "k", centrifuge.MapPublishOptions{ Data: []byte("v"), }) require.NoError(t, err) } // Clear only ch1. err := broker.Clear(ctx, "test_clear_iso_ch1", centrifuge.MapClearOptions{}) require.NoError(t, err) // ch1 empty. stateRes, err := broker.ReadState(ctx, "test_clear_iso_ch1", centrifuge.MapReadStateOptions{Limit: 100}) entries, _, _ := stateRes.Publications, stateRes.Position, stateRes.Cursor require.NoError(t, err) require.Empty(t, entries) // ch2 still intact. stateRes, err = broker.ReadState(ctx, "test_clear_iso_ch2", centrifuge.MapReadStateOptions{Limit: 100}) entries, _, _ = stateRes.Publications, stateRes.Position, stateRes.Cursor require.NoError(t, err) require.Len(t, entries, 1) } // ============================================================================ // EnsureSchema Tests // ============================================================================ // dropAllSchemaObjects drops all centrifuge map objects (both JSONB and binary variants) // in reverse dependency order. func dropAllSchemaObjects(ctx context.Context, pool *pgxpool.Pool) { for _, prefix := range []string{"cf_map_", "cf_binary_map_"} { // Drop functions (they depend on tables). _, _ = pool.Exec(ctx, fmt.Sprintf("DROP FUNCTION IF EXISTS %spublish CASCADE", prefix)) _, _ = pool.Exec(ctx, fmt.Sprintf("DROP FUNCTION IF EXISTS %spublish_strict CASCADE", prefix)) _, _ = pool.Exec(ctx, fmt.Sprintf("DROP FUNCTION IF EXISTS %sremove CASCADE", prefix)) _, _ = pool.Exec(ctx, fmt.Sprintf("DROP FUNCTION IF EXISTS %sremove_strict CASCADE", prefix)) _, _ = pool.Exec(ctx, fmt.Sprintf("DROP FUNCTION IF EXISTS %sexpire_keys CASCADE", prefix)) // Drop tables. _, _ = pool.Exec(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %sidempotency CASCADE", prefix)) _, _ = pool.Exec(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %sstream CASCADE", prefix)) _, _ = pool.Exec(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %sstate CASCADE", prefix)) _, _ = pool.Exec(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %smeta CASCADE", prefix)) _, _ = pool.Exec(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %sshard_lock CASCADE", prefix)) _, _ = pool.Exec(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %sschema_version CASCADE", prefix)) } } // verifySchemaComplete checks that all expected tables, indexes, functions, and column types exist. // prefix is "cf_map_" or "cf_binary_map_". func verifySchemaComplete(t *testing.T, ctx context.Context, pool *pgxpool.Pool, prefix string, expectJSONB bool) { t.Helper() // Check tables exist. for _, suffix := range []string{"stream", "state", "meta", "idempotency", "shard_lock", "schema_version"} { table := prefix + suffix var exists bool err := pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = $1)`, table).Scan(&exists) require.NoError(t, err) require.True(t, exists, "table %s should exist", table) } // Check indexes exist. for _, suffix := range []string{ "stream_channel_epoch_offset_idx", "stream_channel_id_idx", "stream_created_at_idx", "stream_shard_id_idx", "state_expires_idx", "meta_expires_idx", "idempotency_expires_idx", } { idx := prefix + suffix var exists bool err := pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_indexes WHERE indexname = $1)`, idx).Scan(&exists) require.NoError(t, err) require.True(t, exists, "index %s should exist", idx) } // Check functions exist. for _, suffix := range []string{"publish", "publish_strict", "remove", "remove_strict", "expire_keys"} { fn := prefix + suffix var exists bool err := pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_proc WHERE proname = $1)`, fn).Scan(&exists) require.NoError(t, err) require.True(t, exists, "function %s should exist", fn) } // Check data column types. expectedType := "jsonb" if !expectJSONB { expectedType = "bytea" } dataColumns := []struct { table string column string }{ {prefix + "stream", "data"}, {prefix + "stream", "prev_data"}, {prefix + "stream", "conn_info"}, {prefix + "stream", "chan_info"}, {prefix + "state", "data"}, {prefix + "state", "conn_info"}, {prefix + "state", "chan_info"}, } for _, dc := range dataColumns { var dataType string err := pool.QueryRow(ctx, `SELECT data_type FROM information_schema.columns WHERE table_name = $1 AND column_name = $2`, dc.table, dc.column).Scan(&dataType) require.NoError(t, err, "column %s.%s should exist", dc.table, dc.column) require.Equal(t, expectedType, dataType, "column %s.%s should be %s", dc.table, dc.column, expectedType) } // tags should always be JSONB regardless of BinaryData setting. for _, suffix := range []string{"stream", "state"} { table := prefix + suffix var dataType string err := pool.QueryRow(ctx, `SELECT data_type FROM information_schema.columns WHERE table_name = $1 AND column_name = 'tags'`, table).Scan(&dataType) require.NoError(t, err) require.Equal(t, "jsonb", dataType, "%s.tags should always be jsonb", table) } } // TestPostgresMapBroker_EnsureSchema_Fresh tests creating schema from scratch. func TestPostgresMapBroker_EnsureSchema_Fresh(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, }) require.NoError(t, err) t.Cleanup(func() { _ = broker.Close(ctx) _ = node.Shutdown(ctx) }) // Drop everything first. dropAllSchemaObjects(ctx, broker.pool) // Create from scratch. err = broker.EnsureSchema(ctx) require.NoError(t, err) // Verify all objects exist for both prefixes. verifySchemaComplete(t, ctx, broker.pool, "cf_map_", true) verifySchemaComplete(t, ctx, broker.pool, "cf_binary_map_", false) } // TestPostgresMapBroker_EnsureSchema_Idempotent tests calling EnsureSchema twice. func TestPostgresMapBroker_EnsureSchema_Idempotent(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, }) require.NoError(t, err) t.Cleanup(func() { _ = broker.Close(ctx) _ = node.Shutdown(ctx) }) dropAllSchemaObjects(ctx, broker.pool) // First call. err = broker.EnsureSchema(ctx) require.NoError(t, err) // Second call — should succeed without errors. err = broker.EnsureSchema(ctx) require.NoError(t, err) verifySchemaComplete(t, ctx, broker.pool, "cf_map_", true) verifySchemaComplete(t, ctx, broker.pool, "cf_binary_map_", false) } // TestPostgresMapBroker_EnsureSchema_FastPathReachable pins the probe the // fast path gates on. It used to look for the trimmed prefix (cf_map), a // relation this broker never creates — the shape pgstreambroker uses, where // the trimmed prefix IS the table. So it always reported "not present", the // fast path was unreachable, and every single node start re-ran the full DDL // for both variants, CREATE OR REPLACE FUNCTION included. Nothing failed // visibly, which is exactly why it needs a test. func TestPostgresMapBroker_EnsureSchema_FastPathReachable(t *testing.T) { ctx := context.Background() node, err := centrifuge.New(centrifuge.Config{}) require.NoError(t, err) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: getPostgresConnString(t), NumShards: 4, BinaryData: true, }) require.NoError(t, err) t.Cleanup(func() { _ = broker.Close(ctx) _ = node.Shutdown(ctx) }) require.NoError(t, broker.EnsureSchema(ctx)) require.True(t, broker.bothVariantsPresent(ctx), "both variants exist after EnsureSchema, so the fast path must be reachable on the next start") } // TestPostgresMapBroker_EnsureSchema_FastPathRestoresPartitions covers the // one part of the schema that expires on its own. Tables and functions are // created once and stay; the daily partitions only cover // PartitionLookaheadDays ahead, so a cluster that was down longer than that // comes back to a stream table with no partition for today, and every publish // fails with "no partition for value". The fast path is what a restart of a // healthy cluster takes, so it has to repair that — before, the gap stayed // open until the partition worker's first tick, a full CleanupInterval later. func TestPostgresMapBroker_EnsureSchema_FastPathRestoresPartitions(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() // Built directly rather than through the test helper: no event handler // means no partition worker, so EnsureSchema is the only thing that can // create a partition and the assertion below can't be satisfied by a // background tick. node, err := centrifuge.New(centrifuge.Config{}) require.NoError(t, err) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, BinaryData: true, }) require.NoError(t, err) t.Cleanup(func() { _ = broker.Close(ctx) _ = node.Shutdown(ctx) }) require.NoError(t, broker.EnsureSchema(ctx)) // Simulate the calendar having moved past the lookahead window. today := fmt.Sprintf("%s_%s", broker.names.stream, time.Now().UTC().Format("2006_01_02")) _, err = broker.pool.Exec(ctx, "DROP TABLE "+today) require.NoError(t, err) require.False(t, partitionExists(ctx, t, broker.pool, today)) // Schema version is current and both variants are present, so this call // takes the fast path — the one a healthy restart takes. require.NoError(t, broker.EnsureSchema(ctx)) require.True(t, partitionExists(ctx, t, broker.pool, today), "fast path left today's partition missing — publishes would fail until the partition worker's first tick") } func partitionExists(ctx context.Context, t *testing.T, pool *pgxpool.Pool, name string) bool { t.Helper() var exists bool require.NoError(t, pool.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM pg_class WHERE relname = $1 AND relkind = 'r')", name).Scan(&exists)) return exists } // TestPostgresMapBroker_EnsureSchema_ConcurrentNodes reproduces a cluster // whose nodes all start against the same empty database — the shape of a // fresh install rolled out to a whole deployment at once. CREATE TABLE/INDEX // IF NOT EXISTS probes the catalog before it takes any lock, so every node // but the winner loses the race and gets a hard error (42P07, 42710 on the // table's row type, or 23505 on pg_type/pg_class when it reached its own // catalog inserts first) instead of // the NOTICE a sequential re-run produces. EnsureSchema returning that error // means the node refuses to start, so every node here must come up clean. // // The race is timing-dependent and won't fire on every run — the test is a // regression guard, not a reproducer: it must never fail. func TestPostgresMapBroker_EnsureSchema_ConcurrentNodes(t *testing.T) { const ( nodes = 8 rounds = 3 ) connString := getPostgresConnString(t) ctx := context.Background() // Separate brokers with separate pools — one per simulated node. brokers := make([]*PostgresMapBroker, 0, nodes) for range nodes { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, }) require.NoError(t, err) t.Cleanup(func() { _ = broker.Close(ctx) _ = node.Shutdown(ctx) }) brokers = append(brokers, broker) } for round := range rounds { dropAllSchemaObjects(ctx, brokers[0].pool) start := make(chan struct{}) errs := make([]error, nodes) var wg sync.WaitGroup for i, broker := range brokers { wg.Add(1) go func() { defer wg.Done() <-start errs[i] = broker.EnsureSchema(ctx) }() } close(start) wg.Wait() for i, err := range errs { require.NoErrorf(t, err, "round %d: node %d failed to bring up the schema", round, i) } verifySchemaComplete(t, ctx, brokers[0].pool, "cf_map_", true) verifySchemaComplete(t, ctx, brokers[0].pool, "cf_binary_map_", false) } } // TestPostgresMapBroker_EnsureSchema_PartialState tests that EnsureSchema handles partial schema. func TestPostgresMapBroker_EnsureSchema_PartialState(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, }) require.NoError(t, err) t.Cleanup(func() { _ = broker.Close(ctx) _ = node.Shutdown(ctx) }) // Drop everything. dropAllSchemaObjects(ctx, broker.pool) // Create only some tables manually. _, err = broker.pool.Exec(ctx, `CREATE TABLE IF NOT EXISTS cf_map_meta ( channel TEXT PRIMARY KEY, top_offset BIGINT NOT NULL DEFAULT 0, epoch TEXT NOT NULL DEFAULT '', version BIGINT DEFAULT 0, version_epoch TEXT, created_at TIMESTAMPTZ DEFAULT NOW(), updated_at TIMESTAMPTZ DEFAULT NOW(), expires_at TIMESTAMPTZ )`) require.NoError(t, err) // EnsureSchema should create the rest. err = broker.EnsureSchema(ctx) require.NoError(t, err) verifySchemaComplete(t, ctx, broker.pool, "cf_map_", true) verifySchemaComplete(t, ctx, broker.pool, "cf_binary_map_", false) } // TestPostgresMapBroker_EnsureSchema_BinaryData tests BYTEA columns when BinaryData=true. func TestPostgresMapBroker_EnsureSchema_BinaryData(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, BinaryData: true, }) require.NoError(t, err) t.Cleanup(func() { _ = broker.Close(ctx) _ = node.Shutdown(ctx) }) dropAllSchemaObjects(ctx, broker.pool) err = broker.EnsureSchema(ctx) require.NoError(t, err) // Verify both prefixes exist (EnsureSchema creates both). verifySchemaComplete(t, ctx, broker.pool, "cf_map_", true) verifySchemaComplete(t, ctx, broker.pool, "cf_binary_map_", false) } // TestPostgresMapBroker_EnsureSchema_FunctionalAfterSetup tests that the broker works after EnsureSchema. func TestPostgresMapBroker_EnsureSchema_FunctionalAfterSetup(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, Outbox: OutboxConfig{ PollInterval: 10 * time.Millisecond, }, }) require.NoError(t, err) // Drop everything and recreate from scratch. dropAllSchemaObjects(ctx, broker.pool) err = broker.EnsureSchema(ctx) require.NoError(t, err) err = broker.RegisterEventHandler(nil) require.NoError(t, err) t.Cleanup(func() { _ = broker.Close(ctx) _ = node.Shutdown(ctx) }) channel := fmt.Sprintf("test_ensure_func_%d", time.Now().UnixNano()) // Publish. res, err := broker.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte(`{"hello":"world"}`), }) require.NoError(t, err) require.False(t, res.Suppressed) require.Equal(t, uint64(1), res.Position.Offset) // ReadState. stateRes, err := broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{Limit: 100}) require.NoError(t, err) require.Len(t, stateRes.Publications, 1) require.Equal(t, "key1", stateRes.Publications[0].Key) // JSONB normalizes whitespace, so compare via JSONEq. require.JSONEq(t, `{"hello":"world"}`, string(stateRes.Publications[0].Data)) // Remove. removeRes, err := broker.Remove(ctx, channel, "key1", centrifuge.MapRemoveOptions{}) require.NoError(t, err) require.False(t, removeRes.Suppressed) // Verify removed. stateRes, err = broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{Limit: 100}) require.NoError(t, err) require.Empty(t, stateRes.Publications) } // TestPostgresMapBroker_EnsureSchema_VersionTracking tests that schema version is tracked. func TestPostgresMapBroker_EnsureSchema_VersionTracking(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, }) require.NoError(t, err) t.Cleanup(func() { _ = broker.Close(ctx) _ = node.Shutdown(ctx) }) dropAllSchemaObjects(ctx, broker.pool) err = broker.EnsureSchema(ctx) require.NoError(t, err) // Verify version is set in both prefix tables. for _, prefix := range []string{"cf_map_", "cf_binary_map_"} { var version int err := broker.pool.QueryRow(ctx, fmt.Sprintf(`SELECT schema_version FROM %sschema_version WHERE id = 1`, prefix), ).Scan(&version) require.NoError(t, err, "schema_version should exist for prefix %s", prefix) require.Equal(t, schemaVersion, version, "version should match schemaVersion for prefix %s", prefix) } } // TestPostgresMapBroker_EnsureSchema_BothPrefixesCreated tests that EnsureSchema creates both variants. func TestPostgresMapBroker_EnsureSchema_BothPrefixesCreated(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) // BinaryData=false, but both should still be created. broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, }) require.NoError(t, err) t.Cleanup(func() { _ = broker.Close(ctx) _ = node.Shutdown(ctx) }) dropAllSchemaObjects(ctx, broker.pool) err = broker.EnsureSchema(ctx) require.NoError(t, err) // Verify both prefixes have all tables, indexes, and functions. verifySchemaComplete(t, ctx, broker.pool, "cf_map_", true) verifySchemaComplete(t, ctx, broker.pool, "cf_binary_map_", false) // Verify shard_lock populated for both prefixes. for _, prefix := range []string{"cf_map_", "cf_binary_map_"} { var count int err := broker.pool.QueryRow(ctx, fmt.Sprintf(`SELECT COUNT(*) FROM %sshard_lock`, prefix), ).Scan(&count) require.NoError(t, err) require.Equal(t, 4, count, "shard_lock should have 4 rows for prefix %s", prefix) } } // TestPostgresMapBroker_EnsureSchema_MigrationExecution tests that migrations run correctly. func TestPostgresMapBroker_EnsureSchema_MigrationExecution(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, }) require.NoError(t, err) // This test temporarily mutates the package-level schemaVersion to test // the migration loop. Save the original value so cleanup can restore it. origSchemaVersion := schemaVersion t.Cleanup(func() { // Drop the whole schema before restoring the binary's schemaVersion: // this test leaves the DB at baseVersion+1, while the binary returns // to the lower origSchemaVersion. Any subsequent test that calls // EnsureSchema without first dropping the schema would correctly hit // the downgrade rejection. dropAllSchemaObjects(ctx, broker.pool) _ = broker.Close(ctx) _ = node.Shutdown(ctx) schemaVersion = origSchemaVersion delete(schemaMigrations, origSchemaVersion+1) }) dropAllSchemaObjects(ctx, broker.pool) // Force the initial install to be at the baseline so we can then upgrade // to baseline+1 with a migration. schemaVersion = origSchemaVersion - 1 if schemaVersion < 1 { schemaVersion = 1 } baseVersion := schemaVersion // Create baseline schema. err = broker.EnsureSchema(ctx) require.NoError(t, err) // Register migration baseline+1: template uses __PREFIX__, the framework // renders it once per variant (jsonb + binary) and runs both in the same // transaction with atomic version bumps. schemaMigrations[baseVersion+1] = ` ALTER TABLE __PREFIX__state ADD COLUMN IF NOT EXISTS test_col TEXT; ` schemaVersion = baseVersion + 1 // Run EnsureSchema again — should apply migration. err = broker.EnsureSchema(ctx) require.NoError(t, err) // Verify test_col exists on both tables. for _, prefix := range []string{"cf_map_", "cf_binary_map_"} { var exists bool err := broker.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = $1 AND column_name = 'test_col')`, prefix+"state", ).Scan(&exists) require.NoError(t, err) require.True(t, exists, "test_col should exist on %sstate", prefix) } // Verify version is now baseVersion+1. var version int err = broker.pool.QueryRow(ctx, `SELECT schema_version FROM cf_map_schema_version WHERE id = 1`, ).Scan(&version) require.NoError(t, err) require.Equal(t, baseVersion+1, version) } // TestPostgresMapBroker_EnsureSchema_MigrationIdempotent tests that migrations can run twice. func TestPostgresMapBroker_EnsureSchema_MigrationIdempotent(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, }) require.NoError(t, err) origSchemaVersion := schemaVersion t.Cleanup(func() { // Drop the whole schema — see MigrationExecution test for rationale. dropAllSchemaObjects(ctx, broker.pool) _ = broker.Close(ctx) _ = node.Shutdown(ctx) schemaVersion = origSchemaVersion delete(schemaMigrations, origSchemaVersion+1) }) dropAllSchemaObjects(ctx, broker.pool) // Force the initial install to be at the baseline. schemaVersion = origSchemaVersion - 1 if schemaVersion < 1 { schemaVersion = 1 } baseVersion := schemaVersion // Create baseline schema. err = broker.EnsureSchema(ctx) require.NoError(t, err) // Register migration baseline+1 with the standard template syntax. schemaMigrations[baseVersion+1] = ` ALTER TABLE __PREFIX__state ADD COLUMN IF NOT EXISTS test_col TEXT; ` schemaVersion = baseVersion + 1 // First migration run. err = broker.EnsureSchema(ctx) require.NoError(t, err) // Reset version in DB to force re-run of DDL + migration. _, _ = broker.pool.Exec(ctx, fmt.Sprintf( `UPDATE cf_map_schema_version SET schema_version = %d WHERE id = 1`, baseVersion)) _, _ = broker.pool.Exec(ctx, fmt.Sprintf( `UPDATE cf_binary_map_schema_version SET schema_version = %d WHERE id = 1`, baseVersion)) // Second migration run — should succeed (idempotent). err = broker.EnsureSchema(ctx) require.NoError(t, err) } // TestPostgresMapBroker_EnsureSchema_FreshInstallSkipsMigrations tests that fresh installs skip migrations. func TestPostgresMapBroker_EnsureSchema_FreshInstallSkipsMigrations(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, }) require.NoError(t, err) origSchemaVersion := schemaVersion t.Cleanup(func() { // Drop the whole schema — this test bumps schema_version above the // binary's restored origSchemaVersion, which any later test running // EnsureSchema would (correctly) reject as a downgrade. dropAllSchemaObjects(ctx, broker.pool) _ = broker.Close(ctx) _ = node.Shutdown(ctx) schemaVersion = origSchemaVersion delete(schemaMigrations, origSchemaVersion+1) }) dropAllSchemaObjects(ctx, broker.pool) // Set up baseline+1 with a migration template that adds a column. schemaMigrations[origSchemaVersion+1] = ` ALTER TABLE __PREFIX__state ADD COLUMN IF NOT EXISTS test_col TEXT; ` targetVersion := origSchemaVersion + 1 schemaVersion = targetVersion // Fresh install — DDL creates latest schema, migration should be skipped. err = broker.EnsureSchema(ctx) require.NoError(t, err) // Version should be targetVersion (set by UPDATE at end). var version int err = broker.pool.QueryRow(ctx, `SELECT schema_version FROM cf_map_schema_version WHERE id = 1`, ).Scan(&version) require.NoError(t, err) require.Equal(t, targetVersion, version) } // TestPostgresMapBroker_EnsureSchema_VersionPreservedOnDDLRerun tests that DO NOTHING preserves version. func TestPostgresMapBroker_EnsureSchema_VersionPreservedOnDDLRerun(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, }) require.NoError(t, err) origSchemaVersion := schemaVersion t.Cleanup(func() { // Drop the whole schema — this test bumps DB schema_version above // origSchemaVersion to test the fast-path "DO NOTHING preserves // version" property; a subsequent EnsureSchema would correctly // reject the ahead-of-binary DB as a downgrade. dropAllSchemaObjects(ctx, broker.pool) _ = broker.Close(ctx) _ = node.Shutdown(ctx) schemaVersion = origSchemaVersion }) dropAllSchemaObjects(ctx, broker.pool) // Create schema at the current package version. err = broker.EnsureSchema(ctx) require.NoError(t, err) // Manually bump version (simulating a previous upgrade past the current). bumpedVersion := origSchemaVersion + 1 _, err = broker.pool.Exec(ctx, fmt.Sprintf( `UPDATE cf_map_schema_version SET schema_version = %d WHERE id = 1`, bumpedVersion)) require.NoError(t, err) _, err = broker.pool.Exec(ctx, fmt.Sprintf( `UPDATE cf_binary_map_schema_version SET schema_version = %d WHERE id = 1`, bumpedVersion)) require.NoError(t, err) // Set package var so fast path matches the bumped version. schemaVersion = bumpedVersion // Call EnsureSchema — should take fast path (version matches, probe OK). err = broker.EnsureSchema(ctx) require.NoError(t, err) // Verify version is still bumpedVersion (not reset by DDL's DO NOTHING). var version int err = broker.pool.QueryRow(ctx, `SELECT schema_version FROM cf_map_schema_version WHERE id = 1`, ).Scan(&version) require.NoError(t, err) require.Equal(t, bumpedVersion, version) } // TestPostgresMapBroker_EnsureSchema_FunctionalAfterMigration tests that all operations work after migration. func TestPostgresMapBroker_EnsureSchema_FunctionalAfterMigration(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, Outbox: OutboxConfig{ PollInterval: 10 * time.Millisecond, }, }) require.NoError(t, err) t.Cleanup(func() { // Drop the whole schema — see MigrationExecution for the rationale. dropAllSchemaObjects(ctx, broker.pool) _ = broker.Close(ctx) _ = node.Shutdown(ctx) schemaVersion = 1 delete(schemaMigrations, 2) }) dropAllSchemaObjects(ctx, broker.pool) // Create v1 schema. err = broker.EnsureSchema(ctx) require.NoError(t, err) // Register and apply v2 migration via the standard template syntax. schemaMigrations[2] = ` ALTER TABLE __PREFIX__state ADD COLUMN IF NOT EXISTS test_col TEXT; ` schemaVersion = 2 err = broker.EnsureSchema(ctx) require.NoError(t, err) err = broker.RegisterEventHandler(nil) require.NoError(t, err) channel := fmt.Sprintf("test_migration_func_%d", time.Now().UnixNano()) // Publish. res, err := broker.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte(`{"hello":"world"}`), }) require.NoError(t, err) require.False(t, res.Suppressed) // ReadState. stateRes, err := broker.ReadState(ctx, channel, centrifuge.MapReadStateOptions{Limit: 100}) require.NoError(t, err) require.Len(t, stateRes.Publications, 1) require.Equal(t, "key1", stateRes.Publications[0].Key) // ReadStream. streamRes, err := broker.ReadStream(ctx, channel, centrifuge.MapReadStreamOptions{ Filter: centrifuge.StreamFilter{Limit: -1}, }) require.NoError(t, err) require.Len(t, streamRes.Publications, 1) } func TestPostgresMapBroker_ClientInfoInState(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() info := ¢rifuge.ClientInfo{ ClientID: "c1", UserID: "u1", ConnInfo: []byte("conn"), ChanInfo: []byte("chan"), } t.Run("ReadState_with_key_filter", func(t *testing.T) { ch := "client_info_state_key_ch" _, err := broker.Publish(ctx, ch, "k1", centrifuge.MapPublishOptions{ Data: []byte("data1"), ClientInfo: info, }) require.NoError(t, err) result, err := broker.ReadState(ctx, ch, centrifuge.MapReadStateOptions{ Key: "k1", }) require.NoError(t, err) require.Len(t, result.Publications, 1) pub := result.Publications[0] require.NotNil(t, pub.Info, "ClientInfo should be present in state") require.Equal(t, "c1", pub.Info.ClientID) require.Equal(t, "u1", pub.Info.UserID) require.Equal(t, []byte("conn"), pub.Info.ConnInfo) require.Equal(t, []byte("chan"), pub.Info.ChanInfo) }) t.Run("ReadState_paginated", func(t *testing.T) { ch := "client_info_state_pag_ch" _, err := broker.Publish(ctx, ch, "k1", centrifuge.MapPublishOptions{ Data: []byte("data1"), ClientInfo: info, }) require.NoError(t, err) result, err := broker.ReadState(ctx, ch, centrifuge.MapReadStateOptions{ Limit: -1, }) require.NoError(t, err) require.Len(t, result.Publications, 1) pub := result.Publications[0] require.NotNil(t, pub.Info, "ClientInfo should be present in paginated state") require.Equal(t, "c1", pub.Info.ClientID) require.Equal(t, "u1", pub.Info.UserID) require.Equal(t, []byte("conn"), pub.Info.ConnInfo) require.Equal(t, []byte("chan"), pub.Info.ChanInfo) }) } func TestPostgresMapBroker_ClientInfoInStream(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() info := ¢rifuge.ClientInfo{ ClientID: "c1", UserID: "u1", ConnInfo: []byte("conn"), ChanInfo: []byte("chan"), } t.Run("ReadStream_contains_client_info", func(t *testing.T) { ch := fmt.Sprintf("client_info_stream_ch_%d", time.Now().UnixNano()) _, err := broker.Publish(ctx, ch, "k1", centrifuge.MapPublishOptions{ Data: []byte("data1"), ClientInfo: info, }) require.NoError(t, err) result, err := broker.ReadStream(ctx, ch, centrifuge.MapReadStreamOptions{ Filter: centrifuge.StreamFilter{Limit: -1}, }) require.NoError(t, err) require.Len(t, result.Publications, 1) pub := result.Publications[0] require.NotNil(t, pub.Info, "ClientInfo should be present in stream") require.Equal(t, "c1", pub.Info.ClientID) require.Equal(t, "u1", pub.Info.UserID) require.Equal(t, []byte("conn"), pub.Info.ConnInfo) require.Equal(t, []byte("chan"), pub.Info.ChanInfo) }) } // TestPostgresMapBroker_ClientInfoDelivery_Outbox tests that ClientInfo is delivered // via outbox workers (single-node, local delivery). func TestPostgresMapBroker_ClientInfoDelivery_Outbox(t *testing.T) { connString := getPostgresConnString(t) node, err := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) require.NoError(t, err) require.NoError(t, node.Run()) type pubEvent struct { ch string pub *centrifuge.Publication } eventCh := make(chan pubEvent, 10) handler := &testBrokerEventHandler{ HandlePublicationFunc: func(ch string, pub *centrifuge.Publication, sp centrifuge.StreamPosition, delta bool, prevPub *centrifuge.Publication) error { eventCh <- pubEvent{ch: ch, pub: pub} return nil }, } e, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, BinaryData: true, NumShards: 4, Outbox: OutboxConfig{ PollInterval: 10 * time.Millisecond, BatchSize: 100, }, }) require.NoError(t, err) ctx := context.Background() require.NoError(t, e.EnsureSchema(ctx)) cleanupTestTables(ctx, e) err = e.RegisterEventHandler(handler) require.NoError(t, err) t.Cleanup(func() { _ = e.Close(context.Background()) _ = node.Shutdown(context.Background()) }) // Give outbox worker a moment to start polling. time.Sleep(50 * time.Millisecond) channel := fmt.Sprintf("test_client_info_outbox_%d", time.Now().UnixNano()) info := ¢rifuge.ClientInfo{ ClientID: "c1", UserID: "u1", ConnInfo: []byte("conn"), ChanInfo: []byte("chan"), } _, err = e.Publish(ctx, channel, "k1", centrifuge.MapPublishOptions{ Data: []byte("data1"), ClientInfo: info, }) require.NoError(t, err) for { select { case ev := <-eventCh: if ev.ch != channel { continue } require.NotNil(t, ev.pub.Info, "ClientInfo should be present in outbox delivery") require.Equal(t, "c1", ev.pub.Info.ClientID) require.Equal(t, "u1", ev.pub.Info.UserID) require.Equal(t, []byte("conn"), ev.pub.Info.ConnInfo) require.Equal(t, []byte("chan"), ev.pub.Info.ChanInfo) return case <-time.After(10 * time.Second): t.Fatal("timeout waiting for publication event") } } } // TestPostgresMapBroker_AllColumnTypes verifies that every Publication field // is correctly parsed from PostgreSQL across all three read paths: ReadState, // ReadStream, and outbox delivery (HandlePublication). This catches wire-format // mismatches (binary vs text) for all column types. func TestPostgresMapBroker_AllColumnTypes(t *testing.T) { connString := getPostgresConnString(t) node, err := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) require.NoError(t, err) require.NoError(t, node.Run()) type pubEvent struct { ch string pub *centrifuge.Publication } eventCh := make(chan pubEvent, 10) handler := &testBrokerEventHandler{ HandlePublicationFunc: func(ch string, pub *centrifuge.Publication, sp centrifuge.StreamPosition, delta bool, prevPub *centrifuge.Publication) error { eventCh <- pubEvent{ch: ch, pub: pub} return nil }, } e, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, BinaryData: true, NumShards: 4, Outbox: OutboxConfig{ PollInterval: 10 * time.Millisecond, BatchSize: 100, }, }) require.NoError(t, err) ctx := context.Background() require.NoError(t, e.EnsureSchema(ctx)) cleanupTestTables(ctx, e) err = e.RegisterEventHandler(handler) require.NoError(t, err) t.Cleanup(func() { _ = e.Close(context.Background()) _ = node.Shutdown(context.Background()) }) time.Sleep(50 * time.Millisecond) channel := fmt.Sprintf("test_all_cols_%d", time.Now().UnixNano()) info := ¢rifuge.ClientInfo{ ClientID: "client1", UserID: "user1", ConnInfo: []byte(`{"ip":"1.2.3.4"}`), ChanInfo: []byte(`{"role":"admin"}`), } tags := map[string]string{"sector": "tech", "region": "us"} // Publish with all fields populated. _, err = e.Publish(ctx, channel, "k1", centrifuge.MapPublishOptions{ Data: []byte(`{"price":100}`), Tags: tags, ClientInfo: info, }) require.NoError(t, err) // Also publish a removal to test Removed flag. _, err = e.Remove(ctx, channel, "k1", centrifuge.MapRemoveOptions{}) require.NoError(t, err) // --- Verify ReadStream (has both publish and remove entries) --- streamResult, err := e.ReadStream(ctx, channel, centrifuge.MapReadStreamOptions{ Filter: centrifuge.StreamFilter{Limit: -1}, }) require.NoError(t, err) require.Len(t, streamResult.Publications, 2) // First stream entry: the publish. sp := streamResult.Publications[0] require.Equal(t, "k1", sp.Key) require.Equal(t, []byte(`{"price":100}`), sp.Data) require.Equal(t, tags, sp.Tags) require.False(t, sp.Removed) require.NotZero(t, sp.Offset) require.NotNil(t, sp.Info, "ClientInfo must be present in stream") require.Equal(t, "client1", sp.Info.ClientID) require.Equal(t, "user1", sp.Info.UserID) require.Equal(t, []byte(`{"ip":"1.2.3.4"}`), sp.Info.ConnInfo) require.Equal(t, []byte(`{"role":"admin"}`), sp.Info.ChanInfo) // Second stream entry: the removal. sr := streamResult.Publications[1] require.Equal(t, "k1", sr.Key) require.True(t, sr.Removed) require.NotZero(t, sr.Offset) // --- Verify outbox delivery --- // Collect outbox events for our channel. var outboxPubs []*centrifuge.Publication deadline := time.After(10 * time.Second) for len(outboxPubs) < 2 { select { case ev := <-eventCh: if ev.ch == channel { outboxPubs = append(outboxPubs, ev.pub) } case <-deadline: t.Fatalf("timeout waiting for outbox events, got %d", len(outboxPubs)) } } // First outbox event: the publish. op := outboxPubs[0] require.Equal(t, "k1", op.Key) require.Equal(t, []byte(`{"price":100}`), op.Data) require.Equal(t, tags, op.Tags) require.False(t, op.Removed) require.NotZero(t, op.Offset) require.NotNil(t, op.Info, "ClientInfo must be present in outbox delivery") require.Equal(t, "client1", op.Info.ClientID) require.Equal(t, "user1", op.Info.UserID) require.Equal(t, []byte(`{"ip":"1.2.3.4"}`), op.Info.ConnInfo) require.Equal(t, []byte(`{"role":"admin"}`), op.Info.ChanInfo) // Second outbox event: the removal. or := outboxPubs[1] require.Equal(t, "k1", or.Key) require.True(t, or.Removed) // --- Verify ReadState (key was removed, so state should be empty) --- // Publish again to have a key in state for verification. _, err = e.Publish(ctx, channel, "k2", centrifuge.MapPublishOptions{ Data: []byte(`{"price":200}`), Tags: map[string]string{"sector": "finance"}, ClientInfo: info, }) require.NoError(t, err) stateResult, err := e.ReadState(ctx, channel, centrifuge.MapReadStateOptions{ Limit: -1, }) require.NoError(t, err) require.Len(t, stateResult.Publications, 1) sk := stateResult.Publications[0] require.Equal(t, "k2", sk.Key) require.Equal(t, []byte(`{"price":200}`), sk.Data) require.Equal(t, map[string]string{"sector": "finance"}, sk.Tags) require.NotZero(t, sk.Offset) require.NotNil(t, sk.Info, "ClientInfo must be present in state") require.Equal(t, "client1", sk.Info.ClientID) require.Equal(t, "user1", sk.Info.UserID) require.Equal(t, []byte(`{"ip":"1.2.3.4"}`), sk.Info.ConnInfo) require.Equal(t, []byte(`{"role":"admin"}`), sk.Info.ChanInfo) } // ============================================================================ // Redis Broker Fan-out Tests (advisory lock mode) // ============================================================================ // waitForAllShardLocksHeld blocks until every shard's outbox worker has // completed its first InitCursor snapshot — i.e., has snapshotted MAX(id) and // is actively polling. Use it before publishing in fan-out tests: otherwise a // publish committed between lock acquisition and the InitCursor snapshot // would be observed by the snapshot and silently skipped. In production this // race is benign — subscribers recover the gap via insufficient_state — but // tests register a raw BrokerEventHandler that bypasses that recovery path. // // Waiting on pg_locks is insufficient: the lock is held the moment // pg_try_advisory_lock returns, but InitCursor runs strictly after that. func waitForAllShardLocksHeld(tb testing.TB, broker *PostgresMapBroker) { tb.Helper() require.Eventually(tb, func() bool { for i := 0; i < broker.conf.NumShards; i++ { if !broker.workersInitialized[i].Load() { return false } } return true }, 10*time.Second, 50*time.Millisecond, "all shard workers should have initialized cursors") } // newTestRedisBrokerForFanout creates a RedisBroker suitable for PG fan-out testing. // Does NOT call node.Run() or SetBroker — the PG broker handles registration. func newTestRedisBrokerForFanout(tb testing.TB, n *centrifuge.Node) *centrifuge.RedisBroker { tb.Helper() redisConf := centrifuge.RedisShardConfig{ Address: "127.0.0.1:6379", IOTimeout: 10 * time.Second, ConnectTimeout: 10 * time.Second, } s, err := centrifuge.NewRedisShard(n, redisConf) require.NoError(tb, err) prefix := "pg_fanout_test_" + strconv.FormatInt(time.Now().UnixNano(), 10) broker, err := centrifuge.NewRedisBroker(n, centrifuge.RedisBrokerConfig{ Prefix: prefix, Shards: []*centrifuge.RedisShard{s}, }) require.NoError(tb, err) return broker } // TestPostgresMapBroker_RedisFanout_Delivery tests that publications reach // the event handler via Redis PUB/SUB fan-out with advisory locking. func TestPostgresMapBroker_RedisFanout_Delivery(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, err := centrifuge.New(centrifuge.Config{ LogLevel: centrifuge.LogLevelDebug, LogHandler: func(entry centrifuge.LogEntry) {}, Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) require.NoError(t, err) require.NoError(t, node.Run()) redisBroker := newTestRedisBrokerForFanout(t, node) type pubEvent struct { ch string pub *centrifuge.Publication sp centrifuge.StreamPosition delta bool prevPub *centrifuge.Publication } eventCh := make(chan pubEvent, 20) handler := &testBrokerEventHandler{ HandlePublicationFunc: func(ch string, pub *centrifuge.Publication, sp centrifuge.StreamPosition, delta bool, prevPub *centrifuge.Publication) error { eventCh <- pubEvent{ch: ch, pub: pub, sp: sp, delta: delta, prevPub: prevPub} return nil }, } broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, BinaryData: true, NumShards: 4, Broker: redisBroker, Outbox: OutboxConfig{ PollInterval: 10 * time.Millisecond, BatchSize: 100, }, }) require.NoError(t, err) require.NoError(t, broker.EnsureSchema(ctx)) cleanupTestTables(ctx, broker) err = broker.RegisterEventHandler(handler) require.NoError(t, err) t.Cleanup(func() { _ = broker.Close(context.Background()) _ = node.Shutdown(context.Background()) }) channel := fmt.Sprintf("test_redis_fanout_%d", time.Now().UnixNano()) // Subscribe via PG broker (delegates to Redis). err = broker.Subscribe(channel) require.NoError(t, err) // Wait deterministically for all shard locks to be held: the LockWorker // snapshots MAX(id) inside InitCursor on each acquisition, so any publish // that lands before that snapshot would be skipped. waitForAllShardLocksHeld(t, broker) // Publish via PG broker. const numMessages = 5 for i := 0; i < numMessages; i++ { _, err = broker.Publish(ctx, channel, fmt.Sprintf("key%d", i), centrifuge.MapPublishOptions{ Data: []byte(fmt.Sprintf("data%d", i)), }) require.NoError(t, err) } // Wait for all messages via Redis PUB/SUB. var received []pubEvent deadline := time.After(15 * time.Second) for len(received) < numMessages { select { case ev := <-eventCh: if ev.ch == channel { received = append(received, ev) } case <-deadline: t.Fatalf("timeout waiting for publications, received %d/%d", len(received), numMessages) } } require.Len(t, received, numMessages) // Verify fields propagated correctly. for _, ev := range received { require.NotZero(t, ev.sp.Offset, "offset should be set") require.NotEmpty(t, ev.sp.Epoch, "epoch should be set") require.NotEmpty(t, ev.pub.Key, "key should be set") require.NotEmpty(t, ev.pub.Data, "data should be set") } // Unsubscribe. err = broker.Unsubscribe(channel) require.NoError(t, err) } // TestPostgresMapBroker_RedisFanout_Delta tests that delta/prevPub is correctly // propagated through Redis PUB/SUB fan-out. func TestPostgresMapBroker_RedisFanout_Delta(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, err := centrifuge.New(centrifuge.Config{ LogLevel: centrifuge.LogLevelDebug, LogHandler: func(entry centrifuge.LogEntry) {}, Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) require.NoError(t, err) require.NoError(t, node.Run()) redisBroker := newTestRedisBrokerForFanout(t, node) type pubEvent struct { ch string pub *centrifuge.Publication delta bool prevPub *centrifuge.Publication } eventCh := make(chan pubEvent, 20) handler := &testBrokerEventHandler{ HandlePublicationFunc: func(ch string, pub *centrifuge.Publication, sp centrifuge.StreamPosition, delta bool, prevPub *centrifuge.Publication) error { eventCh <- pubEvent{ch: ch, pub: pub, delta: delta, prevPub: prevPub} return nil }, } broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, BinaryData: true, NumShards: 4, Broker: redisBroker, Outbox: OutboxConfig{ PollInterval: 10 * time.Millisecond, BatchSize: 100, }, }) require.NoError(t, err) require.NoError(t, broker.EnsureSchema(ctx)) cleanupTestTables(ctx, broker) err = broker.RegisterEventHandler(handler) require.NoError(t, err) t.Cleanup(func() { _ = broker.Close(context.Background()) _ = node.Shutdown(context.Background()) }) channel := fmt.Sprintf("test_redis_delta_%d", time.Now().UnixNano()) err = broker.Subscribe(channel) require.NoError(t, err) waitForAllShardLocksHeld(t, broker) waitEvent := func(t *testing.T) pubEvent { t.Helper() for { select { case ev := <-eventCh: if ev.ch == channel { return ev } case <-time.After(15 * time.Second): t.Fatal("timeout waiting for publication event") return pubEvent{} } } } // First publish with UseDelta — no previous state. _, err = broker.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("data1"), UseDelta: true, }) require.NoError(t, err) ev := waitEvent(t) require.False(t, ev.delta, "no previous data means useDelta is false") require.Nil(t, ev.prevPub, "no previous state for first publish") require.Equal(t, []byte("data1"), ev.pub.Data) // Second publish same key — should get prevPub with first data via Redis. _, err = broker.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("data1_updated"), UseDelta: true, }) require.NoError(t, err) ev = waitEvent(t) require.True(t, ev.delta) require.NotNil(t, ev.prevPub) require.Equal(t, []byte("data1"), ev.prevPub.Data) require.Equal(t, []byte("data1_updated"), ev.pub.Data) // Different key — no previous state for this key. _, err = broker.Publish(ctx, channel, "key2", centrifuge.MapPublishOptions{ Data: []byte("data2"), UseDelta: true, }) require.NoError(t, err) ev = waitEvent(t) require.False(t, ev.delta, "no previous data for key2") require.Nil(t, ev.prevPub, "no previous state for key2") err = broker.Unsubscribe(channel) require.NoError(t, err) } // TestPostgresMapBroker_RedisFanout_AdvisoryLockExclusion tests that advisory // locks ensure only one node per shard polls the stream table. func TestPostgresMapBroker_RedisFanout_AdvisoryLockExclusion(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() // Create two PG brokers sharing the same advisory lock base ID but // with different Redis brokers for fan-out. Both point at the same PG. // Only one should hold the lock per shard. node1, err := centrifuge.New(centrifuge.Config{ LogLevel: centrifuge.LogLevelDebug, LogHandler: func(entry centrifuge.LogEntry) {}, Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) require.NoError(t, err) require.NoError(t, node1.Run()) node2, err := centrifuge.New(centrifuge.Config{ LogLevel: centrifuge.LogLevelDebug, LogHandler: func(entry centrifuge.LogEntry) {}, Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) require.NoError(t, err) require.NoError(t, node2.Run()) redisBroker1 := newTestRedisBrokerForFanout(t, node1) redisBroker2 := newTestRedisBrokerForFanout(t, node2) lockBaseID := int64(900000000) + time.Now().UnixNano()%1000000 var received1, received2 int var mu1, mu2 sync.Mutex channel := fmt.Sprintf("test_lock_excl_%d", time.Now().UnixNano()) handler1 := &testBrokerEventHandler{ HandlePublicationFunc: func(ch string, pub *centrifuge.Publication, sp centrifuge.StreamPosition, delta bool, prevPub *centrifuge.Publication) error { if ch == channel { mu1.Lock() received1++ mu1.Unlock() } return nil }, } handler2 := &testBrokerEventHandler{ HandlePublicationFunc: func(ch string, pub *centrifuge.Publication, sp centrifuge.StreamPosition, delta bool, prevPub *centrifuge.Publication) error { if ch == channel { mu2.Lock() received2++ mu2.Unlock() } return nil }, } numShards := 2 broker1, err := NewPostgresMapBroker(node1, PostgresMapBrokerConfig{ DSN: connString, BinaryData: true, NumShards: numShards, Broker: redisBroker1, Outbox: OutboxConfig{ PollInterval: 10 * time.Millisecond, BatchSize: 100, AdvisoryLockBaseID: lockBaseID, AdvisoryLockRetryInterval: 500 * time.Millisecond, }, }) require.NoError(t, err) require.NoError(t, broker1.EnsureSchema(ctx)) cleanupTestTables(ctx, broker1) broker2, err := NewPostgresMapBroker(node2, PostgresMapBrokerConfig{ DSN: connString, BinaryData: true, NumShards: numShards, Broker: redisBroker2, Outbox: OutboxConfig{ PollInterval: 10 * time.Millisecond, BatchSize: 100, AdvisoryLockBaseID: lockBaseID, AdvisoryLockRetryInterval: 500 * time.Millisecond, }, }) require.NoError(t, err) err = broker1.RegisterEventHandler(handler1) require.NoError(t, err) err = broker2.RegisterEventHandler(handler2) require.NoError(t, err) // Subscribe both to the same channel. err = broker1.Subscribe(channel) require.NoError(t, err) err = broker2.Subscribe(channel) require.NoError(t, err) t.Cleanup(func() { _ = broker1.Close(context.Background()) _ = broker2.Close(context.Background()) _ = node1.Shutdown(context.Background()) _ = node2.Shutdown(context.Background()) }) // Wait until every shard's advisory lock is held by exactly one session // AND that holder has finished InitCursor (so the cursor snapshot precedes // our publish — otherwise InitCursor's MAX(id) would observe the publish // and skip it; see runOutboxWorkerWithLock). pg_locks alone is not enough: // the lock is recorded the moment pg_try_advisory_lock returns, but // InitCursor runs strictly after that inside runLockedSession. require.Eventually(t, func() bool { for i := 0; i < numShards; i++ { lockID := lockBaseID + int64(i) var count int if err := broker1.pool.QueryRow(ctx, "SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND objid = $1 AND granted = true", lockID).Scan(&count); err != nil { return false } if count != 1 { return false } // Whichever broker acquired the lock must have completed InitCursor. if !broker1.workersInitialized[i].Load() && !broker2.workersInitialized[i].Load() { return false } } return true }, 10*time.Second, 50*time.Millisecond, "each shard should have one initialized lock holder") // Publish some data and verify it arrives via one of the handlers. const numMessages = 5 for i := 0; i < numMessages; i++ { _, err = broker1.Publish(ctx, channel, fmt.Sprintf("key%d", i), centrifuge.MapPublishOptions{ Data: []byte(fmt.Sprintf("data%d", i)), }) require.NoError(t, err) } // Wait for delivery. deadline := time.After(10 * time.Second) for { mu1.Lock() mu2.Lock() total := received1 + received2 mu2.Unlock() mu1.Unlock() if total >= numMessages { break } select { case <-deadline: mu1.Lock() mu2.Lock() t.Fatalf("timeout: received1=%d received2=%d total=%d want=%d", received1, received2, received1+received2, numMessages) mu2.Unlock() mu1.Unlock() case <-time.After(100 * time.Millisecond): } } // Both handlers should have received the messages (via Redis PUB/SUB broadcast). // But the polling should only happen on one broker per shard. mu1.Lock() mu2.Lock() total := received1 + received2 mu2.Unlock() mu1.Unlock() require.GreaterOrEqual(t, total, numMessages, "total received should be at least %d", numMessages) } // TestPostgresMapBroker_RedisFanout_ClientInfo tests that ClientInfo is preserved // through Redis PUB/SUB fan-out. func TestPostgresMapBroker_RedisFanout_ClientInfo(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, err := centrifuge.New(centrifuge.Config{ LogLevel: centrifuge.LogLevelDebug, LogHandler: func(entry centrifuge.LogEntry) {}, Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) require.NoError(t, err) require.NoError(t, node.Run()) redisBroker := newTestRedisBrokerForFanout(t, node) type pubEvent struct { ch string pub *centrifuge.Publication } eventCh := make(chan pubEvent, 10) handler := &testBrokerEventHandler{ HandlePublicationFunc: func(ch string, pub *centrifuge.Publication, sp centrifuge.StreamPosition, delta bool, prevPub *centrifuge.Publication) error { eventCh <- pubEvent{ch: ch, pub: pub} return nil }, } broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, BinaryData: true, NumShards: 4, Broker: redisBroker, Outbox: OutboxConfig{ PollInterval: 10 * time.Millisecond, BatchSize: 100, }, }) require.NoError(t, err) require.NoError(t, broker.EnsureSchema(ctx)) cleanupTestTables(ctx, broker) err = broker.RegisterEventHandler(handler) require.NoError(t, err) t.Cleanup(func() { _ = broker.Close(context.Background()) _ = node.Shutdown(context.Background()) }) channel := fmt.Sprintf("test_redis_clientinfo_%d", time.Now().UnixNano()) err = broker.Subscribe(channel) require.NoError(t, err) waitForAllShardLocksHeld(t, broker) info := ¢rifuge.ClientInfo{ ClientID: "c1", UserID: "u1", ConnInfo: []byte("conn"), ChanInfo: []byte("chan"), } _, err = broker.Publish(ctx, channel, "k1", centrifuge.MapPublishOptions{ Data: []byte("data1"), ClientInfo: info, }) require.NoError(t, err) for { select { case ev := <-eventCh: if ev.ch != channel { continue } require.NotNil(t, ev.pub.Info, "ClientInfo should be present in fan-out delivery") require.Equal(t, "c1", ev.pub.Info.ClientID) require.Equal(t, "u1", ev.pub.Info.UserID) require.Equal(t, []byte("conn"), ev.pub.Info.ConnInfo) require.Equal(t, []byte("chan"), ev.pub.Info.ChanInfo) err = broker.Unsubscribe(channel) require.NoError(t, err) return case <-time.After(15 * time.Second): t.Fatal("timeout waiting for publication event") } } } // ============================================================================ // TablePrefix Tests // ============================================================================ // TestPostgresMapBroker_TablePrefix_CustomPrefix verifies that a broker // configured with a non-default TablePrefix creates its schema under the // custom prefix and can publish/read state normally — the full multi-tenant // use case. func TestPostgresMapBroker_TablePrefix_CustomPrefix(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) connString := getPostgresConnString(t) const customPrefix = "tenant_a_cf" e, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, BinaryData: true, TablePrefix: customPrefix, Outbox: OutboxConfig{ PollInterval: 10 * time.Millisecond, BatchSize: 100, }, }) require.NoError(t, err) // Ensure clean slate: drop anything lingering from a previous run under // either of the computed variant prefixes. ctx := context.Background() for _, prefix := range []string{customPrefix + "_map_", customPrefix + "_binary_map_"} { _, _ = e.pool.Exec(ctx, fmt.Sprintf("DROP FUNCTION IF EXISTS %spublish CASCADE", prefix)) _, _ = e.pool.Exec(ctx, fmt.Sprintf("DROP FUNCTION IF EXISTS %spublish_strict CASCADE", prefix)) _, _ = e.pool.Exec(ctx, fmt.Sprintf("DROP FUNCTION IF EXISTS %sremove CASCADE", prefix)) _, _ = e.pool.Exec(ctx, fmt.Sprintf("DROP FUNCTION IF EXISTS %sremove_strict CASCADE", prefix)) _, _ = e.pool.Exec(ctx, fmt.Sprintf("DROP FUNCTION IF EXISTS %sexpire_keys CASCADE", prefix)) _, _ = e.pool.Exec(ctx, fmt.Sprintf("DROP FUNCTION IF EXISTS %sstream_publish CASCADE", prefix)) _, _ = e.pool.Exec(ctx, fmt.Sprintf("DROP FUNCTION IF EXISTS %sstream_remove CASCADE", prefix)) _, _ = e.pool.Exec(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %sidempotency CASCADE", prefix)) _, _ = e.pool.Exec(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %sstream CASCADE", prefix)) _, _ = e.pool.Exec(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %sstate CASCADE", prefix)) _, _ = e.pool.Exec(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %smeta CASCADE", prefix)) _, _ = e.pool.Exec(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %sshard_lock CASCADE", prefix)) _, _ = e.pool.Exec(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %sschema_version CASCADE", prefix)) } require.NoError(t, e.EnsureSchema(ctx)) require.NoError(t, e.RegisterEventHandler(nil)) // Shutdown order matters: close broker first (stops outbox workers // polling the custom-prefix tables), then drop the schema. Otherwise // workers race the DROP statements and emit stray error logs. t.Cleanup(func() { _ = e.Close(context.Background()) _ = node.Shutdown(context.Background()) }) // Verify BOTH variants exist under the custom prefix. verifySchemaComplete(t, ctx, e.pool, customPrefix+"_map_", true) verifySchemaComplete(t, ctx, e.pool, customPrefix+"_binary_map_", false) // Verify default-prefix tables are NOT created by this broker (i.e. the // custom prefix is honored, not just appended to the default). var defaultExists bool err = e.pool.QueryRow(ctx, ` SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'cf_map_state') `).Scan(&defaultExists) require.NoError(t, err) // Note: cf_map_state may exist from other tests in this run. We can't // assert its absence — we only assert the custom-prefix tables DO exist. // The key invariant is that the broker uses custom-prefix tables, which // the publish/read below exercises. // Publish and read state to confirm the broker actually uses the custom tables. channel := "test_custom_prefix_channel" _, err = e.Publish(ctx, channel, "key1", centrifuge.MapPublishOptions{ Data: []byte("value1"), }) require.NoError(t, err) stateRes, err := e.ReadState(ctx, channel, centrifuge.MapReadStateOptions{Limit: 10}) require.NoError(t, err) state := stateToMapPostgres(stateRes.Publications) require.Equal(t, []byte("value1"), state["key1"]) // Verify the row landed in the CUSTOM-prefix state table, not the default. var rowCount int err = e.pool.QueryRow(ctx, fmt.Sprintf( `SELECT COUNT(*) FROM %s_binary_map_state WHERE channel = $1`, customPrefix), channel).Scan(&rowCount) require.NoError(t, err) require.Equal(t, 1, rowCount, "row should be stored in the custom-prefix state table") } // TestPostgresMapBroker_TablePrefix_TrailingUnderscoreTrimmed verifies that // TablePrefix values ending in one or more underscores are normalized by // setDefaults — both "cf" and "cf_" and "cf__" produce the same full prefix. func TestPostgresMapBroker_TablePrefix_TrailingUnderscoreTrimmed(t *testing.T) { cases := []struct { input string wantRoot string }{ {"cf", "cf"}, {"cf_", "cf"}, {"cf__", "cf"}, {"tenant_a", "tenant_a"}, {"tenant_a_", "tenant_a"}, {"", "cf"}, // empty defaults to "cf" } for _, tc := range cases { t.Run(tc.input, func(t *testing.T) { c := &PostgresMapBrokerConfig{TablePrefix: tc.input} c.setDefaults() require.Equal(t, tc.wantRoot, c.TablePrefix) }) } } // ============================================================================ // Partitioning Tests // ============================================================================ // newTestPostgresMapBrokerWithPartitioning creates a broker with the // partition retention worker tuned for fast test feedback. The partition // maintenance ticker runs at 100ms and retention is set to 1 day so tests // can trigger drop-old-partition behavior quickly. // // As of the always-partitioned restructure, the schema is always partitioned // — the helper name is kept for git history continuity but there's no // "non-partitioned mode" anymore. func newTestPostgresMapBrokerWithPartitioning(tb testing.TB, n *centrifuge.Node) *PostgresMapBroker { connString := getPostgresConnString(tb) e, err := NewPostgresMapBroker(n, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, BinaryData: true, CleanupInterval: 100 * time.Millisecond, PartitionLookaheadDays: 1, PartitionRetentionDays: 1, Outbox: OutboxConfig{ PollInterval: 10 * time.Millisecond, BatchSize: 100, }, }) require.NoError(tb, err) ctx := context.Background() require.NoError(tb, e.EnsureSchema(ctx)) // Force partitioning setup in case EnsureSchema took the fast path. require.NoError(tb, e.ensurePartitionedStream(ctx)) cleanupTestTables(ctx, e) err = e.RegisterEventHandler(nil) require.NoError(tb, err) tb.Cleanup(func() { _ = e.Close(context.Background()) _ = n.Shutdown(context.Background()) }) return e } // listChildPartitions returns the names of child partitions of the stream // table, via pg_inherits. func listChildPartitions(ctx context.Context, e *PostgresMapBroker) ([]string, error) { rows, err := e.pool.Query(ctx, ` SELECT c.relname FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid JOIN pg_class p ON p.oid = i.inhparent WHERE p.relname = $1 ORDER BY c.relname `, e.names.stream) if err != nil { return nil, err } defer rows.Close() var names []string for rows.Next() { var n string if err := rows.Scan(&n); err != nil { return nil, err } names = append(names, n) } return names, nil } // isPartitioned checks whether the stream table is declared PARTITION BY RANGE. func isPartitioned(ctx context.Context, e *PostgresMapBroker) (bool, error) { var ok bool err := e.pool.QueryRow(ctx, ` SELECT EXISTS( SELECT 1 FROM pg_partitioned_table WHERE partrelid = $1::regclass ) `, e.names.stream).Scan(&ok) return ok, err } // TestPostgresMapBroker_Partitioning_LookaheadAndDrop verifies the end-to-end // partitioning refactor: parent is PARTITION BY RANGE, lookahead partitions // exist, and the cleanup ticker drops partitions older than retention. func TestPostgresMapBroker_Partitioning_LookaheadAndDrop(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{}) broker := newTestPostgresMapBrokerWithPartitioning(t, node) ctx := context.Background() // Parent is declared as PARTITION BY RANGE. partitioned, err := isPartitioned(ctx, broker) require.NoError(t, err) require.True(t, partitioned, "stream table should be PARTITION BY RANGE") // Today's and tomorrow's partitions should exist (LookaheadDays=1). now := time.Now().UTC() todayName := fmt.Sprintf("%s_%s", broker.names.stream, now.Format("2006_01_02")) tomorrowName := fmt.Sprintf("%s_%s", broker.names.stream, now.AddDate(0, 0, 1).Format("2006_01_02")) names, err := listChildPartitions(ctx, broker) require.NoError(t, err) require.Contains(t, names, todayName) require.Contains(t, names, tomorrowName) // Manually create an old-dated partition that the cleanup ticker should drop. oldDay := now.AddDate(0, 0, -10) // 10 days ago — well past retention=1 oldName := fmt.Sprintf("%s_%s", broker.names.stream, oldDay.Format("2006_01_02")) oldFrom := oldDay.Format("2006-01-02") oldTo := oldDay.AddDate(0, 0, 1).Format("2006-01-02") _, err = broker.pool.Exec(ctx, fmt.Sprintf( `CREATE TABLE IF NOT EXISTS %s PARTITION OF %s FOR VALUES FROM ('%s') TO ('%s')`, oldName, broker.names.stream, oldFrom, oldTo, )) require.NoError(t, err) t.Cleanup(func() { // Best-effort cleanup in case the test fails before the ticker runs. _, _ = broker.pool.Exec(context.Background(), fmt.Sprintf("DROP TABLE IF EXISTS %s", oldName)) }) // Verify the old partition is currently listed. names, err = listChildPartitions(ctx, broker) require.NoError(t, err) require.Contains(t, names, oldName) // Wait for the cleanup ticker (CleanupInterval=100ms) to drop it. deadline := time.Now().Add(5 * time.Second) dropped := false for time.Now().Before(deadline) { names, err = listChildPartitions(ctx, broker) require.NoError(t, err) found := false for _, n := range names { if n == oldName { found = true break } } if !found { dropped = true break } time.Sleep(50 * time.Millisecond) } require.True(t, dropped, "old partition %s should have been dropped by cleanup ticker", oldName) // Today's and tomorrow's partitions should still be present. names, err = listChildPartitions(ctx, broker) require.NoError(t, err) require.Contains(t, names, todayName, "cleanup must not drop today's partition") require.Contains(t, names, tomorrowName, "cleanup must not drop tomorrow's partition") } // TestPostgresMapBroker_PartitionRetention_RetentionZero_NeverDrops verifies // the OSS-equivalent path: with PartitionRetentionDays = 0, the broker creates // lookahead partitions but never drops old ones. The pgoutbox.Partitioner // guard treats RetentionDays <= 0 as a no-op DROP — old partitions accumulate. func TestPostgresMapBroker_PartitionRetention_RetentionZero_NeverDrops(t *testing.T) { connString := getPostgresConnString(t) node, _ := centrifuge.New(centrifuge.Config{}) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, BinaryData: true, CleanupInterval: 100 * time.Millisecond, PartitionLookaheadDays: 1, PartitionRetentionDays: 0, // explicit zero — unlimited retention Outbox: OutboxConfig{ PollInterval: 10 * time.Millisecond, BatchSize: 100, }, }) require.NoError(t, err) ctx := context.Background() require.NoError(t, broker.EnsureSchema(ctx)) require.NoError(t, broker.RegisterEventHandler(nil)) t.Cleanup(func() { _ = broker.Close(ctx) _ = node.Shutdown(ctx) }) // Manually create an old partition (10 days ago). oldDay := time.Now().UTC().AddDate(0, 0, -10) oldName := fmt.Sprintf("%s_%s", broker.names.stream, oldDay.Format("2006_01_02")) oldFrom := oldDay.Format("2006-01-02") oldTo := oldDay.AddDate(0, 0, 1).Format("2006-01-02") _, err = broker.pool.Exec(ctx, fmt.Sprintf( `CREATE TABLE IF NOT EXISTS %s PARTITION OF %s FOR VALUES FROM ('%s') TO ('%s')`, oldName, broker.names.stream, oldFrom, oldTo, )) require.NoError(t, err) t.Cleanup(func() { _, _ = broker.pool.Exec(context.Background(), fmt.Sprintf("DROP TABLE IF EXISTS %s", oldName)) }) // Verify the old partition is listed. names, err := listChildPartitions(ctx, broker) require.NoError(t, err) require.Contains(t, names, oldName) // Wait for several cleanup ticks (~500ms with 100ms interval). time.Sleep(500 * time.Millisecond) // The old partition should STILL exist — RetentionDays=0 means no drops. names, err = listChildPartitions(ctx, broker) require.NoError(t, err) require.Contains(t, names, oldName, "old partition should NOT be dropped with RetentionDays=0") } // TestPostgresMapBroker_Partitioning_EnsureLookahead_Idempotent verifies that // calling ensurePartitionedStream twice is safe and does not duplicate // partitions. func TestPostgresMapBroker_Partitioning_EnsureLookahead_Idempotent(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{}) broker := newTestPostgresMapBrokerWithPartitioning(t, node) ctx := context.Background() // Snapshot the current partition list after schema init. before, err := listChildPartitions(ctx, broker) require.NoError(t, err) require.NotEmpty(t, before, "at least today+tomorrow partitions should exist after init") // Call ensurePartitionedStream a second time — must not error and must // not duplicate partitions. err = broker.ensurePartitionedStream(ctx) require.NoError(t, err, "ensurePartitionedStream must be idempotent") after, err := listChildPartitions(ctx, broker) require.NoError(t, err) require.ElementsMatch(t, before, after, "partition list should be unchanged after second ensurePartitionedStream") } // TestPostgresMapBroker_Partitioning_DropOldPartitions_IgnoresInvalidNames // verifies that partitions with names that don't match the expected // {parent}_{YYYY}_{MM}_{DD} convention are left alone by cleanup. func TestPostgresMapBroker_Partitioning_DropOldPartitions_IgnoresInvalidNames(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{}) broker := newTestPostgresMapBrokerWithPartitioning(t, node) ctx := context.Background() // Create a partition with a non-date suffix name. Partition bounds are // required, but the name deliberately does not end in _YYYY_MM_DD. malformedName := broker.names.stream + "_legacy_backup" // Use a date range far in the future so it doesn't conflict with lookahead. farFuture := time.Now().UTC().AddDate(10, 0, 0) from := farFuture.Format("2006-01-02") to := farFuture.AddDate(0, 0, 1).Format("2006-01-02") _, err := broker.pool.Exec(ctx, fmt.Sprintf( `CREATE TABLE IF NOT EXISTS %s PARTITION OF %s FOR VALUES FROM ('%s') TO ('%s')`, malformedName, broker.names.stream, from, to, )) require.NoError(t, err) t.Cleanup(func() { _, _ = broker.pool.Exec(context.Background(), fmt.Sprintf("DROP TABLE IF EXISTS %s", malformedName)) }) // Wait long enough for several cleanup ticks to have run. time.Sleep(500 * time.Millisecond) // The malformed partition must still exist — its non-standard name // means parsePartitionDate returns false and cleanup skips it. names, err := listChildPartitions(ctx, broker) require.NoError(t, err) require.Contains(t, names, malformedName, "partition with non-standard name should not be dropped by automatic cleanup") } // TestPostgresMapBroker_CheckOrder verifies the canonical check order across // brokers: Idempotency → Version → KeyMode → CAS. Mirrors the Centrifuge // shared conformance tests so PG semantics stay aligned with Memory and Redis. func TestPostgresMapBroker_CheckOrder(t *testing.T) { makeBroker := func(t *testing.T) *PostgresMapBroker { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) return newTestPostgresMapBroker(t, node) } ctx := context.Background() t.Run("Version_runs_before_KeyMode", func(t *testing.T) { broker := makeBroker(t) ch := "order_v_before_km_" + strconv.FormatInt(time.Now().UnixNano(), 36) _, err := broker.Publish(ctx, ch, "k", centrifuge.MapPublishOptions{ Data: []byte("v1"), Version: 10, }) require.NoError(t, err) res, err := broker.Publish(ctx, ch, "k", centrifuge.MapPublishOptions{ Data: []byte("v2"), Version: 5, KeyMode: centrifuge.KeyModeIfNew, }) require.NoError(t, err) require.True(t, res.Suppressed) require.Equal(t, centrifuge.SuppressReasonVersion, res.SuppressReason, "Version check must run before KeyMode") }) t.Run("KeyMode_runs_before_CAS", func(t *testing.T) { broker := makeBroker(t) ch := "order_km_before_cas_" + strconv.FormatInt(time.Now().UnixNano(), 36) res1, err := broker.Publish(ctx, ch, "k", centrifuge.MapPublishOptions{ Data: []byte("v1"), }) require.NoError(t, err) require.False(t, res1.Suppressed) res, err := broker.Publish(ctx, ch, "k", centrifuge.MapPublishOptions{ Data: []byte("v2"), KeyMode: centrifuge.KeyModeIfNew, ExpectedPosition: ¢rifuge.StreamPosition{ Offset: 999, Epoch: res1.Position.Epoch, }, }) require.NoError(t, err) require.True(t, res.Suppressed) require.Equal(t, centrifuge.SuppressReasonKeyExists, res.SuppressReason, "KeyMode check must run before CAS") }) t.Run("Version_runs_before_CAS", func(t *testing.T) { broker := makeBroker(t) ch := "order_v_before_cas_" + strconv.FormatInt(time.Now().UnixNano(), 36) res1, err := broker.Publish(ctx, ch, "k", centrifuge.MapPublishOptions{ Data: []byte("v1"), Version: 10, }) require.NoError(t, err) res, err := broker.Publish(ctx, ch, "k", centrifuge.MapPublishOptions{ Data: []byte("v2"), Version: 5, ExpectedPosition: ¢rifuge.StreamPosition{ Offset: 999, Epoch: res1.Position.Epoch, }, }) require.NoError(t, err) require.True(t, res.Suppressed) require.Equal(t, centrifuge.SuppressReasonVersion, res.SuppressReason, "Version check must run before CAS") }) } // TestPostgresMapBroker_VersionPreserved verifies that publishing a key // without a version does NOT reset the stored version (matches Redis + // Memory). Mirrors the Centrifuge shared conformance test. func TestPostgresMapBroker_VersionPreserved(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() ch := "version_preserved_" + strconv.FormatInt(time.Now().UnixNano(), 36) _, err := broker.Publish(ctx, ch, "k", centrifuge.MapPublishOptions{ Data: []byte("v10"), Version: 10, }) require.NoError(t, err) _, err = broker.Publish(ctx, ch, "k", centrifuge.MapPublishOptions{ Data: []byte("unversioned"), }) require.NoError(t, err) res, err := broker.Publish(ctx, ch, "k", centrifuge.MapPublishOptions{ Data: []byte("v5_should_be_dropped"), Version: 5, }) require.NoError(t, err) require.True(t, res.Suppressed, "stale version should still be suppressed after unversioned publish") require.Equal(t, centrifuge.SuppressReasonVersion, res.SuppressReason) res, err = broker.Publish(ctx, ch, "k", centrifuge.MapPublishOptions{ Data: []byte("v11"), Version: 11, }) require.NoError(t, err) require.False(t, res.Suppressed, "newer version should still be accepted after unversioned publish") } // TestPostgresMapBroker_RefreshTTLOnSuppress_RefreshesMetaTTL verifies that a // suppressed if_new+refresh_ttl_on_suppress publish (the periodic presence // keepalive path) extends meta.expires_at. Without this, meta dies under quiet // steady-state and the next publish creates a new epoch — clients see a // spurious ErrorUnrecoverablePosition. func TestPostgresMapBroker_RefreshTTLOnSuppress_RefreshesMetaTTL(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, StreamTTL: 60 * time.Second, MetaTTL: 120 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() ch := fmt.Sprintf("test_meta_ttl_refresh_%d", time.Now().UnixNano()) // First publish creates meta with expires_at = NOW() + MetaTTL. _, err := broker.Publish(ctx, ch, "k", centrifuge.MapPublishOptions{ Data: []byte("v1"), KeyMode: centrifuge.KeyModeIfNew, }) require.NoError(t, err) var firstExpiresAt time.Time err = broker.pool.QueryRow(ctx, fmt.Sprintf( `SELECT expires_at FROM %s WHERE channel = $1`, broker.names.meta), ch, ).Scan(&firstExpiresAt) require.NoError(t, err) require.False(t, firstExpiresAt.IsZero(), "meta expires_at must be set when MetaTTL > 0") // Wait long enough for NOW()+MetaTTL on the second call to clearly exceed // the first expires_at; needs to be larger than clock resolution. time.Sleep(1100 * time.Millisecond) // Suppressed keepalive: same key, if_new + refresh_ttl_on_suppress. res, err := broker.Publish(ctx, ch, "k", centrifuge.MapPublishOptions{ Data: []byte("v1"), KeyMode: centrifuge.KeyModeIfNew, RefreshTTLOnSuppress: true, }) require.NoError(t, err) require.True(t, res.Suppressed) require.Equal(t, centrifuge.SuppressReasonKeyExists, res.SuppressReason) var secondExpiresAt time.Time err = broker.pool.QueryRow(ctx, fmt.Sprintf( `SELECT expires_at FROM %s WHERE channel = $1`, broker.names.meta), ch, ).Scan(&secondExpiresAt) require.NoError(t, err) require.True(t, secondExpiresAt.After(firstExpiresAt), "suppressed keepalive must extend meta.expires_at: was %s, now %s", firstExpiresAt, secondExpiresAt) } // TestPostgresMapBroker_RefreshTTLOnSuppress_DoesNotExtendWhenFlagOff verifies // the negative case — without RefreshTTLOnSuppress, meta TTL is unchanged. // Guards against accidentally bumping meta on every suppressed publish. func TestPostgresMapBroker_RefreshTTLOnSuppress_DoesNotExtendWhenFlagOff(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, StreamTTL: 60 * time.Second, MetaTTL: 120 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() ch := fmt.Sprintf("test_meta_ttl_no_refresh_%d", time.Now().UnixNano()) _, err := broker.Publish(ctx, ch, "k", centrifuge.MapPublishOptions{ Data: []byte("v1"), KeyMode: centrifuge.KeyModeIfNew, }) require.NoError(t, err) var firstExpiresAt time.Time err = broker.pool.QueryRow(ctx, fmt.Sprintf( `SELECT expires_at FROM %s WHERE channel = $1`, broker.names.meta), ch, ).Scan(&firstExpiresAt) require.NoError(t, err) time.Sleep(1100 * time.Millisecond) // Suppressed publish without RefreshTTLOnSuppress. res, err := broker.Publish(ctx, ch, "k", centrifuge.MapPublishOptions{ Data: []byte("v1"), KeyMode: centrifuge.KeyModeIfNew, }) require.NoError(t, err) require.True(t, res.Suppressed) var secondExpiresAt time.Time err = broker.pool.QueryRow(ctx, fmt.Sprintf( `SELECT expires_at FROM %s WHERE channel = $1`, broker.names.meta), ch, ).Scan(&secondExpiresAt) require.NoError(t, err) require.True(t, secondExpiresAt.Equal(firstExpiresAt), "suppressed publish without RefreshTTLOnSuppress must not change meta.expires_at") } // TestPostgresMapBroker_ReadStream_FiltersDeadEpoch verifies that ReadStream // rejects rows from a previous epoch that linger in the partitioned stream // table after meta TTL expiry + recreation. Reproduces the bug where a fresh // subscriber received hundreds of stale publications. func TestPostgresMapBroker_ReadStream_FiltersDeadEpoch(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() ch := fmt.Sprintf("test_dead_epoch_%d", time.Now().UnixNano()) // Build the dead epoch: 3 publishes leave 3 stream rows tagged with epoch1. for i := 0; i < 3; i++ { _, err := broker.Publish(ctx, ch, fmt.Sprintf("k%d", i), centrifuge.MapPublishOptions{ Data: []byte(fmt.Sprintf("dead-%d", i)), }) require.NoError(t, err) } // Capture the dead epoch and drop the meta row. This is what // cleanupEntries would do when meta.expires_at < NOW(); we do it // directly so the test doesn't depend on TTL timing. var deadEpoch string err := broker.pool.QueryRow(ctx, fmt.Sprintf( `SELECT epoch FROM %s WHERE channel = $1`, broker.names.meta), ch, ).Scan(&deadEpoch) require.NoError(t, err) _, err = broker.pool.Exec(ctx, fmt.Sprintf( `DELETE FROM %s WHERE channel = $1`, broker.names.meta), ch) require.NoError(t, err) // Next publish creates fresh meta with a new random epoch + top_offset=0. res, err := broker.Publish(ctx, ch, "k0", centrifuge.MapPublishOptions{ Data: []byte("alive-0"), }) require.NoError(t, err) require.False(t, res.Suppressed) require.NotEqual(t, deadEpoch, res.Position.Epoch, "new meta must have a fresh epoch") require.Equal(t, uint64(1), res.Position.Offset, "fresh meta starts at offset 1") // Sanity: the partitioned stream table still contains rows from the dead // epoch (partition retention hasn't run). Without the epoch predicate // fix, ReadStream would return all 4 rows; with the fix, only the // current-epoch row is returned. var totalRows int err = broker.pool.QueryRow(ctx, fmt.Sprintf( `SELECT COUNT(*) FROM %s WHERE channel = $1`, broker.names.stream), ch, ).Scan(&totalRows) require.NoError(t, err) require.Equal(t, 4, totalRows, "stream must still contain dead-epoch rows for the test premise to hold") // Forward read from offset 0 must return only the live-epoch row. streamRes, err := broker.ReadStream(ctx, ch, centrifuge.MapReadStreamOptions{ Filter: centrifuge.StreamFilter{ Since: ¢rifuge.StreamPosition{Offset: 0}, Limit: 100, }, }) require.NoError(t, err) require.Len(t, streamRes.Publications, 1, "ReadStream must filter out dead-epoch rows") require.Equal(t, []byte("alive-0"), streamRes.Publications[0].Data) require.Equal(t, uint64(1), streamRes.Publications[0].Offset) require.Equal(t, res.Position.Epoch, streamRes.Position.Epoch) // Reverse read also enforces the predicate. since := centrifuge.StreamPosition{Offset: uint64(1 << 62)} streamRes, err = broker.ReadStream(ctx, ch, centrifuge.MapReadStreamOptions{ Filter: centrifuge.StreamFilter{ Since: &since, Limit: 100, Reverse: true, }, }) require.NoError(t, err) require.Len(t, streamRes.Publications, 1, "reverse ReadStream must also filter dead-epoch rows") require.Equal(t, []byte("alive-0"), streamRes.Publications[0].Data) } // TestPostgresMapBroker_PublishWipesStateOnEpochReset verifies that when // cf_map_publish creates a fresh meta row (UPSERT inserted, not updated), the // channel's prior state rows are wiped. Without this, ReadState/Stats would // return zombie keys with stale key_offsets under the new meta's epoch — a // transient inconsistency that lasts until expire_keys cleans them up. func TestPostgresMapBroker_PublishWipesStateOnEpochReset(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() ch := fmt.Sprintf("test_state_wipe_%d", time.Now().UnixNano()) // Build state under epoch_old: three keys. for _, k := range []string{"k1", "k2", "k3"} { _, err := broker.Publish(ctx, ch, k, centrifuge.MapPublishOptions{ Data: []byte("dead"), }) require.NoError(t, err) } // Drop meta to simulate cleanupEntries firing while state rows linger // (their expires_at hasn't been reaped by expire_keys yet). var deadEpoch string err := broker.pool.QueryRow(ctx, fmt.Sprintf( `SELECT epoch FROM %s WHERE channel = $1`, broker.names.meta), ch, ).Scan(&deadEpoch) require.NoError(t, err) _, err = broker.pool.Exec(ctx, fmt.Sprintf( `DELETE FROM %s WHERE channel = $1`, broker.names.meta), ch) require.NoError(t, err) // Pre-fix: the three k1/k2/k3 rows would still be in cf_map_state. var preCount int err = broker.pool.QueryRow(ctx, fmt.Sprintf( `SELECT COUNT(*) FROM %s WHERE channel = $1`, broker.names.state), ch, ).Scan(&preCount) require.NoError(t, err) require.Equal(t, 3, preCount, "test premise: zombie state rows must exist before fresh publish") // Fresh publish creates new meta + new epoch + new state row for "k4". res, err := broker.Publish(ctx, ch, "k4", centrifuge.MapPublishOptions{ Data: []byte("alive"), }) require.NoError(t, err) require.NotEqual(t, deadEpoch, res.Position.Epoch) require.Equal(t, uint64(1), res.Position.Offset) // Only the new key must remain; the three zombies must be gone. stateRes, err := broker.ReadState(ctx, ch, centrifuge.MapReadStateOptions{Limit: 100}) require.NoError(t, err) require.Len(t, stateRes.Publications, 1, "ReadState must return only the new-epoch row") require.Equal(t, "k4", stateRes.Publications[0].Key) require.Equal(t, []byte("alive"), stateRes.Publications[0].Data) require.Equal(t, uint64(1), stateRes.Publications[0].Offset) stats, err := broker.Stats(ctx, ch) require.NoError(t, err) require.Equal(t, 1, stats.NumKeys, "Stats must reflect wiped zombies") } // TestPostgresMapBroker_PublishDoesNotWipeStateOnNormalPublish guards against // the wipe firing when meta already exists — every subsequent publish must // preserve other keys' state, only the conflict path runs. func TestPostgresMapBroker_PublishDoesNotWipeStateOnNormalPublish(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() ch := fmt.Sprintf("test_state_no_wipe_%d", time.Now().UnixNano()) for _, k := range []string{"k1", "k2", "k3"} { _, err := broker.Publish(ctx, ch, k, centrifuge.MapPublishOptions{ Data: []byte("v"), }) require.NoError(t, err) } // Subsequent publish hits the CONFLICT path (xmax != 0), so state must // NOT be wiped. _, err := broker.Publish(ctx, ch, "k4", centrifuge.MapPublishOptions{ Data: []byte("v"), }) require.NoError(t, err) stateRes, err := broker.ReadState(ctx, ch, centrifuge.MapReadStateOptions{Limit: 100}) require.NoError(t, err) require.Len(t, stateRes.Publications, 4, "all four keys must remain after a normal publish") } // TestPostgresMapBroker_RemoveAtomicVsCleanup reproduces the race where // cf_map_remove's SELECT was not FOR UPDATE: a concurrent meta DELETE // (simulating cleanupEntries) between the SELECT and the UPDATE would let // remove insert a stream row under a dead epoch with a stale v_offset. With // the FOR UPDATE fix the meta delete blocks until remove commits, so when we // DELETE the meta row first and then call Remove the function correctly // returns key_not_found instead of leaving a phantom stream row. func TestPostgresMapBroker_RemoveAtomicVsCleanup(t *testing.T) { node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker := newTestPostgresMapBroker(t, node) ctx := context.Background() ch := fmt.Sprintf("test_remove_race_%d", time.Now().UnixNano()) _, err := broker.Publish(ctx, ch, "k", centrifuge.MapPublishOptions{ Data: []byte("v"), }) require.NoError(t, err) // Simulate cleanupEntries deleting the meta row before the remove call. // We also drop the state row — otherwise the wiping fix from // PublishWipesStateOnEpochReset would have removed it on the next // publish, but here we want to test the remove path specifically. _, err = broker.pool.Exec(ctx, fmt.Sprintf( `DELETE FROM %s WHERE channel = $1`, broker.names.meta), ch) require.NoError(t, err) // Remove must observe meta is gone and report key_not_found, not silently // proceed with a stale v_offset and insert a phantom stream row. res, err := broker.Remove(ctx, ch, "k", centrifuge.MapRemoveOptions{}) require.NoError(t, err) require.True(t, res.Suppressed, "Remove must be suppressed when meta is missing") require.Equal(t, centrifuge.SuppressReasonKeyNotFound, res.SuppressReason) // Stream must NOT contain a removal row inserted by the racing remove // (channel had 1 publication; with the fix, no further row is appended). var streamCount int err = broker.pool.QueryRow(ctx, fmt.Sprintf( `SELECT COUNT(*) FROM %s WHERE channel = $1`, broker.names.stream), ch, ).Scan(&streamCount) require.NoError(t, err) require.Equal(t, 1, streamCount, "no phantom removal row may be inserted under a dead epoch") } // TestPostgresMapBroker_EnsureSchema_DowngradeRejected verifies the EnsureSchema // safety property: when the DB's schema_version is NEWER than the binary's // schemaVersion (e.g. an operator rolled back the binary without restoring an // older DB snapshot), EnsureSchema must REFUSE rather than silently rewrite // the row backward and leave columns from the newer migrations stranded. func TestPostgresMapBroker_EnsureSchema_DowngradeRejected(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, }) require.NoError(t, err) t.Cleanup(func() { // Reset the schema completely — this test intentionally leaves // schema_version in a state that no subsequent EnsureSchema can run on. dropAllSchemaObjects(ctx, broker.pool) _ = broker.Close(ctx) _ = node.Shutdown(ctx) }) dropAllSchemaObjects(ctx, broker.pool) // First, create baseline schema at the current binary's schemaVersion. require.NoError(t, broker.EnsureSchema(ctx)) // Simulate a future DB: set schema_version to one past the current binary. futureVersion := schemaVersion + 1 for _, prefix := range []string{"cf_map_", "cf_binary_map_"} { _, err = broker.pool.Exec(ctx, fmt.Sprintf( `UPDATE %sschema_version SET schema_version = $1 WHERE id = 1`, prefix), futureVersion) require.NoError(t, err) } // EnsureSchema must reject — not silently rewrite the row backward. err = broker.EnsureSchema(ctx) require.Error(t, err) require.Contains(t, err.Error(), "downgrade not supported") // Verify the schema_version row is unchanged (binary didn't quietly roll it back). for _, prefix := range []string{"cf_map_", "cf_binary_map_"} { var v int err = broker.pool.QueryRow(ctx, fmt.Sprintf( `SELECT schema_version FROM %sschema_version WHERE id = 1`, prefix)).Scan(&v) require.NoError(t, err) require.Equal(t, futureVersion, v, "downgrade attempt must not mutate %sschema_version", prefix) } } // TestPostgresMapBroker_EnsureSchema_TransientReadPropagates is the critical // safety test for the read-discrimination change: a SELECT failure that is NOT // "table missing" or "row missing" must propagate as an error, NOT be silently // treated as a fresh install. The pre-fix code path would have skipped // migrations and forced schema_version forward, leaving the DB at the old // shape while the row claimed the new version (silent corruption). // // We simulate the transient-read class by replacing schema_version with a // table that lacks the `schema_version` column — SELECT then errors with // 42703 (undefined_column), the same propagation path as a permission denied // or timeout would take. func TestPostgresMapBroker_EnsureSchema_TransientReadPropagates(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, }) require.NoError(t, err) t.Cleanup(func() { // The schema_version table this test creates has the wrong shape — // no subsequent EnsureSchema can read it. Reset before exit. dropAllSchemaObjects(ctx, broker.pool) _ = broker.Close(ctx) _ = node.Shutdown(ctx) }) dropAllSchemaObjects(ctx, broker.pool) // Build the active variant's schema_version table with the WRONG column name. // EnsureSchema will SELECT schema_version, get 42703 — must NOT silently // proceed as fresh install. _, err = broker.pool.Exec(ctx, ` CREATE TABLE cf_map_schema_version ( id INTEGER PRIMARY KEY, wrong_column INTEGER NOT NULL )`) require.NoError(t, err) _, err = broker.pool.Exec(ctx, `INSERT INTO cf_map_schema_version (id, wrong_column) VALUES (1, 42)`) require.NoError(t, err) err = broker.EnsureSchema(ctx) require.Error(t, err, "non-fresh non-NoRows error must propagate") require.Contains(t, err.Error(), "read schema_version") } // TestPostgresMapBroker_EnsureSchema_MigrationFailureRollsBackVersion // verifies that a failing migration leaves the version row at the previous // value AND the partial DDL changes are rolled back. Atomic migration + version // bump is the whole point of running them in a single transaction. func TestPostgresMapBroker_EnsureSchema_MigrationFailureRollsBackVersion(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, }) require.NoError(t, err) origSchemaVersion := schemaVersion t.Cleanup(func() { _ = broker.Close(ctx) _ = node.Shutdown(ctx) schemaVersion = origSchemaVersion delete(schemaMigrations, origSchemaVersion+1) for _, prefix := range []string{"cf_map_", "cf_binary_map_"} { _, _ = broker.pool.Exec(ctx, fmt.Sprintf( `ALTER TABLE %sstate DROP COLUMN IF EXISTS rollback_probe_col`, prefix)) } }) dropAllSchemaObjects(ctx, broker.pool) // Start at baseline. schemaVersion = origSchemaVersion require.NoError(t, broker.EnsureSchema(ctx)) // Register a migration that ADDs a column then errors. Both effects must // roll back: column must not exist AND schema_version stays at baseline. baseVersion := schemaVersion schemaMigrations[baseVersion+1] = ` ALTER TABLE __PREFIX__state ADD COLUMN rollback_probe_col TEXT; -- Now force a failure inside the same tx. SELECT undefined_function_that_does_not_exist(); ` schemaVersion = baseVersion + 1 err = broker.EnsureSchema(ctx) require.Error(t, err, "migration with failing statement must abort EnsureSchema") // Verify column does NOT exist (full migration rolled back). for _, prefix := range []string{"cf_map_", "cf_binary_map_"} { var exists bool err := broker.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = $1 AND column_name = 'rollback_probe_col')`, prefix+"state", ).Scan(&exists) require.NoError(t, err) require.False(t, exists, "rollback_probe_col must not exist on %sstate after failed migration", prefix) } // Verify schema_version is STILL at baseline (no partial commit). for _, prefix := range []string{"cf_map_", "cf_binary_map_"} { var v int err = broker.pool.QueryRow(ctx, fmt.Sprintf( `SELECT schema_version FROM %sschema_version WHERE id = 1`, prefix)).Scan(&v) require.NoError(t, err) require.Equal(t, baseVersion, v, "schema_version must not have advanced past baseline") } } // TestPostgresMapBroker_EnsureSchema_MultiStepMigrationChain verifies that // migrations are applied in order, each in its own committed transaction. // We bump schemaVersion by two steps with both migrations registered and // confirm both columns were added and schema_version ends at the new top. func TestPostgresMapBroker_EnsureSchema_MultiStepMigrationChain(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, }) require.NoError(t, err) origSchemaVersion := schemaVersion t.Cleanup(func() { // This test leaves schema_version=origSchemaVersion+2 in the DB while // the package-level schemaVersion is restored to origSchemaVersion — // a downgrade situation for any subsequent EnsureSchema. Drop the // whole schema so the next test starts fresh. dropAllSchemaObjects(ctx, broker.pool) _ = broker.Close(ctx) _ = node.Shutdown(ctx) schemaVersion = origSchemaVersion delete(schemaMigrations, origSchemaVersion+1) delete(schemaMigrations, origSchemaVersion+2) }) dropAllSchemaObjects(ctx, broker.pool) require.NoError(t, broker.EnsureSchema(ctx)) baseVersion := schemaVersion schemaMigrations[baseVersion+1] = ` ALTER TABLE __PREFIX__state ADD COLUMN chain_col_a TEXT; ` schemaMigrations[baseVersion+2] = ` ALTER TABLE __PREFIX__state ADD COLUMN chain_col_b TEXT; ` schemaVersion = baseVersion + 2 require.NoError(t, broker.EnsureSchema(ctx)) // Both columns present. for _, prefix := range []string{"cf_map_", "cf_binary_map_"} { for _, col := range []string{"chain_col_a", "chain_col_b"} { var exists bool err = broker.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = $1 AND column_name = $2)`, prefix+"state", col, ).Scan(&exists) require.NoError(t, err) require.True(t, exists, "%s should exist on %sstate", col, prefix) } } // schema_version landed at the new top. for _, prefix := range []string{"cf_map_", "cf_binary_map_"} { var v int err = broker.pool.QueryRow(ctx, fmt.Sprintf( `SELECT schema_version FROM %sschema_version WHERE id = 1`, prefix)).Scan(&v) require.NoError(t, err) require.Equal(t, baseVersion+2, v) } } // TestPostgresMapBroker_EnsureSchema_LargeVersionJump simulates a user who // ran Centrifugo at schema_version=1, then upgraded several releases later to // a binary at schema_version=4. The migration framework must apply v2, v3, // AND v4 in order on a single EnsureSchema call — never skip a step, never // run them out of order. This is the realistic "Centrifugo evolved a lot" // upgrade path the user explicitly called out. func TestPostgresMapBroker_EnsureSchema_LargeVersionJump(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, }) require.NoError(t, err) origSchemaVersion := schemaVersion t.Cleanup(func() { dropAllSchemaObjects(ctx, broker.pool) _ = broker.Close(ctx) _ = node.Shutdown(ctx) schemaVersion = origSchemaVersion for v := origSchemaVersion + 1; v <= origSchemaVersion+3; v++ { delete(schemaMigrations, v) } }) dropAllSchemaObjects(ctx, broker.pool) // Pin baseline at 1 so we can simulate the "user is at v1" scenario, // regardless of what the current production schemaVersion is. Then bring // the binary up to v=baseline+3 with three registered migrations. schemaVersion = origSchemaVersion if origSchemaVersion > 1 { // Run baseline first using the current value, then force-reset to 1. require.NoError(t, broker.EnsureSchema(ctx)) for _, prefix := range []string{"cf_map_", "cf_binary_map_"} { _, err = broker.pool.Exec(ctx, fmt.Sprintf( `UPDATE %sschema_version SET schema_version = 1 WHERE id = 1`, prefix)) require.NoError(t, err) } } else { require.NoError(t, broker.EnsureSchema(ctx)) } // Three migrations as TEMPLATES — the framework renders each once per // variant (jsonb + binary) inside a single transaction. schemaMigrations[origSchemaVersion+1] = ` ALTER TABLE __PREFIX__state ADD COLUMN IF NOT EXISTS jump_col_v2 TEXT; ` schemaMigrations[origSchemaVersion+2] = ` ALTER TABLE __PREFIX__state ADD COLUMN IF NOT EXISTS jump_col_v3 TEXT; ` schemaMigrations[origSchemaVersion+3] = ` ALTER TABLE __PREFIX__state ADD COLUMN IF NOT EXISTS jump_col_v4 TEXT; ` t.Cleanup(func() { for _, prefix := range []string{"cf_map_", "cf_binary_map_"} { for _, col := range []string{"jump_col_v2", "jump_col_v3", "jump_col_v4"} { _, _ = broker.pool.Exec(ctx, fmt.Sprintf( `ALTER TABLE %sstate DROP COLUMN IF EXISTS %s`, prefix, col)) } } }) schemaVersion = origSchemaVersion + 3 // Single EnsureSchema call must apply all three migrations. require.NoError(t, broker.EnsureSchema(ctx)) // All three columns present on both variants. for _, prefix := range []string{"cf_map_", "cf_binary_map_"} { for _, col := range []string{"jump_col_v2", "jump_col_v3", "jump_col_v4"} { var exists bool err = broker.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = $1 AND column_name = $2)`, prefix+"state", col, ).Scan(&exists) require.NoError(t, err) require.True(t, exists, "%s should exist on %sstate after large-jump upgrade", col, prefix) } } // Final schema_version = top. for _, prefix := range []string{"cf_map_", "cf_binary_map_"} { var v int err = broker.pool.QueryRow(ctx, fmt.Sprintf( `SELECT schema_version FROM %sschema_version WHERE id = 1`, prefix)).Scan(&v) require.NoError(t, err) require.Equal(t, origSchemaVersion+3, v) } } // TestPostgresMapBroker_EnsureSchema_ConcurrentMigratorsLockSerialised // exercises the advisory-lock serialisation guarantee: two EnsureSchema calls // running concurrently against the same DB must run the migration chain // exactly once between them. We register a NON-IDEMPOTENT migration (INSERT // without ON CONFLICT) so a missing lock would produce two rows and the test // would catch it. func TestPostgresMapBroker_EnsureSchema_ConcurrentMigratorsLockSerialised(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, }) require.NoError(t, err) origSchemaVersion := schemaVersion t.Cleanup(func() { dropAllSchemaObjects(ctx, broker.pool) _, _ = broker.pool.Exec(ctx, `DROP TABLE IF EXISTS cf_lock_test_probe CASCADE`) _ = broker.Close(ctx) _ = node.Shutdown(ctx) schemaVersion = origSchemaVersion delete(schemaMigrations, origSchemaVersion+1) }) dropAllSchemaObjects(ctx, broker.pool) _, err = broker.pool.Exec(ctx, `DROP TABLE IF EXISTS cf_lock_test_probe CASCADE`) require.NoError(t, err) // Baseline. require.NoError(t, broker.EnsureSchema(ctx)) // Reset DB version to baseline-1 so the new migration we register has // somewhere to apply against. Probe table for counting migration runs. for _, prefix := range []string{"cf_map_", "cf_binary_map_"} { _, err = broker.pool.Exec(ctx, fmt.Sprintf( `UPDATE %sschema_version SET schema_version = $1 WHERE id = 1`, prefix), origSchemaVersion) require.NoError(t, err) } _, err = broker.pool.Exec(ctx, `CREATE TABLE cf_lock_test_probe (n SERIAL PRIMARY KEY)`) require.NoError(t, err) // Non-idempotent migration: a plain INSERT. The framework renders the // migration once per variant (jsonb + binary) inside a SINGLE transaction, // so ONE successful migration cycle yields 2 rows. Without the advisory // lock serialising the migration loop, two concurrent EnsureSchema calls // would both pass the initial version check, both enter the loop, both // commit → 4 rows. With the lock + re-read inside the lock, only one // migrator runs the chain → 2 rows. schemaMigrations[origSchemaVersion+1] = ` INSERT INTO cf_lock_test_probe DEFAULT VALUES; ` schemaVersion = origSchemaVersion + 1 // Run two EnsureSchema goroutines that start nearly together. const N = 2 errCh := make(chan error, N) startGate := make(chan struct{}) for i := 0; i < N; i++ { go func() { <-startGate errCh <- broker.EnsureSchema(ctx) }() } close(startGate) for i := 0; i < N; i++ { require.NoError(t, <-errCh) } // Probe table must have exactly two rows — one per variant in the single // migration cycle that the advisory lock allowed to run. var rows int err = broker.pool.QueryRow(ctx, `SELECT COUNT(*) FROM cf_lock_test_probe`).Scan(&rows) require.NoError(t, err) require.Equal(t, 2, rows, "advisory lock must serialise migrations so the non-idempotent INSERT runs exactly once per variant (2 variants × 1 cycle)") // Final schema_version reflects success on both variants. for _, prefix := range []string{"cf_map_", "cf_binary_map_"} { var v int err = broker.pool.QueryRow(ctx, fmt.Sprintf( `SELECT schema_version FROM %sschema_version WHERE id = 1`, prefix)).Scan(&v) require.NoError(t, err) require.Equal(t, origSchemaVersion+1, v) } } // TestPostgresMapBroker_EnsureSchema_MigrationTemplateUsesCustomTablePrefix // proves the templating path: a migration written with __PREFIX__ works // against a non-default TablePrefix. Pre-templating, migration authors // would have to hardcode `cf_map_` / `cf_binary_map_` in every migration, // silently breaking any operator who set `table_prefix: `. func TestPostgresMapBroker_EnsureSchema_MigrationTemplateUsesCustomTablePrefix(t *testing.T) { connString := getPostgresConnString(t) ctx := context.Background() node, _ := centrifuge.New(centrifuge.Config{ Map: centrifuge.MapConfig{ GetMapChannelOptions: func(channel string) centrifuge.MapChannelOptions { return centrifuge.MapChannelOptions{ Mode: centrifuge.MapModeRecoverable, KeyTTL: 60 * time.Second, } }, }, }) const customPrefix = "tplprobe" broker, err := NewPostgresMapBroker(node, PostgresMapBrokerConfig{ DSN: connString, NumShards: 4, TablePrefix: customPrefix, }) require.NoError(t, err) origSchemaVersion := schemaVersion cleanup := func() { // Drop all objects created with the custom prefix. for _, prefix := range []string{customPrefix + "_map_", customPrefix + "_binary_map_"} { for _, fn := range []string{"publish", "publish_strict", "remove", "remove_strict", "expire_keys"} { _, _ = broker.pool.Exec(ctx, fmt.Sprintf("DROP FUNCTION IF EXISTS %s%s CASCADE", prefix, fn)) } for _, tbl := range []string{"idempotency", "stream", "state", "meta", "shard_lock", "schema_version"} { _, _ = broker.pool.Exec(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s%s CASCADE", prefix, tbl)) } } } t.Cleanup(func() { cleanup() _ = broker.Close(ctx) _ = node.Shutdown(ctx) schemaVersion = origSchemaVersion delete(schemaMigrations, origSchemaVersion+1) }) cleanup() // Install baseline at current schemaVersion. require.NoError(t, broker.EnsureSchema(ctx)) // Reset to one below current and add a templated migration. baseVersion := origSchemaVersion for _, prefix := range []string{customPrefix + "_map_", customPrefix + "_binary_map_"} { _, err = broker.pool.Exec(ctx, fmt.Sprintf( `UPDATE %sschema_version SET schema_version = $1 WHERE id = 1`, prefix), baseVersion) require.NoError(t, err) } // Migration uses __PREFIX__ placeholder — must resolve to the user's // custom prefix, NOT the default `cf_map_`. schemaMigrations[baseVersion+1] = ` ALTER TABLE __PREFIX__state ADD COLUMN IF NOT EXISTS template_probe_col TEXT; ` schemaVersion = baseVersion + 1 require.NoError(t, broker.EnsureSchema(ctx)) // Verify the column landed on BOTH custom-prefixed variant tables. for _, prefix := range []string{customPrefix + "_map_", customPrefix + "_binary_map_"} { var exists bool err = broker.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = $1 AND column_name = 'template_probe_col')`, prefix+"state", ).Scan(&exists) require.NoError(t, err) require.True(t, exists, "template_probe_col should exist on %sstate under custom TablePrefix", prefix) } // Verify the default cf_-prefixed tables were NOT touched (proves the // custom prefix really was used in the migration SQL). var defaultExists bool err = broker.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = $1 AND column_name = 'template_probe_col')`, "cf_map_state", ).Scan(&defaultExists) require.NoError(t, err) require.False(t, defaultExists, "default cf_map_ tables must NOT be touched when migration uses __PREFIX__ under a custom TablePrefix") // Final schema_version reflects success on both custom-prefixed variants. for _, prefix := range []string{customPrefix + "_map_", customPrefix + "_binary_map_"} { var v int err = broker.pool.QueryRow(ctx, fmt.Sprintf( `SELECT schema_version FROM %sschema_version WHERE id = 1`, prefix)).Scan(&v) require.NoError(t, err) require.Equal(t, baseVersion+1, v) } }