snapshot of https://github.com/livekit/livekit.git @ 08c5433449c106943231037d0b892895f6e92d3b (2026-09-15)

test/agent.go

@@ -0,0 +1,198 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"sync"
+
+	"github.com/gorilla/websocket"
+	"go.uber.org/atomic"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/livekit/protocol/livekit"
+)
+
+type agentClient struct {
+	mu   sync.Mutex
+	conn *websocket.Conn
+
+	registered              atomic.Int32
+	roomAvailability        atomic.Int32
+	roomJobs                atomic.Int32
+	publisherAvailability   atomic.Int32
+	publisherJobs           atomic.Int32
+	participantAvailability atomic.Int32
+	participantJobs         atomic.Int32
+
+	requestedJobs chan *livekit.Job
+
+	done chan struct{}
+}
+
+func newAgentClient(token string, port uint32) (*agentClient, error) {
+	host := fmt.Sprintf("ws://localhost:%d", port)
+	u, err := url.Parse(host + "/agent")
+	if err != nil {
+		return nil, err
+	}
+	requestHeader := make(http.Header)
+	requestHeader.Set("Authorization", "Bearer "+token)
+
+	connectUrl := u.String()
+	conn, _, err := websocket.DefaultDialer.Dial(connectUrl, requestHeader)
+	if err != nil {
+		return nil, err
+	}
+
+	return &agentClient{
+		conn:          conn,
+		requestedJobs: make(chan *livekit.Job, 100),
+		done:          make(chan struct{}),
+	}, nil
+}
+
+func (c *agentClient) Run(jobType livekit.JobType, namespace string) (err error) {
+	go c.read()
+
+	switch jobType {
+	case livekit.JobType_JT_ROOM:
+		err = c.write(&livekit.WorkerMessage{
+			Message: &livekit.WorkerMessage_Register{
+				Register: &livekit.RegisterWorkerRequest{
+					Type:      livekit.JobType_JT_ROOM,
+					Version:   "version",
+					Namespace: &namespace,
+				},
+			},
+		})
+
+	case livekit.JobType_JT_PUBLISHER:
+		err = c.write(&livekit.WorkerMessage{
+			Message: &livekit.WorkerMessage_Register{
+				Register: &livekit.RegisterWorkerRequest{
+					Type:      livekit.JobType_JT_PUBLISHER,
+					Version:   "version",
+					Namespace: &namespace,
+				},
+			},
+		})
+
+	case livekit.JobType_JT_PARTICIPANT:
+		err = c.write(&livekit.WorkerMessage{
+			Message: &livekit.WorkerMessage_Register{
+				Register: &livekit.RegisterWorkerRequest{
+					Type:      livekit.JobType_JT_PARTICIPANT,
+					Version:   "version",
+					Namespace: &namespace,
+				},
+			},
+		})
+	}
+
+	return err
+}
+
+func (c *agentClient) read() {
+	for {
+		select {
+		case <-c.done:
+			return
+		default:
+			_, b, err := c.conn.ReadMessage()
+			if err != nil {
+				return
+			}
+
+			msg := &livekit.ServerMessage{}
+			if err = proto.Unmarshal(b, msg); err != nil {
+				return
+			}
+
+			switch m := msg.Message.(type) {
+			case *livekit.ServerMessage_Assignment:
+				go c.handleAssignment(m.Assignment)
+			case *livekit.ServerMessage_Availability:
+				go c.handleAvailability(m.Availability)
+			case *livekit.ServerMessage_Register:
+				go c.handleRegister(m.Register)
+			}
+		}
+	}
+}
+
+func (c *agentClient) handleAssignment(req *livekit.JobAssignment) {
+	switch req.Job.Type {
+	case livekit.JobType_JT_ROOM:
+		c.roomJobs.Inc()
+	case livekit.JobType_JT_PUBLISHER:
+		c.publisherJobs.Inc()
+	case livekit.JobType_JT_PARTICIPANT:
+		c.participantJobs.Inc()
+	}
+}
+
+func (c *agentClient) handleAvailability(req *livekit.AvailabilityRequest) {
+	switch req.Job.Type {
+	case livekit.JobType_JT_ROOM:
+		c.roomAvailability.Inc()
+	case livekit.JobType_JT_PUBLISHER:
+		c.publisherAvailability.Inc()
+	case livekit.JobType_JT_PARTICIPANT:
+		c.participantAvailability.Inc()
+	}
+
+	c.requestedJobs <- req.Job
+
+	c.write(&livekit.WorkerMessage{
+		Message: &livekit.WorkerMessage_Availability{
+			Availability: &livekit.AvailabilityResponse{
+				JobId:     req.Job.Id,
+				Available: true,
+			},
+		},
+	})
+}
+
+func (c *agentClient) handleRegister(req *livekit.RegisterWorkerResponse) {
+	c.registered.Inc()
+}
+
+func (c *agentClient) write(msg *livekit.WorkerMessage) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	select {
+	case <-c.done:
+		return nil
+	default:
+		b, err := proto.Marshal(msg)
+		if err != nil {
+			return err
+		}
+
+		return c.conn.WriteMessage(websocket.BinaryMessage, b)
+	}
+}
+
+func (c *agentClient) close() {
+	c.mu.Lock()
+	close(c.done)
+	_ = c.conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
+	_ = c.conn.Close()
+	c.mu.Unlock()
+}

test/agent_test.go

@@ -0,0 +1,244 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/livekit-server/pkg/testutils"
+	"github.com/livekit/protocol/auth"
+	"github.com/livekit/protocol/livekit"
+)
+
+var (
+	RegisterTimeout  = 2 * time.Second
+	AssignJobTimeout = 3 * time.Second
+)
+
+func TestAgents(t *testing.T) {
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			_, finish := setupSingleNodeTest("TestAgents")
+			defer finish()
+
+			ac1, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			ac2, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			ac3, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			ac4, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			ac5, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			ac6, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			defer ac1.close()
+			defer ac2.close()
+			defer ac3.close()
+			defer ac4.close()
+			defer ac5.close()
+			defer ac6.close()
+			ac1.Run(livekit.JobType_JT_ROOM, "default")
+			ac2.Run(livekit.JobType_JT_ROOM, "default")
+			ac3.Run(livekit.JobType_JT_PUBLISHER, "default")
+			ac4.Run(livekit.JobType_JT_PUBLISHER, "default")
+			ac5.Run(livekit.JobType_JT_PARTICIPANT, "default")
+			ac6.Run(livekit.JobType_JT_PARTICIPANT, "default")
+
+			testutils.WithTimeout(t, func() string {
+				if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 || ac3.registered.Load() != 1 || ac4.registered.Load() != 1 || ac5.registered.Load() != 1 || ac6.registered.Load() != 1 {
+					return "worker not registered"
+				}
+
+				return ""
+			}, RegisterTimeout)
+
+			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
+			c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1, c2)
+
+			// publish 2 tracks
+			t1, err := c1.AddStaticTrack("audio/opus", "audio", "micro")
+			require.NoError(t, err)
+			defer t1.Stop()
+			t2, err := c1.AddStaticTrack("video/vp8", "video", "webcam")
+			require.NoError(t, err)
+			defer t2.Stop()
+
+			testutils.WithTimeout(t, func() string {
+				if ac1.roomJobs.Load()+ac2.roomJobs.Load() != 1 {
+					return "room job not assigned"
+				}
+
+				if ac3.publisherJobs.Load()+ac4.publisherJobs.Load() != 1 {
+					return fmt.Sprintf("publisher jobs not assigned, ac3: %d, ac4: %d", ac3.publisherJobs.Load(), ac4.publisherJobs.Load())
+				}
+
+				if ac5.participantJobs.Load()+ac6.participantJobs.Load() != 2 {
+					return fmt.Sprintf("participant jobs not assigned, ac5: %d, ac6: %d", ac5.participantJobs.Load(), ac6.participantJobs.Load())
+				}
+
+				return ""
+			}, 6*time.Second)
+
+			// publish 2 tracks
+			t3, err := c2.AddStaticTrack("audio/opus", "audio", "micro")
+			require.NoError(t, err)
+			defer t3.Stop()
+			t4, err := c2.AddStaticTrack("video/vp8", "video", "webcam")
+			require.NoError(t, err)
+			defer t4.Stop()
+
+			testutils.WithTimeout(t, func() string {
+				if ac1.roomJobs.Load()+ac2.roomJobs.Load() != 1 {
+					return "room job must be assigned 1 time"
+				}
+
+				if ac3.publisherJobs.Load()+ac4.publisherJobs.Load() != 2 {
+					return "2 publisher jobs must assigned"
+				}
+
+				if ac5.participantJobs.Load()+ac6.participantJobs.Load() != 2 {
+					return "2 participant jobs must assigned"
+				}
+
+				return ""
+			}, AssignJobTimeout)
+		})
+	}
+}
+
+func TestAgentNamespaces(t *testing.T) {
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			_, finish := setupSingleNodeTest("TestAgentNamespaces")
+			defer finish()
+
+			ac1, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			ac2, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			defer ac1.close()
+			defer ac2.close()
+			ac1.Run(livekit.JobType_JT_ROOM, "namespace1")
+			ac2.Run(livekit.JobType_JT_ROOM, "namespace2")
+
+			_, err = roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{
+				Name: testRoom,
+				Agents: []*livekit.RoomAgentDispatch{
+					{},
+					{
+						AgentName: "ag",
+					},
+				},
+			})
+			require.NoError(t, err)
+
+			testutils.WithTimeout(t, func() string {
+				if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 {
+					return "worker not registered"
+				}
+				return ""
+			}, RegisterTimeout)
+
+			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1)
+
+			testutils.WithTimeout(t, func() string {
+				if ac1.roomJobs.Load() != 1 || ac2.roomJobs.Load() != 1 {
+					return "room job not assigned"
+				}
+
+				job1 := <-ac1.requestedJobs
+				job2 := <-ac2.requestedJobs
+
+				if job1.Namespace != "namespace1" {
+					return "namespace is not 'namespace'"
+				}
+
+				if job2.Namespace != "namespace2" {
+					return "namespace is not 'namespace2'"
+				}
+
+				if job1.Id == job2.Id {
+					return "job ids are the same"
+				}
+
+				return ""
+			}, AssignJobTimeout)
+		})
+	}
+}
+
+func TestAgentMultiNode(t *testing.T) {
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			_, _, finish := setupMultiNodeTest("TestAgentMultiNode")
+			defer finish()
+
+			ac1, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			ac2, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			defer ac1.close()
+			defer ac2.close()
+			ac1.Run(livekit.JobType_JT_ROOM, "default")
+			ac2.Run(livekit.JobType_JT_PUBLISHER, "default")
+
+			testutils.WithTimeout(t, func() string {
+				if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 {
+					return "worker not registered"
+				}
+				return ""
+			}, RegisterTimeout)
+
+			c1 := createRTCClient("c1", secondServerPort, testRTCServicePath, nil) // Create a room on the second node
+			waitUntilConnected(t, c1)
+
+			t1, err := c1.AddStaticTrack("audio/opus", "audio", "micro")
+			require.NoError(t, err)
+			defer t1.Stop()
+
+			time.Sleep(time.Second * 10)
+
+			testutils.WithTimeout(t, func() string {
+				if ac1.roomJobs.Load() != 1 {
+					return "room job not assigned"
+				}
+
+				if ac2.publisherJobs.Load() != 1 {
+					return "participant job not assigned"
+				}
+
+				return ""
+			}, AssignJobTimeout)
+		})
+	}
+}
+
+func agentToken() string {
+	at := auth.NewAccessToken(testApiKey, testApiSecret).
+		AddGrant(&auth.VideoGrant{Agent: true})
+	t, err := at.ToJWT()
+	if err != nil {
+		panic(err)
+	}
+	return t
+}

test/client/datachannel_reader.go

@@ -0,0 +1,31 @@
+package client
+
+import (
+	"time"
+
+	"github.com/livekit/livekit-server/pkg/sfu/datachannel"
+)
+
+type DataChannelReader struct {
+	bitrate *datachannel.BitrateCalculator
+	target  int
+}
+
+func NewDataChannelReader(bitrate int) *DataChannelReader {
+	return &DataChannelReader{
+		target:  bitrate,
+		bitrate: datachannel.NewBitrateCalculator(datachannel.BitrateDuration*5, datachannel.BitrateWindow, datachannel.BitrateModeWallClock),
+	}
+}
+
+func (d *DataChannelReader) Read(p []byte, sid string) {
+	for {
+		if bitrate, ok := d.bitrate.ForceBitrate(time.Now()); ok && bitrate > 0 && bitrate > d.target {
+			time.Sleep(10 * time.Millisecond)
+			d.bitrate.AddBytes(0, 0, time.Now())
+			continue
+		}
+		break
+	}
+	d.bitrate.AddBytes(len(p), 0, time.Now())
+}

test/client/datatrack_remote.go

@@ -0,0 +1,84 @@
+package client
+
+import (
+	"github.com/frostbyte73/core"
+	"github.com/livekit/protocol/datatrack"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	"go.uber.org/atomic"
+)
+
+type DataTrackRemote struct {
+	publisherIdentity  livekit.ParticipantIdentity
+	publisherID        livekit.ParticipantID
+	handle             uint16
+	trackID            livekit.TrackID
+	logger             logger.Logger
+	numReceivedPackets atomic.Uint32
+
+	closed core.Fuse
+}
+
+func NewDataTrackRemote(
+	publisherIdentity livekit.ParticipantIdentity,
+	publisherID livekit.ParticipantID,
+	handle uint16,
+	trackID livekit.TrackID,
+	logger logger.Logger,
+) *DataTrackRemote {
+	logger.Infow(
+		"creating data track remote",
+		"publisherIdentity", publisherIdentity,
+		"publisherID", publisherID,
+		"handle", handle,
+		"trackID", trackID,
+	)
+	return &DataTrackRemote{
+		publisherIdentity: publisherIdentity,
+		publisherID:       publisherID,
+		handle:            handle,
+		trackID:           trackID,
+		logger:            logger,
+	}
+}
+
+func (d *DataTrackRemote) Close() {
+	d.logger.Infow(
+		"closing data track remote",
+		"publisherIdentity", d.publisherIdentity,
+		"publisherID", d.publisherID,
+		"handle", d.handle,
+		"trackID", d.trackID,
+	)
+	d.closed.Break()
+}
+
+func (d *DataTrackRemote) Handle() uint16 {
+	return d.handle
+}
+
+func (d *DataTrackRemote) ID() livekit.TrackID {
+	return d.trackID
+}
+
+func (d *DataTrackRemote) PacketReceived(packet *datatrack.Packet) {
+	if d.closed.IsBroken() {
+		return
+	}
+
+	valid := len(packet.Payload) != 0
+
+	for i := range packet.Payload {
+		if packet.Payload[i] != byte(255-i) {
+			valid = false
+			break
+		}
+	}
+	if valid {
+		d.numReceivedPackets.Inc()
+	}
+}
+
+func (d *DataTrackRemote) NumReceivedPackets() uint32 {
+	return d.numReceivedPackets.Load()
+}

test/client/datatrack_writer.go

@@ -0,0 +1,80 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+	"context"
+	"math/rand"
+	"time"
+
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/protocol/datatrack"
+	"github.com/livekit/protocol/datatrack/datatracktest"
+	"github.com/livekit/protocol/logger"
+)
+
+type dataTrackWriter struct {
+	ctx       context.Context
+	cancel    context.CancelFunc
+	handle    uint16
+	transport types.DataTrackTransport
+}
+
+func NewDataTrackWriter(ctx context.Context, handle uint16, transport types.DataTrackTransport) TrackWriter {
+	ctx, cancel := context.WithCancel(ctx)
+	return &dataTrackWriter{
+		ctx:       ctx,
+		cancel:    cancel,
+		handle:    handle,
+		transport: transport,
+	}
+}
+
+func (d *dataTrackWriter) Start() error {
+	go d.writeFrames()
+	return nil
+}
+
+func (d *dataTrackWriter) Stop() {
+	d.cancel()
+}
+
+func (d *dataTrackWriter) writeFrames() {
+	seqNum := uint16(0)
+	frameNum := uint16(0)
+	for {
+		select {
+		case <-d.ctx.Done():
+			return
+
+		default:
+			packets := datatracktest.GenerateRawDataPackets(d.handle, seqNum, frameNum, 1, rand.Intn(2048)+1, 100*time.Millisecond)
+			for _, packet := range packets {
+				if err := d.transport.SendDataTrackMessage(packet); err != nil {
+					logger.Errorw("could not send data track packet", err)
+				}
+			}
+
+			if len(packets) != 0 {
+				var lastPacket datatrack.Packet
+				if err := lastPacket.Unmarshal(packets[len(packets)-1]); err == nil {
+					seqNum = lastPacket.SequenceNumber + 1
+					frameNum = lastPacket.FrameNumber + 1
+				}
+			}
+			time.Sleep(100 * time.Millisecond)
+		}
+	}
+}

test/client/trackwriter.go

@@ -0,0 +1,191 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+	"context"
+	"io"
+	"os"
+	"time"
+
+	"github.com/pion/webrtc/v4"
+	"github.com/pion/webrtc/v4/pkg/media"
+	"github.com/pion/webrtc/v4/pkg/media/h264reader"
+	"github.com/pion/webrtc/v4/pkg/media/ivfreader"
+	"github.com/pion/webrtc/v4/pkg/media/oggreader"
+
+	"github.com/livekit/protocol/codecs/mime"
+	"github.com/livekit/protocol/logger"
+)
+
+type TrackWriter interface {
+	Start() error
+	Stop()
+}
+
+// Writes a file to an RTP track.
+// makes it easier to debug and create RTP streams
+type trackWriter struct {
+	ctx      context.Context
+	cancel   context.CancelFunc
+	track    *webrtc.TrackLocalStaticSample
+	filePath string
+	mime     mime.MimeType
+
+	ogg       *oggreader.OggReader
+	ivfheader *ivfreader.IVFFileHeader
+	ivf       *ivfreader.IVFReader
+	h264      *h264reader.H264Reader
+}
+
+func NewTrackWriter(ctx context.Context, track *webrtc.TrackLocalStaticSample, filePath string) TrackWriter {
+	ctx, cancel := context.WithCancel(ctx)
+	return &trackWriter{
+		ctx:      ctx,
+		cancel:   cancel,
+		track:    track,
+		filePath: filePath,
+		mime:     mime.NormalizeMimeType(track.Codec().MimeType),
+	}
+}
+
+func (w *trackWriter) Start() error {
+	if w.filePath == "" {
+		go w.writeNull()
+		return nil
+	}
+
+	file, err := os.Open(w.filePath)
+	if err != nil {
+		return err
+	}
+
+	logger.Debugw(
+		"starting track writer",
+		"trackID", w.track.ID(),
+		"mime", w.mime,
+	)
+	switch w.mime {
+	case mime.MimeTypeOpus:
+		w.ogg, _, err = oggreader.NewWith(file)
+		if err != nil {
+			return err
+		}
+		go w.writeOgg()
+	case mime.MimeTypeVP8:
+		w.ivf, w.ivfheader, err = ivfreader.NewWith(file)
+		if err != nil {
+			return err
+		}
+		go w.writeVP8()
+	case mime.MimeTypeH264:
+		w.h264, err = h264reader.NewReader(file)
+		if err != nil {
+			return err
+		}
+		go w.writeH264()
+	}
+	return nil
+}
+
+func (w *trackWriter) Stop() {
+	w.cancel()
+}
+
+func (w *trackWriter) writeNull() {
+	defer w.onWriteComplete()
+	sample := media.Sample{Data: []byte{0x0, 0xff, 0xff, 0xff, 0xff}, Duration: 30 * time.Millisecond}
+	h264Sample := media.Sample{Data: []byte{0x00, 0x00, 0x00, 0x01, 0x7, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x8, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x5, 0xff, 0xff, 0xff, 0xff}, Duration: 30 * time.Millisecond}
+	for {
+		select {
+		case <-time.After(20 * time.Millisecond):
+			if w.mime == mime.MimeTypeH264 {
+				w.track.WriteSample(h264Sample)
+			} else {
+				w.track.WriteSample(sample)
+			}
+		case <-w.ctx.Done():
+			return
+		}
+	}
+}
+
+func (w *trackWriter) writeOgg() {
+	// Keep track of last granule, the difference is the amount of samples in the buffer
+	var lastGranule uint64
+	for {
+		if w.ctx.Err() != nil {
+			return
+		}
+		pageData, pageHeader, err := w.ogg.ParseNextPage()
+		if err == io.EOF {
+			logger.Debugw("all audio samples parsed and sent")
+			w.onWriteComplete()
+			return
+		}
+
+		if err != nil {
+			logger.Errorw("could not parse ogg page", err)
+			return
+		}
+
+		// The amount of samples is the difference between the last and current timestamp
+		sampleCount := float64(pageHeader.GranulePosition - lastGranule)
+		lastGranule = pageHeader.GranulePosition
+		sampleDuration := time.Duration((sampleCount/48000)*1000) * time.Millisecond
+
+		if err = w.track.WriteSample(media.Sample{Data: pageData, Duration: sampleDuration}); err != nil {
+			logger.Errorw("could not write sample", err)
+			return
+		}
+
+		time.Sleep(sampleDuration)
+	}
+}
+
+func (w *trackWriter) writeVP8() {
+	// Send our video file frame at a time. Pace our sending such that we send it at the same speed it should be played back as.
+	// This isn't required since the video is timestamped, but we will such much higher loss if we send all at once.
+	sleepTime := time.Millisecond * time.Duration((float32(w.ivfheader.TimebaseNumerator)/float32(w.ivfheader.TimebaseDenominator))*1000)
+	for {
+		if w.ctx.Err() != nil {
+			return
+		}
+		frame, _, err := w.ivf.ParseNextFrame()
+		if err == io.EOF {
+			logger.Debugw("all video frames parsed and sent")
+			w.onWriteComplete()
+			return
+		}
+
+		if err != nil {
+			logger.Errorw("could not parse VP8 frame", err)
+			return
+		}
+
+		time.Sleep(sleepTime)
+		if err = w.track.WriteSample(media.Sample{Data: frame, Duration: time.Second}); err != nil {
+			logger.Errorw("could not write sample", err)
+			return
+		}
+	}
+}
+
+func (w *trackWriter) writeH264() {
+	// TODO: this is harder
+}
+
+func (w *trackWriter) onWriteComplete() {
+}

test/integration_helpers.go

@@ -0,0 +1,458 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/pion/transport/v4/vnet"
+	"github.com/redis/go-redis/v9"
+	"github.com/stretchr/testify/require"
+	"github.com/twitchtv/twirp"
+
+	"github.com/livekit/mediatransportutil/pkg/rtcconfig"
+	"github.com/livekit/protocol/auth"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils/guid"
+
+	"github.com/livekit/livekit-server/pkg/config"
+	"github.com/livekit/livekit-server/pkg/routing"
+	"github.com/livekit/livekit-server/pkg/rtc"
+	"github.com/livekit/livekit-server/pkg/service"
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
+	"github.com/livekit/livekit-server/pkg/testutils"
+	"github.com/livekit/livekit-server/pkg/testutils/vnettest"
+	testclient "github.com/livekit/livekit-server/test/client"
+)
+
+const (
+	testApiKey        = "apikey"
+	testApiSecret     = "apiSecretExtendTo32BytesAsThatIsMinimum"
+	testRoom          = "mytestroom"
+	defaultServerPort = 7880
+	secondServerPort  = 8880
+	nodeID1           = "node-1"
+	nodeID2           = "node-2"
+
+	syncDelay = 100 * time.Millisecond
+	// if there are deadlocks, it's helpful to set a short test timeout (i.e. go test -timeout=30s)
+	// let connection timeout happen
+	// connectTimeout = 5000 * time.Second
+)
+
+var roomClient livekit.RoomService
+
+func init() {
+	config.InitLoggerFromConfig(&config.DefaultConfig.Logging)
+
+	prometheus.Init("test", livekit.NodeType_SERVER)
+}
+
+func setupSingleNodeTest(name string) (*service.LivekitServer, func()) {
+	return setupSingleNodeTestWithConfig(name, nil)
+}
+
+func setupSingleNodeTestWithConfig(name string, configUpdater func(*config.Config)) (*service.LivekitServer, func()) {
+	logger.Infow("----------------STARTING TEST----------------", "test", name)
+	s := createSingleNodeServer(configUpdater)
+	go func() {
+		if err := s.Start(); err != nil {
+			logger.Errorw("server returned error", err)
+		}
+	}()
+
+	waitForServerToStart(s)
+
+	return s, func() {
+		s.Stop(true)
+		logger.Infow("----------------FINISHING TEST----------------", "test", name)
+	}
+}
+
+func setupMultiNodeTest(name string) (*service.LivekitServer, *service.LivekitServer, func()) {
+	return setupMultiNodeTestWithConfig(name, nil)
+}
+
+func setupMultiNodeTestWithConfig(name string, configUpdater func(*config.Config)) (*service.LivekitServer, *service.LivekitServer, func()) {
+	logger.Infow("----------------STARTING TEST----------------", "test", name)
+	s1 := createMultiNodeServer(guid.New(nodeID1), defaultServerPort, configUpdater)
+	s2 := createMultiNodeServer(guid.New(nodeID2), secondServerPort, configUpdater)
+	go s1.Start()
+	go s2.Start()
+
+	waitForServerToStart(s1)
+	waitForServerToStart(s2)
+
+	return s1, s2, func() {
+		s1.Stop(true)
+		s2.Stop(true)
+		redisClient().FlushAll(context.Background())
+		logger.Infow("----------------FINISHING TEST----------------", "test", name)
+	}
+}
+
+func contextWithToken(token string) context.Context {
+	header := make(http.Header)
+	testclient.SetAuthorizationToken(header, token)
+	tctx, err := twirp.WithHTTPRequestHeaders(context.Background(), header)
+	if err != nil {
+		panic(err)
+	}
+	return tctx
+}
+
+func waitForServerToStart(s *service.LivekitServer) {
+	// wait till ready
+	ctx, cancel := context.WithTimeout(context.Background(), testutils.ConnectTimeout)
+	defer cancel()
+	for {
+		select {
+		case <-ctx.Done():
+			panic("could not start server after timeout")
+		case <-time.After(10 * time.Millisecond):
+			if s.IsRunning() {
+				// ensure we can connect to it
+				res, err := http.Get(fmt.Sprintf("http://localhost:%d", s.HTTPPort()))
+				if err == nil && res.StatusCode == http.StatusOK {
+					return
+				}
+			}
+		}
+	}
+}
+
+func waitUntilConnected(t *testing.T, clients ...*testclient.RTCClient) {
+	logger.Infow("waiting for clients to become connected")
+	wg := sync.WaitGroup{}
+	for i := range clients {
+		c := clients[i]
+		wg.Go(func() {
+			err := c.WaitUntilConnected(5 * time.Second)
+			if err != nil {
+				t.Error(err)
+			}
+		})
+	}
+	wg.Wait()
+	if t.Failed() {
+		t.FailNow()
+	}
+}
+
+func ensureNotConnected(t *testing.T, clients ...*testclient.RTCClient) {
+	logger.Infow("checking if clients connect")
+	wg := sync.WaitGroup{}
+	for i := range clients {
+		c := clients[i]
+		wg.Go(func() {
+			err := c.WaitUntilConnected(5 * time.Second)
+			if err == nil {
+				t.Error(fmt.Errorf("expected client to not connect: %s", c.ID()))
+			}
+		})
+	}
+	wg.Wait()
+	if t.Failed() {
+		t.FailNow()
+	}
+}
+
+func createSingleNodeServer(configUpdater func(*config.Config)) *service.LivekitServer {
+	var err error
+	conf, err := config.NewConfig("", true, nil, nil)
+	if err != nil {
+		panic(fmt.Sprintf("could not create config: %v", err))
+	}
+	conf.Keys = map[string]string{testApiKey: testApiSecret}
+	conf.EnableDataTracks = true
+	if configUpdater != nil {
+		configUpdater(conf)
+	}
+
+	currentNode, err := routing.NewLocalNode(conf)
+	if err != nil {
+		panic(fmt.Sprintf("could not create local node: %v", err))
+	}
+	currentNode.SetNodeID(livekit.NodeID(guid.New(nodeID1)))
+
+	s, err := service.InitializeServer(conf, currentNode)
+	if err != nil {
+		panic(fmt.Sprintf("could not create server: %v", err))
+	}
+
+	roomClient = livekit.NewRoomServiceJSONClient(fmt.Sprintf("http://localhost:%d", defaultServerPort), &http.Client{})
+	return s
+}
+
+func createMultiNodeServer(nodeID string, port uint32, configUpdater func(*config.Config)) *service.LivekitServer {
+	var err error
+	conf, err := config.NewConfig("", true, nil, nil)
+	if err != nil {
+		panic(fmt.Sprintf("could not create config: %v", err))
+	}
+	conf.Port = port
+	conf.RTC.UDPPort = rtcconfig.PortRange{Start: int(port) + 1}
+	conf.RTC.TCPPort = port + 2
+	conf.Redis.Address = "localhost:6379"
+	conf.Keys = map[string]string{testApiKey: testApiSecret}
+	conf.EnableDataTracks = true
+	if configUpdater != nil {
+		configUpdater(conf)
+	}
+
+	currentNode, err := routing.NewLocalNode(conf)
+	if err != nil {
+		panic(err)
+	}
+	currentNode.SetNodeID(livekit.NodeID(nodeID))
+
+	// redis routing and store
+	s, err := service.InitializeServer(conf, currentNode)
+	if err != nil {
+		panic(fmt.Sprintf("could not create server: %v", err))
+	}
+
+	roomClient = livekit.NewRoomServiceJSONClient(fmt.Sprintf("http://localhost:%d", port), &http.Client{})
+	return s
+}
+
+type testRTCServicePath int
+
+const (
+	testRTCServicePathv0 testRTCServicePath = iota
+	testRTCServicePathv0SinglePeerConnection
+	testRTCServicePathv1
+)
+
+func (t testRTCServicePath) String() string {
+	switch t {
+	case testRTCServicePathv0:
+		return "v0"
+	case testRTCServicePathv0SinglePeerConnection:
+		return "v0-single-peer-connection"
+	case testRTCServicePathv1:
+		return "v1"
+	default:
+		return fmt.Sprintf("unknown: %d", t)
+	}
+}
+
+var testRTCServicePaths = []testRTCServicePath{
+	testRTCServicePathv0,
+	testRTCServicePathv0SinglePeerConnection,
+	testRTCServicePathv1,
+}
+
+func testRTCServicePathToTestClientOptions(testRTCServicePath testRTCServicePath, opts *testclient.Options) {
+	if opts == nil {
+		return
+	}
+
+	switch testRTCServicePath {
+	case testRTCServicePathv0:
+		opts.RTCServicePath = "/rtc"
+	case testRTCServicePathv0SinglePeerConnection:
+		opts.RTCServicePath = "/rtc"
+		opts.UseJoinRequestQueryParam = true
+	case testRTCServicePathv1:
+		opts.RTCServicePath = "/rtc/v1"
+		opts.UseJoinRequestQueryParam = true
+	default:
+		opts.RTCServicePath = "/rtc"
+	}
+}
+
+// creates a client and runs against server
+func createRTCClient(name string, port int, testRTCServicePath testRTCServicePath, opts *testclient.Options) *testclient.RTCClient {
+	var customizer func(token *auth.AccessToken, grants *auth.VideoGrant)
+	if opts != nil {
+		customizer = opts.TokenCustomizer
+	}
+	token := joinToken(testRoom, name, customizer)
+
+	return createRTCClientWithToken(token, port, testRTCServicePath, opts)
+}
+
+// creates a client and runs against server
+func createRTCClientWithToken(token string, port int, testRTCServicePath testRTCServicePath, opts *testclient.Options) *testclient.RTCClient {
+	if opts == nil {
+		opts = &testclient.Options{
+			AutoSubscribe: true,
+		}
+	}
+	testRTCServicePathToTestClientOptions(testRTCServicePath, opts)
+	ws, err := testclient.NewWebSocketConn(fmt.Sprintf("ws://localhost:%d", port), token, opts)
+	if err != nil {
+		panic(err)
+	}
+
+	c, err := testclient.NewRTCClient(ws, opts.UseJoinRequestQueryParam, opts)
+	if err != nil {
+		panic(err)
+	}
+
+	go c.Run()
+
+	return c
+}
+
+func redisClient() *redis.Client {
+	return redis.NewClient(&redis.Options{
+		Addr: "localhost:6379",
+	})
+}
+
+func joinToken(room, name string, customFn func(token *auth.AccessToken, grants *auth.VideoGrant)) string {
+	at := auth.NewAccessToken(testApiKey, testApiSecret).
+		SetIdentity(name).
+		SetName(name).
+		SetMetadata("metadata" + name)
+	grant := &auth.VideoGrant{RoomJoin: true, Room: room}
+	if customFn != nil {
+		customFn(at, grant)
+	}
+	at.AddGrant(grant)
+	t, err := at.ToJWT()
+	if err != nil {
+		panic(err)
+	}
+	return t
+}
+
+func joinTokenWithGrant(name string, grant *auth.VideoGrant) string {
+	at := auth.NewAccessToken(testApiKey, testApiSecret).
+		AddGrant(grant).
+		SetIdentity(name).
+		SetName(name)
+	t, err := at.ToJWT()
+	if err != nil {
+		panic(err)
+	}
+	return t
+}
+
+func createRoomToken() string {
+	at := auth.NewAccessToken(testApiKey, testApiSecret).
+		AddGrant(&auth.VideoGrant{RoomCreate: true})
+	t, err := at.ToJWT()
+	if err != nil {
+		panic(err)
+	}
+	return t
+}
+
+func adminRoomToken(name string) string {
+	at := auth.NewAccessToken(testApiKey, testApiSecret).
+		AddGrant(&auth.VideoGrant{RoomAdmin: true, Room: name})
+	t, err := at.ToJWT()
+	if err != nil {
+		panic(err)
+	}
+	return t
+}
+
+func listRoomToken() string {
+	at := auth.NewAccessToken(testApiKey, testApiSecret).
+		AddGrant(&auth.VideoGrant{RoomList: true})
+	t, err := at.ToJWT()
+	if err != nil {
+		panic(err)
+	}
+	return t
+}
+
+func stopWriters(writers ...testclient.TrackWriter) {
+	for _, w := range writers {
+		w.Stop()
+	}
+}
+
+func stopClients(clients ...*testclient.RTCClient) {
+	for _, c := range clients {
+		c.Stop()
+	}
+}
+
+// -----------------------------------------------------------------------------
+// vnet media harness
+//
+// Setup specific to driving a real server transport over a virtual network. The
+// pion side lives in pkg/testutils/vnettest, shared with the pkg/sfu media tests.
+// -----------------------------------------------------------------------------
+
+// newVNetWebRTCConfig builds the server side WebRTCConfig on net. The direction
+// configs come from the production NewWebRTCConfig so the negotiated extensions and
+// feedback stay in step with it; the setting engine is replaced so no real socket or
+// ICE mux is bound.
+func newVNetWebRTCConfig(t *testing.T, net *vnet.Net, bufferFactory *buffer.Factory) *rtc.WebRTCConfig {
+	t.Helper()
+
+	conf, err := config.NewConfig("", true, nil, nil)
+	require.NoError(t, err)
+
+	// an ephemeral port range instead of the dev mode single port, which would bind a mux
+	conf.RTC.TCPPort = 0
+	conf.RTC.UDPPort = rtcconfig.PortRange{}
+	conf.RTC.ICEPortRangeStart = 50000
+	conf.RTC.ICEPortRangeEnd = 60000
+
+	rtcConf, err := rtc.NewWebRTCConfig(conf)
+	require.NoError(t, err)
+	require.Nil(t, rtcConf.UDPMux, "test config must not bind a udp mux")
+
+	rtcConf.SettingEngine = vnettest.NewSettingEngine(net)
+	rtcConf.SetBufferFactory(bufferFactory)
+
+	return rtcConf
+}
+
+// stripDeclaredSSRCs removes the a=ssrc lines pion puts in its offer. Browsers doing
+// rid based simulcast do not declare per-layer SSRCs, which is why a repair SSRC has to
+// be learned at all; leaving them in would let the receiver resolve everything from SDP.
+func stripDeclaredSSRCs(offer string) string {
+	lines := strings.Split(offer, "\r\n")
+	filtered := lines[:0]
+	for _, line := range lines {
+		if strings.HasPrefix(line, "a=ssrc") {
+			continue
+		}
+		filtered = append(filtered, line)
+	}
+	return strings.Join(filtered, "\r\n")
+}
+
+// sendUntil calls send every 20ms until done reports true or the timeout expires,
+// returning the final state of done.
+func sendUntil(t *testing.T, timeout time.Duration, done func() bool, send func()) bool {
+	t.Helper()
+
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		if done() {
+			return true
+		}
+		send()
+		time.Sleep(20 * time.Millisecond)
+	}
+	return done()
+}

test/multinode_roomservice_test.go

@@ -0,0 +1,189 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/protocol/livekit"
+
+	"github.com/livekit/livekit-server/pkg/testutils"
+)
+
+func TestMultiNodeRoomList(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+		return
+	}
+	_, _, finish := setupMultiNodeTest("TestMultiNodeRoomList")
+	defer finish()
+
+	roomServiceListRoom(t)
+}
+
+// update room metadata when it's empty
+func TestMultiNodeUpdateRoomMetadata(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+		return
+	}
+
+	t.Run("when room is empty", func(t *testing.T) {
+		_, _, finish := setupMultiNodeTest("TestMultiNodeUpdateRoomMetadata_empty")
+		defer finish()
+
+		_, err := roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{
+			Name: "emptyRoom",
+		})
+		require.NoError(t, err)
+
+		rm, err := roomClient.UpdateRoomMetadata(contextWithToken(adminRoomToken("emptyRoom")), &livekit.UpdateRoomMetadataRequest{
+			Room:     "emptyRoom",
+			Metadata: "updated metadata",
+		})
+		require.NoError(t, err)
+		require.Equal(t, "updated metadata", rm.Metadata)
+	})
+
+	t.Run("when room has a participant", func(t *testing.T) {
+		for _, testRTCServicePath := range testRTCServicePaths {
+			t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+				_, _, finish := setupMultiNodeTest("TestMultiNodeUpdateRoomMetadata_with_participant")
+				defer finish()
+
+				c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
+				waitUntilConnected(t, c1)
+				defer c1.Stop()
+
+				_, err := roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{
+					Name: "emptyRoom",
+				})
+				require.NoError(t, err)
+
+				rm, err := roomClient.UpdateRoomMetadata(contextWithToken(adminRoomToken("emptyRoom")), &livekit.UpdateRoomMetadataRequest{
+					Room:     "emptyRoom",
+					Metadata: "updated metadata",
+				})
+				require.NoError(t, err)
+				require.Equal(t, "updated metadata", rm.Metadata)
+			})
+		}
+	})
+}
+
+// remove a participant
+func TestMultiNodeRemoveParticipant(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+		return
+	}
+
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			_, _, finish := setupMultiNodeTest("TestMultiNodeRemoveParticipant")
+			defer finish()
+
+			c1 := createRTCClient("mn_remove_participant", defaultServerPort, testRTCServicePath, nil)
+			defer c1.Stop()
+			waitUntilConnected(t, c1)
+
+			ctx := contextWithToken(adminRoomToken(testRoom))
+			_, err := roomClient.RemoveParticipant(ctx, &livekit.RoomParticipantIdentity{
+				Room:     testRoom,
+				Identity: "mn_remove_participant",
+			})
+			require.NoError(t, err)
+
+			// participant list doesn't show the participant
+			listRes, err := roomClient.ListParticipants(ctx, &livekit.ListParticipantsRequest{
+				Room: testRoom,
+			})
+			require.NoError(t, err)
+			require.Len(t, listRes.Participants, 0)
+		})
+	}
+}
+
+// update participant metadata
+func TestMultiNodeUpdateParticipantMetadata(t *testing.T) {
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			_, _, finish := setupMultiNodeTest("TestMultiNodeUpdateParticipantMetadata")
+			defer finish()
+
+			c1 := createRTCClient("update_participant_metadata", defaultServerPort, testRTCServicePath, nil)
+			defer c1.Stop()
+			waitUntilConnected(t, c1)
+
+			ctx := contextWithToken(adminRoomToken(testRoom))
+			res, err := roomClient.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{
+				Room:     testRoom,
+				Identity: "update_participant_metadata",
+				Metadata: "the new metadata",
+			})
+			require.NoError(t, err)
+			require.Equal(t, "the new metadata", res.Metadata)
+		})
+	}
+}
+
+// admin mute published track
+func TestMultiNodeMutePublishedTrack(t *testing.T) {
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			_, _, finish := setupMultiNodeTest("TestMultiNodeMutePublishedTrack")
+			defer finish()
+
+			identity := "mute_published_track"
+			c1 := createRTCClient(identity, defaultServerPort, testRTCServicePath, nil)
+			defer c1.Stop()
+			waitUntilConnected(t, c1)
+
+			writers := publishTracksForClients(t, c1)
+			defer stopWriters(writers...)
+
+			trackIDs := c1.GetPublishedTrackIDs()
+			require.NotEmpty(t, trackIDs)
+
+			ctx := contextWithToken(adminRoomToken(testRoom))
+			// wait for it to be published before
+			testutils.WithTimeout(t, func() string {
+				res, err := roomClient.GetParticipant(ctx, &livekit.RoomParticipantIdentity{
+					Room:     testRoom,
+					Identity: identity,
+				})
+				require.NoError(t, err)
+				if len(res.Tracks) == 2 {
+					return ""
+				} else {
+					return fmt.Sprintf("expected 2 tracks to be published, actual: %d", len(res.Tracks))
+				}
+			})
+
+			res, err := roomClient.MutePublishedTrack(ctx, &livekit.MuteRoomTrackRequest{
+				Room:     testRoom,
+				Identity: identity,
+				TrackSid: trackIDs[0],
+				Muted:    true,
+			})
+			require.NoError(t, err)
+			require.Equal(t, trackIDs[0], res.Track.Sid)
+			require.True(t, res.Track.Muted)
+		})
+	}
+}

test/multinode_test.go

@@ -0,0 +1,554 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/protocol/auth"
+	"github.com/livekit/protocol/livekit"
+
+	"github.com/livekit/livekit-server/pkg/config"
+	"github.com/livekit/livekit-server/pkg/rtc"
+	"github.com/livekit/livekit-server/pkg/testutils"
+	"github.com/livekit/livekit-server/test/client"
+)
+
+func TestMultiNodeRouting(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+		return
+	}
+
+	_, _, finish := setupMultiNodeTest("TestMultiNodeRouting")
+	defer finish()
+
+	// creating room on node 1
+	_, err := roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{
+		Name: testRoom,
+	})
+	require.NoError(t, err)
+
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			// one node connecting to node 1, and another connecting to node 2
+			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
+			c2 := createRTCClient("c2", secondServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1, c2)
+			defer stopClients(c1, c2)
+
+			// c1 publishing, and c2 receiving
+			t1, err := c1.AddStaticTrack("audio/opus", "audio", "webcam")
+			require.NoError(t, err)
+			if t1 != nil {
+				defer t1.Stop()
+			}
+
+			testutils.WithTimeout(t, func() string {
+				if len(c2.SubscribedTracks()) == 0 {
+					return "c2 received no tracks"
+				}
+				if len(c2.SubscribedTracks()[c1.ID()]) != 1 {
+					return "c2 didn't receive track published by c1"
+				}
+				tr1 := c2.SubscribedTracks()[c1.ID()][0]
+				streamID, _ := rtc.UnpackStreamID(tr1.StreamID())
+				require.Equal(t, c1.ID(), streamID)
+				return ""
+			})
+
+			remoteC1 := c2.GetRemoteParticipant(c1.ID())
+			require.Equal(t, "c1", remoteC1.Name)
+			require.Equal(t, "metadatac1", remoteC1.Metadata)
+		})
+	}
+}
+
+func TestConnectWithoutCreation(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+		return
+	}
+
+	_, _, finish := setupMultiNodeTest("TestConnectWithoutCreation")
+	defer finish()
+
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1)
+
+			c1.Stop()
+		})
+	}
+}
+
+func TestMultinodePublishingUponJoining(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+		return
+	}
+	_, _, finish := setupMultiNodeTest("TestMultinodePublishingUponJoining")
+	defer finish()
+
+	scenarioPublishingUponJoining(t)
+}
+
+func TestMultinodeReceiveBeforePublish(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+		return
+	}
+	_, _, finish := setupMultiNodeTest("TestMultinodeReceiveBeforePublish")
+	defer finish()
+
+	scenarioReceiveBeforePublish(t)
+}
+
+// reconnecting to the same room, after one of the servers has gone away
+func TestMultinodeReconnectAfterNodeShutdown(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+		return
+	}
+
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			_, s2, finish := setupMultiNodeTest("TestMultinodeReconnectAfterNodeShutdown")
+			defer finish()
+
+			// creating room on node 1
+			_, err := roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{
+				Name:   testRoom,
+				NodeId: s2.Node().Id,
+			})
+			require.NoError(t, err)
+
+			// one node connecting to node 1, and another connecting to node 2
+			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
+			c2 := createRTCClient("c2", secondServerPort, testRTCServicePath, nil)
+
+			waitUntilConnected(t, c1, c2)
+			stopClients(c1, c2)
+
+			// stop s2, and connect to room again
+			s2.Stop(true)
+
+			time.Sleep(syncDelay)
+
+			c3 := createRTCClient("c3", defaultServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c3)
+		})
+	}
+}
+
+func TestMultinodeDataPublishing(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+		return
+	}
+
+	_, _, finish := setupMultiNodeTest("TestMultinodeDataPublishing")
+	defer finish()
+
+	scenarioDataPublish(t)
+	scenarioDataUnlabeledPublish(t)
+	scenarioDataTracksPublishingUponJoining(t)
+}
+
+func TestMultiNodeJoinAfterClose(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+		return
+	}
+
+	_, _, finish := setupMultiNodeTest("TestMultiNodeJoinAfterClose")
+	defer finish()
+
+	scenarioJoinClosedRoom(t)
+}
+
+func TestMultiNodeCloseNonRTCRoom(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+		return
+	}
+
+	_, _, finish := setupMultiNodeTest("closeNonRTCRoom")
+	defer finish()
+
+	closeNonRTCRoom(t)
+}
+
+// ensure that token accurately reflects out of band updates
+func TestMultiNodeRefreshToken(t *testing.T) {
+	_, _, finish := setupMultiNodeTest("TestMultiNodeJoinAfterClose")
+	defer finish()
+
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			// a participant joining with full permissions
+			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1)
+
+			// update permissions and metadata
+			ctx := contextWithToken(adminRoomToken(testRoom))
+			_, err := roomClient.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{
+				Room:     testRoom,
+				Identity: "c1",
+				Permission: &livekit.ParticipantPermission{
+					CanPublish:   false,
+					CanSubscribe: true,
+				},
+				Metadata: "metadata",
+			})
+			require.NoError(t, err)
+
+			testutils.WithTimeout(t, func() string {
+				if c1.RefreshToken() == "" {
+					return "did not receive refresh token"
+				}
+				// parse token to ensure it's correct
+				verifier, err := auth.ParseAPIToken(c1.RefreshToken())
+				require.NoError(t, err)
+
+				_, grants, err := verifier.Verify(testApiSecret)
+				require.NoError(t, err)
+
+				if grants.Metadata != "metadata" {
+					return "metadata did not match"
+				}
+				if *grants.Video.CanPublish {
+					return "canPublish should be false"
+				}
+				if *grants.Video.CanPublishData {
+					return "canPublishData should be false"
+				}
+				if !*grants.Video.CanSubscribe {
+					return "canSubscribe should be true"
+				}
+				return ""
+			})
+		})
+	}
+}
+
+// ensure that token accurately reflects out of band updates
+func TestMultiNodeUpdateAttributes(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+		return
+	}
+
+	_, _, finish := setupMultiNodeTest("TestMultiNodeUpdateAttributes")
+	defer finish()
+
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("au1", defaultServerPort, testRTCServicePath, &client.Options{
+				TokenCustomizer: func(token *auth.AccessToken, grants *auth.VideoGrant) {
+					token.SetAttributes(map[string]string{
+						"mykey": "au1",
+					})
+				},
+			})
+			c2 := createRTCClient("au2", secondServerPort, testRTCServicePath, &client.Options{
+				TokenCustomizer: func(token *auth.AccessToken, grants *auth.VideoGrant) {
+					token.SetAttributes(map[string]string{
+						"mykey": "au2",
+					})
+					grants.SetCanUpdateOwnMetadata(true)
+				},
+			})
+			waitUntilConnected(t, c1, c2)
+
+			testutils.WithTimeout(t, func() string {
+				rc2 := c1.GetRemoteParticipant(c2.ID())
+				rc1 := c2.GetRemoteParticipant(c1.ID())
+				if rc2 == nil || rc1 == nil {
+					return "participants could not see each other"
+				}
+				if rc1.Attributes == nil || rc1.Attributes["mykey"] != "au1" {
+					return "rc1's initial attributes are incorrect"
+				}
+				if rc2.Attributes == nil || rc2.Attributes["mykey"] != "au2" {
+					return "rc2's initial attributes are incorrect"
+				}
+				return ""
+			})
+
+			// this one should not go through
+			_ = c1.SetAttributes(map[string]string{"mykey": "shouldnotchange"})
+			_ = c2.SetAttributes(map[string]string{"secondkey": "au2"})
+
+			// updates using room API should succeed
+			_, err := roomClient.UpdateParticipant(contextWithToken(adminRoomToken(testRoom)), &livekit.UpdateParticipantRequest{
+				Room:     testRoom,
+				Identity: "au1",
+				Attributes: map[string]string{
+					"secondkey": "au1",
+				},
+			})
+			require.NoError(t, err)
+
+			testutils.WithTimeout(t, func() string {
+				rc1 := c2.GetRemoteParticipant(c1.ID())
+				rc2 := c1.GetRemoteParticipant(c2.ID())
+				if rc1.Attributes["secondkey"] != "au1" {
+					return "au1's attribute update failed"
+				}
+				if rc2.Attributes["secondkey"] != "au2" {
+					return "au2's attribute update failed"
+				}
+				if rc1.Attributes["mykey"] != "au1" {
+					return "au1's mykey should not change"
+				}
+				if rc2.Attributes["mykey"] != "au2" {
+					return "au2's mykey should not change"
+				}
+				return ""
+			})
+		})
+	}
+}
+
+func TestMultiNodeRevokePublishPermission(t *testing.T) {
+	_, _, finish := setupMultiNodeTest("TestMultiNodeRevokePublishPermission")
+	defer finish()
+
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
+			c2 := createRTCClient("c2", secondServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1, c2)
+
+			// c1 publishes a track for c2
+			writers := publishTracksForClients(t, c1)
+			defer stopWriters(writers...)
+
+			testutils.WithTimeout(t, func() string {
+				if len(c2.SubscribedTracks()[c1.ID()]) != 2 {
+					return "c2 did not receive c1's tracks"
+				}
+				return ""
+			})
+
+			// revoke permission
+			ctx := contextWithToken(adminRoomToken(testRoom))
+			_, err := roomClient.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{
+				Room:     testRoom,
+				Identity: "c1",
+				Permission: &livekit.ParticipantPermission{
+					CanPublish:     false,
+					CanPublishData: true,
+					CanSubscribe:   true,
+				},
+			})
+			require.NoError(t, err)
+
+			// ensure c1 no longer has track published, c2 no longer see track under C1
+			testutils.WithTimeout(t, func() string {
+				if len(c1.GetPublishedTrackIDs()) != 0 {
+					return "c1 did not unpublish tracks"
+				}
+				remoteC1 := c2.GetRemoteParticipant(c1.ID())
+				if remoteC1 == nil {
+					return "c2 doesn't know about c1"
+				}
+				if len(remoteC1.Tracks) != 0 {
+					return "c2 still has c1's tracks"
+				}
+				return ""
+			})
+		})
+	}
+}
+
+func TestCloseDisconnectedParticipantOnSignalClose(t *testing.T) {
+	_, _, finish := setupMultiNodeTest("TestCloseDisconnectedParticipantOnSignalClose")
+	defer finish()
+
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("c1", secondServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1)
+
+			c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, &client.Options{
+				SignalRequestInterceptor: func(msg *livekit.SignalRequest, next client.SignalRequestHandler) error {
+					switch msg.Message.(type) {
+					case *livekit.SignalRequest_Offer, *livekit.SignalRequest_Answer, *livekit.SignalRequest_Leave:
+						return nil
+					default:
+						return next(msg)
+					}
+				},
+				SignalResponseInterceptor: func(msg *livekit.SignalResponse, next client.SignalResponseHandler) error {
+					switch msg.Message.(type) {
+					case *livekit.SignalResponse_Offer, *livekit.SignalResponse_Answer:
+						return nil
+					default:
+						return next(msg)
+					}
+				},
+			})
+
+			testutils.WithTimeout(t, func() string {
+				if len(c1.RemoteParticipants()) != 1 {
+					return "c1 did not see c2 join"
+				}
+				return ""
+			})
+
+			c2.Stop()
+
+			testutils.WithTimeout(t, func() string {
+				if len(c1.RemoteParticipants()) != 0 {
+					return "c1 did not see c2 removed"
+				}
+				return ""
+			})
+		})
+	}
+}
+
+func TestMultiNodeDataBlob(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+		return
+	}
+
+	_, _, finish := setupMultiNodeTestWithConfig("TestMultiNodeDataBlob", func(c *config.Config) {
+		c.EnableParticipantDataBlob = true
+		c.Limit.MaxDataBlobSize = 1024
+	})
+	defer finish()
+
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			pubCapture := &dataBlobCapture{}
+			subCapture := &dataBlobCapture{}
+
+			// publisher on node 1, subscriber on node 2
+			pub := createRTCClient("pub", defaultServerPort, testRTCServicePath, &client.Options{
+				AutoSubscribe:             true,
+				SignalResponseInterceptor: pubCapture.interceptor(),
+			})
+			sub := createRTCClient("sub", secondServerPort, testRTCServicePath, &client.Options{
+				AutoSubscribe:             true,
+				SignalResponseInterceptor: subCapture.interceptor(),
+			})
+			waitUntilConnected(t, pub, sub)
+			defer stopClients(pub, sub)
+
+			// wait for both nodes to see each other so the get request routes correctly
+			testutils.WithTimeout(t, func() string {
+				if sub.GetRemoteParticipant(pub.ID()) == nil {
+					return "sub does not see pub yet"
+				}
+				return ""
+			})
+
+			key := &livekit.DataBlobKey{
+				Key: &livekit.DataBlobKey_Generic{
+					Generic: "blob-multinode",
+				},
+			}
+			contents := []byte("multinode-content")
+
+			require.NoError(t, pub.SendRequest(&livekit.SignalRequest{
+				Message: &livekit.SignalRequest_StoreDataBlobRequest{
+					StoreDataBlobRequest: &livekit.StoreDataBlobRequest{
+						RequestId: 1,
+						Blob: &livekit.DataBlob{
+							Key:      key,
+							Contents: contents,
+						},
+					},
+				},
+			}))
+
+			testutils.WithTimeout(t, func() string {
+				resp := pubCapture.takeStoreResponse()
+				if resp == nil {
+					return "publisher did not receive store response"
+				}
+				if resp.RequestId != 1 {
+					return fmt.Sprintf("expected store response request id 1, got %d", resp.RequestId)
+				}
+				if resp.Key == nil {
+					return "store response missing key"
+				}
+				if resp.Key.String() != key.String() {
+					return fmt.Sprintf("expected stored blob key %s, got %s", key.String(), resp.Key.String())
+				}
+				return ""
+			})
+			require.Equal(t, 0, pubCapture.requestResponseCount(), "publisher should not receive an error response on success")
+
+			// subscriber on a different node asks for the blob; the request routes
+			// across nodes to the publisher.
+			require.NoError(t, sub.SendRequest(&livekit.SignalRequest{
+				Message: &livekit.SignalRequest_GetDataBlobRequest{
+					GetDataBlobRequest: &livekit.GetDataBlobRequest{
+						ParticipantIdentity: "pub",
+						Key:                 key,
+					},
+				},
+			}))
+
+			testutils.WithTimeout(t, func() string {
+				resp := subCapture.takeBlobResponse()
+				if resp == nil {
+					return "subscriber did not receive blob response"
+				}
+				if resp.Blob == nil {
+					return "blob response missing blob"
+				}
+				if resp.Blob.Key.String() != key.String() {
+					return fmt.Sprintf("expected data blob key %s, got %s", key.String(), resp.Blob.Key.String())
+				}
+				if string(resp.Blob.Contents) != string(contents) {
+					return fmt.Sprintf("expected contents %q, got %q", contents, resp.Blob.Contents)
+				}
+				return ""
+			})
+
+			// requesting an unknown publisher identity should return NOT_FOUND
+			require.NoError(t, sub.SendRequest(&livekit.SignalRequest{
+				Message: &livekit.SignalRequest_GetDataBlobRequest{
+					GetDataBlobRequest: &livekit.GetDataBlobRequest{
+						ParticipantIdentity: "unknown-publisher",
+						Key:                 key,
+					},
+				},
+			}))
+
+			testutils.WithTimeout(t, func() string {
+				rr := subCapture.takeRequestResponse()
+				if rr == nil {
+					return "subscriber did not receive RequestResponse for unknown publisher"
+				}
+				if rr.Reason != livekit.RequestResponse_NOT_FOUND {
+					return fmt.Sprintf("expected NOT_FOUND, got %s", rr.Reason)
+				}
+				return ""
+			})
+		})
+	}
+}

test/rtx_pairing_integration_test.go

@@ -0,0 +1,562 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+// RTX repair stream pairing on simulcast (rid) streams, over a real PCTransport driven
+// by a real pion publisher on a virtual network.
+//
+// RID based simulcast has no a=ssrc-group:FID line, so the pairing cannot come from
+// SDP. The repair SSRC of a layer comes either from the mid/rsid header extensions on
+// its packets, or - for a migrated publisher, which is mid-stream and no longer sends
+// those extensions - from the migration info in TransportParams.SimTracks.
+//
+// Neither source fails loudly when it breaks: the repair buffer just accumulates
+// packets that are never applied and NACK recovery for simulcast stops working. Both
+// paths are therefore asserted end to end, by retransmitting a sequence number that is
+// never sent on the primary stream and requiring it to surface on the primary buffer.
+
+import (
+	"encoding/binary"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/pion/rtp"
+	"github.com/pion/sdp/v3"
+	"github.com/pion/transport/v4/vnet"
+	"github.com/pion/webrtc/v4"
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/livekit-server/pkg/rtc"
+	"github.com/livekit/livekit-server/pkg/rtc/transport/transportfakes"
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	sfuinterceptor "github.com/livekit/livekit-server/pkg/sfu/interceptor"
+	"github.com/livekit/livekit-server/pkg/testutils/vnettest"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+)
+
+// the single video m-line of the publisher's offer
+const rtxTestMid = "0"
+
+const (
+	sendsExtensions = false
+	omitsExtensions = true
+)
+
+// SSRCs are chosen by the test rather than taken from the offer; see stripDeclaredSSRCs.
+var rtxTestLayers = []struct {
+	rid          string
+	ssrc         uint32
+	rtxSSRC      uint32
+	recoveredSeq uint16
+}{
+	{rid: "q", ssrc: 1001, rtxSSRC: 2001, recoveredSeq: 50001},
+	{rid: "h", ssrc: 1002, rtxSSRC: 2002, recoveredSeq: 50002},
+	{rid: "f", ssrc: 1003, rtxSSRC: 2003, recoveredSeq: 50003},
+}
+
+// TestSimulcastRTXPairing covers a publisher sending mid/rid/rsid: the pairing comes
+// from probing the packets.
+func TestSimulcastRTXPairing(t *testing.T) {
+	h := newRTXHarness(t, nil)
+
+	h.run(t, sendsExtensions)
+
+	// pairing is also reported through the callback mediatrack subscribes to
+	require.Equal(t, len(rtxTestLayers), h.tracker.pairCount(), "not all rtx pairs found: %s", h.tracker.describe())
+	for _, w := range h.writers {
+		base, repair := h.tracker.pair(w.rid)
+		require.Equal(t, w.ssrc, base, "wrong base ssrc paired for rid %q", w.rid)
+		require.Equal(t, w.rtxSSRC, repair, "wrong repair ssrc paired for rid %q", w.rid)
+	}
+}
+
+// TestSimulcastRTXPairingAfterMigration covers a migrated publisher: it is mid-stream
+// and sends no mid/rid/rsid, so UnhandleSimulcastInterceptor synthesises them for pion
+// and the pairing has to come from SimTracks. RepairSSRC names the repair stream.
+func TestSimulcastRTXPairingAfterMigration(t *testing.T) {
+	simTracks := make(map[uint32]sfuinterceptor.SimulcastTrackInfo, 2*len(rtxTestLayers))
+	for _, l := range rtxTestLayers {
+		simTracks[l.ssrc] = sfuinterceptor.SimulcastTrackInfo{
+			Mid:        rtxTestMid,
+			StreamID:   l.rid,
+			RepairSSRC: l.rtxSSRC,
+		}
+		simTracks[l.rtxSSRC] = sfuinterceptor.SimulcastTrackInfo{
+			Mid:            rtxTestMid,
+			StreamID:       l.rid,
+			IsRepairStream: true,
+		}
+	}
+
+	newRTXHarness(t, simTracks).run(t, omitsExtensions)
+}
+
+// TestSimulcastRTXPairingAfterMigrationWithoutRepairSSRC covers migration info that
+// marks the repair stream but leaves RepairSSRC unset on the primary entry.
+func TestSimulcastRTXPairingAfterMigrationWithoutRepairSSRC(t *testing.T) {
+	simTracks := make(map[uint32]sfuinterceptor.SimulcastTrackInfo, 2*len(rtxTestLayers))
+	for _, l := range rtxTestLayers {
+		simTracks[l.ssrc] = sfuinterceptor.SimulcastTrackInfo{Mid: rtxTestMid, StreamID: l.rid}
+		simTracks[l.rtxSSRC] = sfuinterceptor.SimulcastTrackInfo{
+			Mid:            rtxTestMid,
+			StreamID:       l.rid,
+			IsRepairStream: true,
+		}
+	}
+
+	newRTXHarness(t, simTracks).run(t, omitsExtensions)
+}
+
+// -----------------------------------------------------------------------------
+// harness
+// -----------------------------------------------------------------------------
+
+type rtxHarness struct {
+	transport *rtc.PCTransport
+	pubPC     *webrtc.PeerConnection
+	writers   []*simulcastWriter
+	tracker   *rtxPairTracker
+}
+
+func newRTXHarness(t *testing.T, simTracks map[uint32]sfuinterceptor.SimulcastTrackInfo) *rtxHarness {
+	t.Helper()
+
+	hosts := vnettest.NewHosts(t)
+	tracker := newRTXPairTracker()
+
+	bufferFactory := buffer.NewFactoryOfBufferFactory(500, 200).CreateBufferFactory()
+	pcTransport := newPublisherTransportForTest(t, hosts.AnswerNet, bufferFactory, simTracks, tracker)
+	pubPC, writers := newSimulcastPublisherPC(t, hosts.OfferNet)
+
+	return &rtxHarness{
+		transport: pcTransport,
+		pubPC:     pubPC,
+		writers:   writers,
+		tracker:   tracker,
+	}
+}
+
+// run negotiates, publishes every layer, then retransmits a sequence number that is
+// never sent on the primary stream and waits for it to surface on the primary buffer.
+func (h *rtxHarness) run(t *testing.T, omitExtensions bool) {
+	t.Helper()
+
+	signalToTransport(t, h.pubPC, h.transport)
+	require.Equal(t, rtxTestMid, h.pubPC.GetTransceivers()[0].Mid())
+	for _, w := range h.writers {
+		w.mid = rtxTestMid
+		w.omitExtensions = omitExtensions
+	}
+
+	// every layer has to bind before RTX is sent, which is also the production ordering:
+	// a retransmission only follows a NACK for an established layer
+	require.True(
+		t,
+		sendUntil(t, 20*time.Second, func() bool { return h.tracker.boundCount() == len(rtxTestLayers) }, func() {
+			for _, w := range h.writers {
+				w.writePrimary(t)
+			}
+		}),
+		"timed out waiting for all simulcast layers to bind: %s", h.tracker.describe(),
+	)
+
+	require.True(
+		t,
+		sendUntil(t, 20*time.Second, func() bool {
+			for _, w := range h.writers {
+				if !h.tracker.sawSeq(w.rid, w.recoveredSeq) {
+					return false
+				}
+			}
+			return true
+		}, func() {
+			for _, w := range h.writers {
+				w.writeRepair(t, h.tracker.rtxPayloadType(w.rid), w.recoveredSeq)
+			}
+		}),
+		"retransmissions never recovered into the primary buffers: %s", h.tracker.describe(),
+	)
+}
+
+// newPublisherTransportForTest builds the production publisher transport on net and
+// wires up what ParticipantImpl/MediaTrack do with a published layer.
+func newPublisherTransportForTest(
+	t *testing.T,
+	net *vnet.Net,
+	bufferFactory *buffer.Factory,
+	simTracks map[uint32]sfuinterceptor.SimulcastTrackInfo,
+	tracker *rtxPairTracker,
+) *rtc.PCTransport {
+	t.Helper()
+
+	rtcConf := newVNetWebRTCConfig(t, net, bufferFactory)
+
+	handler := &transportfakes.FakeHandler{}
+	params := rtc.TransportParams{
+		Handler:         handler,
+		Config:          rtcConf,
+		DirectionConfig: rtcConf.Publisher,
+		ProtocolVersion: 6,
+		Logger:          logger.GetLogger(),
+		Transport:       livekit.SignalTarget_PUBLISHER,
+		SimTracks:       simTracks,
+		EnabledPublishCodecs: []*livekit.Codec{
+			{Mime: webrtc.MimeTypeVP8},
+			{Mime: webrtc.MimeTypeRTX},
+		},
+		// all candidates are carried in the answer, so the test needs no trickle
+		UseOneShotSignallingMode: true,
+	}
+
+	pcTransport, err := rtc.NewPCTransport(params)
+	require.NoError(t, err)
+	t.Cleanup(pcTransport.Close)
+
+	// mirror mediatrack.addReceiver: bind the buffer of each published layer, subscribe
+	// to the pairing notification, and drain the buffer the way WebRTCReceiver does
+	handler.OnTrackCalls(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) {
+		rid, ssrc := track.RID(), uint32(track.SSRC())
+
+		buff := bufferFactory.GetBuffer(ssrc)
+		if buff == nil {
+			t.Errorf("no buffer for published ssrc %d (rid %q)", ssrc, rid)
+			return
+		}
+		if err := buff.Bind(receiver.GetParameters(), track.Codec().RTPCodecCapability, 0); err != nil {
+			t.Errorf("binding buffer for rid %q failed: %v", rid, err)
+			return
+		}
+		buff.OnNotifyRTX(func(base, repair uint32, rsid string) {
+			tracker.pairFound(rsid, base, repair)
+		})
+
+		// mirror ParticipantImpl.onMediaTrack
+		pcTransport.RTPStreamPublished(ssrc, pcTransport.GetMid(receiver), rid)
+
+		tracker.layerBound(rid, ssrc, buff, receiver.GetParameters())
+		go tracker.drain(rid, buff)
+	})
+
+	return pcTransport
+}
+
+// signalToTransport runs a one-shot offer/answer against the transport and waits for
+// the publisher to connect.
+func signalToTransport(t *testing.T, pub *webrtc.PeerConnection, pcTransport *rtc.PCTransport) {
+	t.Helper()
+
+	connected := vnettest.UntilConnected(pub)
+
+	offer := vnettest.GatheredOffer(t, pub)
+	offer.SDP = stripDeclaredSSRCs(offer.SDP)
+	require.NoError(t, pcTransport.HandleRemoteDescription(offer, 1))
+
+	answer, _, err := pcTransport.GetAnswer()
+	require.NoError(t, err)
+	require.NoError(t, pub.SetRemoteDescription(answer))
+
+	select {
+	case <-connected:
+	case <-time.After(30 * time.Second):
+		t.Fatal("timed out waiting for the publisher to connect")
+	}
+}
+
+// -----------------------------------------------------------------------------
+// publisher: raw simulcast writer with a per-layer repair stream
+// -----------------------------------------------------------------------------
+
+// rawTrackLocal hands the test the negotiated write stream directly. Unlike
+// TrackLocalStaticRTP it does not rewrite SSRC or payload type, which is what lets a
+// repair stream be emitted on its own SSRC alongside the primary stream of the same rid.
+type rawTrackLocal struct {
+	id       string
+	streamID string
+	rid      string
+
+	lock    sync.Mutex
+	writers []webrtc.TrackLocalWriter
+	exts    []webrtc.RTPHeaderExtensionParameter
+}
+
+func (t *rawTrackLocal) Bind(ctx webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, error) {
+	for _, c := range ctx.CodecParameters() {
+		if c.PayloadType != vnettest.VP8PayloadType {
+			continue
+		}
+
+		t.lock.Lock()
+		t.writers = append(t.writers, ctx.WriteStream())
+		t.exts = ctx.HeaderExtensions()
+		t.lock.Unlock()
+		return c, nil
+	}
+	return webrtc.RTPCodecParameters{}, fmt.Errorf("vp8 not negotiated for rid %q", t.rid)
+}
+
+func (t *rawTrackLocal) Unbind(webrtc.TrackLocalContext) error { return nil }
+func (t *rawTrackLocal) ID() string                            { return t.id }
+func (t *rawTrackLocal) RID() string                           { return t.rid }
+func (t *rawTrackLocal) StreamID() string                      { return t.streamID }
+func (t *rawTrackLocal) Kind() webrtc.RTPCodecType             { return webrtc.RTPCodecTypeVideo }
+
+func (t *rawTrackLocal) extensionID(uri string) uint8 {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	for _, e := range t.exts {
+		if e.URI == uri {
+			return uint8(e.ID)
+		}
+	}
+	return 0
+}
+
+func (t *rawTrackLocal) write(header *rtp.Header, payload []byte) {
+	t.lock.Lock()
+	writers := append([]webrtc.TrackLocalWriter(nil), t.writers...)
+	t.lock.Unlock()
+
+	for _, w := range writers {
+		_, _ = w.WriteRTP(header, payload)
+	}
+}
+
+// simulcastWriter emits the primary and repair streams of one simulcast layer.
+type simulcastWriter struct {
+	track   *rawTrackLocal
+	mid     string
+	rid     string
+	ssrc    uint32
+	rtxSSRC uint32
+
+	// omitExtensions emulates a migrated publisher, which sends no mid/rid/rsid
+	omitExtensions bool
+
+	// recoveredSeq is only ever sent inside an RTX payload, never on the primary
+	// stream, so its arrival on the primary buffer proves RTX recovery worked
+	recoveredSeq uint16
+
+	lock sync.Mutex
+	seq  uint16
+}
+
+func (w *simulcastWriter) nextSeq() uint16 {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	w.seq++
+	return w.seq
+}
+
+func (w *simulcastWriter) header(t *testing.T, ssrc uint32, pt uint8, seq uint16, rid, rsid string) *rtp.Header {
+	t.Helper()
+
+	h := &rtp.Header{
+		Version:        2,
+		PayloadType:    pt,
+		SequenceNumber: seq,
+		Timestamp:      uint32(seq) * 3000,
+		SSRC:           ssrc,
+	}
+	if w.omitExtensions {
+		return h
+	}
+
+	midID := w.track.extensionID(sdp.SDESMidURI)
+	require.NotZero(t, midID, "sdes:mid not negotiated")
+	require.NoError(t, h.SetExtension(midID, []byte(w.mid)))
+
+	if rid != "" {
+		ridID := w.track.extensionID(sdp.SDESRTPStreamIDURI)
+		require.NotZero(t, ridID, "sdes:rtp-stream-id not negotiated")
+		require.NoError(t, h.SetExtension(ridID, []byte(rid)))
+	}
+	if rsid != "" {
+		rsidID := w.track.extensionID(sdp.SDESRepairRTPStreamIDURI)
+		require.NotZero(t, rsidID, "sdes:repaired-rtp-stream-id not negotiated")
+		require.NoError(t, h.SetExtension(rsidID, []byte(rsid)))
+	}
+	return h
+}
+
+func (w *simulcastWriter) writePrimary(t *testing.T) {
+	t.Helper()
+
+	w.track.write(w.header(t, w.ssrc, vnettest.VP8PayloadType, w.nextSeq(), w.rid, ""), vp8TestPayload())
+}
+
+// writeRepair emits an RFC 4588 repair packet: the sequence number being retransmitted
+// is prepended to the payload.
+func (w *simulcastWriter) writeRepair(t *testing.T, rtxPT uint8, originalSeq uint16) {
+	t.Helper()
+
+	if rtxPT == 0 {
+		rtxPT = vnettest.RTXPayloadType
+	}
+
+	inner := vp8TestPayload()
+	payload := make([]byte, 2+len(inner))
+	binary.BigEndian.PutUint16(payload[:2], originalSeq)
+	copy(payload[2:], inner)
+
+	w.track.write(w.header(t, w.rtxSSRC, rtxPT, w.nextSeq(), "", w.rid), payload)
+}
+
+func vp8TestPayload() []byte {
+	return []byte{0x10, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x40, 0x01, 0xf0, 0x00}
+}
+
+// newSimulcastPublisherPC builds the publishing peer connection and one writer per
+// simulcast layer.
+func newSimulcastPublisherPC(t *testing.T, net *vnet.Net) (*webrtc.PeerConnection, []*simulcastWriter) {
+	t.Helper()
+
+	pc := vnettest.NewPeerConnection(t, vnettest.PCConfig{
+		Net: net,
+		MediaEngine: vnettest.MediaEngineConfig{
+			Video:               true,
+			HeaderExtensions:    true,
+			SimulcastExtensions: true,
+		},
+	})
+
+	writers := make([]*simulcastWriter, 0, len(rtxTestLayers))
+	for _, l := range rtxTestLayers {
+		writers = append(writers, &simulcastWriter{
+			track:        &rawTrackLocal{id: "video", streamID: "pion", rid: l.rid},
+			rid:          l.rid,
+			ssrc:         l.ssrc,
+			rtxSSRC:      l.rtxSSRC,
+			recoveredSeq: l.recoveredSeq,
+		})
+	}
+
+	sender, err := pc.AddTrack(writers[0].track)
+	require.NoError(t, err)
+	for _, w := range writers[1:] {
+		require.NoError(t, sender.AddEncoding(w.track))
+	}
+
+	return pc, writers
+}
+
+// -----------------------------------------------------------------------------
+// tracking
+// -----------------------------------------------------------------------------
+
+type rtxPairTracker struct {
+	lock sync.Mutex
+
+	bound  map[string]uint32 // rid -> base ssrc
+	params map[string]webrtc.RTPParameters
+	pairs  map[string][2]uint32       // rsid -> {base ssrc, repair ssrc}
+	seen   map[string]map[uint16]bool // rid -> sequence numbers read off the primary buffer
+}
+
+func newRTXPairTracker() *rtxPairTracker {
+	return &rtxPairTracker{
+		bound:  make(map[string]uint32),
+		params: make(map[string]webrtc.RTPParameters),
+		pairs:  make(map[string][2]uint32),
+		seen:   make(map[string]map[uint16]bool),
+	}
+}
+
+func (t *rtxPairTracker) layerBound(rid string, ssrc uint32, buff *buffer.Buffer, params webrtc.RTPParameters) {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	t.bound[rid] = ssrc
+	t.params[rid] = params
+	t.seen[rid] = make(map[uint16]bool)
+}
+
+func (t *rtxPairTracker) pairFound(rsid string, base, repair uint32) {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	t.pairs[rsid] = [2]uint32{base, repair}
+}
+
+func (t *rtxPairTracker) boundCount() int {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	return len(t.bound)
+}
+
+func (t *rtxPairTracker) pairCount() int {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	return len(t.pairs)
+}
+
+func (t *rtxPairTracker) pair(rid string) (uint32, uint32) {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	p := t.pairs[rid]
+	return p[0], p[1]
+}
+
+func (t *rtxPairTracker) rtxPayloadType(rid string) uint8 {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	for _, c := range t.params[rid].Codecs {
+		if c.MimeType == webrtc.MimeTypeRTX {
+			return uint8(c.PayloadType)
+		}
+	}
+	return 0
+}
+
+func (t *rtxPairTracker) sawSeq(rid string, seq uint16) bool {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	return t.seen[rid][seq]
+}
+
+// drain consumes the primary buffer the way WebRTCReceiver does, recording which
+// sequence numbers made it through.
+func (t *rtxPairTracker) drain(rid string, buff *buffer.Buffer) {
+	b := make([]byte, 1500)
+	for {
+		ep, err := buff.ReadExtended(b)
+		if err != nil {
+			return
+		}
+		if ep == nil || ep.Packet == nil {
+			continue
+		}
+
+		t.lock.Lock()
+		t.seen[rid][ep.Packet.SequenceNumber] = true
+		t.lock.Unlock()
+	}
+}
+
+func (t *rtxPairTracker) describe() string {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	return fmt.Sprintf("bound=%v pairs=%v", t.bound, t.pairs)
+}

test/scenarios.go

@@ -0,0 +1,388 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"go.uber.org/atomic"
+
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+
+	"github.com/livekit/livekit-server/pkg/testutils"
+	testclient "github.com/livekit/livekit-server/test/client"
+)
+
+// a scenario with lots of clients connecting, publishing, and leaving at random periods
+func scenarioPublishingUponJoining(t *testing.T) {
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("puj_1", defaultServerPort, testRTCServicePath, nil)
+			c2 := createRTCClient("puj_2", secondServerPort, testRTCServicePath, &testclient.Options{AutoSubscribe: true})
+			c3 := createRTCClient("puj_3", defaultServerPort, testRTCServicePath, &testclient.Options{AutoSubscribe: true})
+			defer stopClients(c1, c2, c3)
+
+			waitUntilConnected(t, c1, c2, c3)
+
+			// c1 and c2 publishing, c3 just receiving
+			writers := publishTracksForClients(t, c1, c2)
+			defer stopWriters(writers...)
+
+			logger.Infow("waiting to receive tracks from c1 and c2")
+			testutils.WithTimeout(t, func() string {
+				tracks := c3.SubscribedTracks()
+				if len(tracks[c1.ID()]) != 2 {
+					return "did not receive tracks from c1"
+				}
+				if len(tracks[c2.ID()]) != 2 {
+					return "did not receive tracks from c2"
+				}
+				return ""
+			})
+
+			// after a delay, c2 reconnects, then publishing
+			time.Sleep(syncDelay)
+			c2.Stop()
+
+			logger.Infow("waiting for c2 tracks to be gone")
+			testutils.WithTimeout(t, func() string {
+				tracks := c3.SubscribedTracks()
+
+				if len(tracks[c1.ID()]) != 2 {
+					return fmt.Sprintf("c3 should be subscribed to 2 tracks from c1, actual: %d", len(tracks[c1.ID()]))
+				}
+				if len(tracks[c2.ID()]) != 0 {
+					return fmt.Sprintf("c3 should be subscribed to 0 tracks from c2, actual: %d", len(tracks[c2.ID()]))
+				}
+				if len(c1.SubscribedTracks()[c2.ID()]) != 0 {
+					return fmt.Sprintf("c3 should be subscribed to 0 tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()]))
+				}
+				return ""
+			})
+
+			logger.Infow("c2 reconnecting")
+			// connect to a diff port
+			c2 = createRTCClient("puj_2", defaultServerPort, testRTCServicePath, nil)
+			defer c2.Stop()
+			waitUntilConnected(t, c2)
+			writers = publishTracksForClients(t, c2)
+			defer stopWriters(writers...)
+
+			testutils.WithTimeout(t, func() string {
+				tracks := c3.SubscribedTracks()
+				// "new c2 tracks should be published again",
+				if len(tracks[c2.ID()]) != 2 {
+					return fmt.Sprintf("c3 should be subscribed to 2 tracks from c2, actual: %d", len(tracks[c2.ID()]))
+				}
+				if len(c1.SubscribedTracks()[c2.ID()]) != 2 {
+					return fmt.Sprintf("c1 should be subscribed to 2 tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()]))
+				}
+				return ""
+			})
+		})
+	}
+}
+
+func scenarioReceiveBeforePublish(t *testing.T) {
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("rbp_1", defaultServerPort, testRTCServicePath, nil)
+			c2 := createRTCClient("rbp_2", defaultServerPort, testRTCServicePath, nil)
+
+			waitUntilConnected(t, c1, c2)
+			defer stopClients(c1, c2)
+
+			// c1 publishes
+			writers := publishTracksForClients(t, c1)
+			defer stopWriters(writers...)
+
+			// c2 should see some bytes flowing through
+			testutils.WithTimeout(t, func() string {
+				if c2.BytesReceived() > 20 {
+					return ""
+				} else {
+					return fmt.Sprintf("c2 only received %d bytes", c2.BytesReceived())
+				}
+			})
+
+			// now publish on C2
+			writers = publishTracksForClients(t, c2)
+			defer stopWriters(writers...)
+
+			testutils.WithTimeout(t, func() string {
+				if len(c1.SubscribedTracks()[c2.ID()]) == 2 {
+					return ""
+				} else {
+					return fmt.Sprintf("expected c1 to receive 2 tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()]))
+				}
+			})
+
+			// now leave, and ensure that it's immediate
+			c2.Stop()
+
+			testutils.WithTimeout(t, func() string {
+				if len(c1.RemoteParticipants()) > 0 {
+					return fmt.Sprintf("expected no remote participants, actual: %v", c1.RemoteParticipants())
+				}
+				return ""
+			})
+		})
+	}
+}
+
+func scenarioDataPublish(t *testing.T) {
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("scenarioDataPublish/testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("dp1", defaultServerPort, testRTCServicePath, nil)
+			c2 := createRTCClient("dp2", secondServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1, c2)
+			defer stopClients(c1, c2)
+
+			payload := "test bytes"
+
+			received := atomic.NewBool(false)
+			c2.OnDataReceived = func(data []byte, sid string) {
+				if string(data) == payload && livekit.ParticipantID(sid) == c1.ID() {
+					received.Store(true)
+				}
+			}
+
+			require.NoError(t, c1.PublishData([]byte(payload), livekit.DataPacket_RELIABLE))
+
+			testutils.WithTimeout(t, func() string {
+				if received.Load() {
+					return ""
+				} else {
+					return "c2 did not receive published data"
+				}
+			})
+		})
+	}
+}
+
+func scenarioDataUnlabeledPublish(t *testing.T) {
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("scenarioDataUnlabeledPublish/testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("dp1", defaultServerPort, testRTCServicePath, nil)
+			c2 := createRTCClient("dp2", secondServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1, c2)
+			defer stopClients(c1, c2)
+
+			payload := "test unlabeled bytes"
+
+			received := atomic.NewBool(false)
+			c2.OnDataReceived = func(data []byte, _sid string) {
+				if string(data) == payload {
+					received.Store(true)
+				}
+			}
+
+			require.NoError(t, c1.PublishDataUnlabeled([]byte(payload)))
+
+			testutils.WithTimeout(t, func() string {
+				if received.Load() {
+					return ""
+				} else {
+					return "c2 did not receive published data unlabeled"
+				}
+			})
+		})
+	}
+}
+
+func scenarioDataTracksPublishingUponJoining(t *testing.T) {
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("scenarioDataTracksPublishingUponJoining/testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("dtpuj_1", defaultServerPort, testRTCServicePath, &testclient.Options{AutoSubscribeDataTrack: true})
+			c2 := createRTCClient("dtpuj_2", secondServerPort, testRTCServicePath, &testclient.Options{AutoSubscribeDataTrack: true})
+			c3 := createRTCClient("dtpuj_3", defaultServerPort, testRTCServicePath, &testclient.Options{AutoSubscribeDataTrack: true})
+			defer stopClients(c1, c2, c3)
+
+			waitUntilConnected(t, c1, c2, c3)
+
+			// c1 and c2 publishing, c3 just receiving
+			writers := publishDataTracksForClients(t, c1, c2)
+			defer stopWriters(writers...)
+
+			logger.Infow("waiting to receive data tracks from c1 and c2")
+			testutils.WithTimeout(t, func() string {
+				tracks := c3.SubscribedDataTracks()
+				if len(tracks[c1.ID()]) != 2 {
+					return "did not receive data tracks from c1"
+				}
+				if len(tracks[c2.ID()]) != 2 {
+					return "did not receive data tracks from c2"
+				}
+				for _, dts := range tracks {
+					for _, dt := range dts {
+						if dt.NumReceivedPackets() == 0 {
+							return fmt.Sprintf("no packets received from %s", dt.ID())
+						}
+					}
+				}
+				return ""
+			})
+
+			// after a delay, c2 reconnects, then publishing
+			time.Sleep(syncDelay)
+			c2.Stop()
+
+			logger.Infow("waiting for c2 data tracks to be gone")
+			testutils.WithTimeout(t, func() string {
+				tracks := c3.SubscribedDataTracks()
+
+				if len(tracks[c1.ID()]) != 2 {
+					return fmt.Sprintf("c3 should be subscribed to 2 data tracks from c1, actual: %d", len(tracks[c1.ID()]))
+				}
+				if len(tracks[c2.ID()]) != 0 {
+					return fmt.Sprintf("c3 should be subscribed to 0 data tracks from c2, actual: %d", len(tracks[c2.ID()]))
+				}
+				if len(c1.SubscribedDataTracks()[c2.ID()]) != 0 {
+					return fmt.Sprintf("c3 should be subscribed to 0 data tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()]))
+				}
+				return ""
+			})
+
+			logger.Infow("c2 reconnecting")
+			// connect to a diff port
+			c2 = createRTCClient("dtpuj_2", defaultServerPort, testRTCServicePath, &testclient.Options{AutoSubscribeDataTrack: true})
+			defer c2.Stop()
+			waitUntilConnected(t, c2)
+			writers = publishDataTracksForClients(t, c2)
+			defer stopWriters(writers...)
+
+			testutils.WithTimeout(t, func() string {
+				tracks := c3.SubscribedDataTracks()
+				// new c2 data tracks should be published again
+				if len(tracks[c2.ID()]) != 2 {
+					return fmt.Sprintf("c3 should be subscribed to 2 data tracks from c2, actual: %d", len(tracks[c2.ID()]))
+				}
+				for _, dt := range tracks[c2.ID()] {
+					if dt.NumReceivedPackets() == 0 {
+						return fmt.Sprintf("c3 did not receive packets from c2 data track after reconnecting %s", dt.ID())
+					}
+				}
+
+				if len(c1.SubscribedDataTracks()[c2.ID()]) != 2 {
+					return fmt.Sprintf("c1 should be subscribed to 2 data tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()]))
+				}
+				for _, dt := range c1.SubscribedDataTracks()[c2.ID()] {
+					if dt.NumReceivedPackets() == 0 {
+						return fmt.Sprintf("c1 did not receive packets from c2 data track after reconnecting %s", dt.ID())
+					}
+				}
+				return ""
+			})
+		})
+	}
+}
+
+func scenarioJoinClosedRoom(t *testing.T) {
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("jcr1", defaultServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1)
+
+			// close room with room client
+			_, err := roomClient.DeleteRoom(contextWithToken(createRoomToken()), &livekit.DeleteRoomRequest{
+				Room: testRoom,
+			})
+			require.NoError(t, err)
+
+			// now join again
+			c2 := createRTCClient("jcr2", defaultServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c2)
+			stopClients(c2)
+		})
+	}
+}
+
+// close a room that has been created, but no participant has joined
+func closeNonRTCRoom(t *testing.T) {
+	createCtx := contextWithToken(createRoomToken())
+	_, err := roomClient.CreateRoom(createCtx, &livekit.CreateRoomRequest{
+		Name: testRoom,
+	})
+	require.NoError(t, err)
+
+	_, err = roomClient.DeleteRoom(createCtx, &livekit.DeleteRoomRequest{
+		Room: testRoom,
+	})
+	require.NoError(t, err)
+}
+
+func publishTracksForClients(t *testing.T, clients ...*testclient.RTCClient) []testclient.TrackWriter {
+	logger.Infow("publishing tracks for clients")
+	var writers []testclient.TrackWriter
+	for i := range clients {
+		c := clients[i]
+		tw, err := c.AddStaticTrack("audio/opus", "audio", "webcam")
+		require.NoError(t, err)
+		writers = append(writers, tw)
+
+		tw, err = c.AddStaticTrack("video/vp8", "video", "webcam")
+		require.NoError(t, err)
+		writers = append(writers, tw)
+	}
+	return writers
+}
+
+func publishDataTracksForClients(t *testing.T, clients ...*testclient.RTCClient) []testclient.TrackWriter {
+	logger.Infow("publishing data tracks for clients")
+	var writers []testclient.TrackWriter
+	for i := range clients {
+		c := clients[i]
+		for range 2 {
+			dtw, err := c.PublishDataTrack()
+			require.NoError(t, err)
+			writers = append(writers, dtw)
+		}
+	}
+	return writers
+}
+
+// Room service tests
+
+func roomServiceListRoom(t *testing.T) {
+	createCtx := contextWithToken(createRoomToken())
+	listCtx := contextWithToken(listRoomToken())
+	// create rooms
+	_, err := roomClient.CreateRoom(createCtx, &livekit.CreateRoomRequest{
+		Name: testRoom,
+	})
+	require.NoError(t, err)
+	_, err = roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{
+		Name: "yourroom",
+	})
+	require.NoError(t, err)
+
+	t.Run("list all rooms", func(t *testing.T) {
+		res, err := roomClient.ListRooms(listCtx, &livekit.ListRoomsRequest{})
+		require.NoError(t, err)
+		require.Len(t, res.Rooms, 2)
+	})
+	t.Run("list specific rooms", func(t *testing.T) {
+		res, err := roomClient.ListRooms(listCtx, &livekit.ListRoomsRequest{
+			Names: []string{"yourroom"},
+		})
+		require.NoError(t, err)
+		require.Len(t, res.Rooms, 1)
+		require.Equal(t, "yourroom", res.Rooms[0].Name)
+	})
+}

test/signal_message_size_limit_test.go

@@ -0,0 +1,223 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/gorilla/websocket"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/livekit/protocol/livekit"
+
+	"github.com/livekit/livekit-server/pkg/config"
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	testclient "github.com/livekit/livekit-server/test/client"
+)
+
+// TestSignalMessageSizeLimit drives the full /rtc signalling path: a real client
+// connects to a single-node server configured with a small read-message size
+// limit and sends an oversized frame. The server must reject it at the transport
+// and close the connection with a 1009 (message too big) close code.
+func TestSignalMessageSizeLimit(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+		return
+	}
+
+	const limit = 1024
+	_, finish := setupSingleNodeTestWithConfig("TestSignalMessageSizeLimit", func(conf *config.Config) {
+		conf.Limit.SignalMessageSizeLimit = limit
+	})
+	defer finish()
+
+	token := joinToken(testRoom, "oversized", nil)
+	opts := &testclient.Options{AutoSubscribe: true}
+	testRTCServicePathToTestClientOptions(testRTCServicePathv0, opts)
+
+	ws, err := testclient.NewWebSocketConn(fmt.Sprintf("ws://localhost:%d", defaultServerPort), token, opts)
+	require.NoError(t, err)
+	defer ws.Close()
+
+	// send a frame well over the configured limit
+	require.NoError(t, ws.WriteMessage(websocket.BinaryMessage, make([]byte, limit*4)))
+
+	// the server rejects the oversized frame and closes the connection; drain any
+	// buffered server messages (e.g. JoinResponse) until the close surfaces
+	_ = ws.SetReadDeadline(time.Now().Add(5 * time.Second))
+	for {
+		if _, _, err = ws.ReadMessage(); err != nil {
+			break
+		}
+	}
+	require.True(t, websocket.IsCloseError(err, websocket.CloseMessageTooBig), "got %v", err)
+}
+
+// TestSignalMessageSizeLimitDisabled verifies that a limit of 0 leaves the
+// connection unbounded: a frame larger than the default limit is not rejected by
+// the transport with a 1009 close, and the server still delivers its signalling
+// messages (e.g. the JoinResponse).
+func TestSignalMessageSizeLimitDisabled(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+		return
+	}
+
+	_, finish := setupSingleNodeTestWithConfig("TestSignalMessageSizeLimitDisabled", func(conf *config.Config) {
+		conf.Limit.SignalMessageSizeLimit = 0
+	})
+	defer finish()
+
+	token := joinToken(testRoom, "oversized-disabled", nil)
+	opts := &testclient.Options{AutoSubscribe: true}
+	testRTCServicePathToTestClientOptions(testRTCServicePathv0, opts)
+
+	ws, err := testclient.NewWebSocketConn(fmt.Sprintf("ws://localhost:%d", defaultServerPort), token, opts)
+	require.NoError(t, err)
+	defer ws.Close()
+
+	// send a frame larger than the default 2 MiB limit; with the limit disabled the
+	// transport must not reject it
+	require.NoError(t, ws.WriteMessage(websocket.BinaryMessage, make([]byte, 3<<20)))
+
+	// the server still delivers signalling (the JoinResponse) and never closes with
+	// a 1009; drain until an error and assert it is not a message-too-big close
+	_ = ws.SetReadDeadline(time.Now().Add(5 * time.Second))
+	var reads int
+	for {
+		if _, _, err = ws.ReadMessage(); err != nil {
+			break
+		}
+		reads++
+	}
+	require.Positive(t, reads, "expected to read at least one server message")
+	require.False(t, websocket.IsCloseError(err, websocket.CloseMessageTooBig), "got %v", err)
+}
+
+// dialCompressed opens a permessage-deflate signalling connection to /rtc and
+// returns a valid, highly compressible SignalRequest whose marshaled (uncompressed)
+// size exceeds sizeAtLeast. On the wire the frame compresses to a few hundred
+// bytes, staying well under the transport read limit, so only the decompressed
+// bound can reject it.
+func dialCompressed(t *testing.T, identity string, sizeAtLeast int) (*websocket.Conn, []byte) {
+	t.Helper()
+
+	token := joinToken(testRoom, identity, nil)
+	header := make(http.Header)
+	testclient.SetAuthorizationToken(header, token)
+
+	dialer := websocket.Dialer{EnableCompression: true}
+	url := fmt.Sprintf(
+		"ws://localhost:%d/rtc?protocol=%d&auto_subscribe=true&sdk=go",
+		defaultServerPort, types.CurrentProtocol,
+	)
+	ws, _, err := dialer.Dial(url, header)
+	require.NoError(t, err)
+	ws.EnableWriteCompression(true)
+
+	req := &livekit.SignalRequest{
+		Message: &livekit.SignalRequest_UpdateMetadata{
+			UpdateMetadata: &livekit.UpdateParticipantMetadata{
+				Metadata: strings.Repeat("a", sizeAtLeast),
+			},
+		},
+	}
+	payload, err := proto.Marshal(req)
+	require.NoError(t, err)
+	require.Greater(t, len(payload), sizeAtLeast)
+
+	return ws, payload
+}
+
+// TestSignalMessageSizeLimitDecompressed reproduces the permessage-deflate
+// amplification case: a valid frame stays tiny on the wire (well under the
+// transport read limit) but inflates past the limit once decompressed. The
+// decompressed-size guard must reject it and the server must close the connection.
+func TestSignalMessageSizeLimitDecompressed(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+		return
+	}
+
+	const limit = 64 << 10 // 64 KiB
+	_, finish := setupSingleNodeTestWithConfig("TestSignalMessageSizeLimitDecompressed", func(conf *config.Config) {
+		conf.Limit.SignalMessageSizeLimit = limit
+	})
+	defer finish()
+
+	ws, payload := dialCompressed(t, "compressed", limit*8)
+	defer ws.Close()
+	require.NoError(t, ws.WriteMessage(websocket.BinaryMessage, payload))
+
+	// the server rejects the oversized decompressed message and tears down the
+	// connection: draining eventually yields a close/EOF rather than a read timeout
+	_ = ws.SetReadDeadline(time.Now().Add(5 * time.Second))
+	var err error
+	for {
+		if _, _, err = ws.ReadMessage(); err != nil {
+			break
+		}
+	}
+	require.Error(t, err)
+	require.False(t, isTimeout(err), "connection should be closed by the server, not time out; got %v", err)
+	require.False(t, websocket.IsCloseError(err, websocket.CloseMessageTooBig), "got %v", err)
+}
+
+// TestSignalMessageSizeLimitDecompressedDisabled is the counterpart to
+// TestSignalMessageSizeLimitDecompressed: with the limit disabled the same valid
+// large frame is accepted and the connection stays open (the drain times out
+// rather than being closed).
+func TestSignalMessageSizeLimitDecompressedDisabled(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+		return
+	}
+
+	_, finish := setupSingleNodeTestWithConfig("TestSignalMessageSizeLimitDecompressedDisabled", func(conf *config.Config) {
+		conf.Limit.SignalMessageSizeLimit = 0
+	})
+	defer finish()
+
+	ws, payload := dialCompressed(t, "compressed-disabled", 1<<20)
+	defer ws.Close()
+	require.NoError(t, ws.WriteMessage(websocket.BinaryMessage, payload))
+
+	// the message is accepted; the connection stays open, so draining blocks until
+	// the read deadline instead of observing a server-initiated close
+	_ = ws.SetReadDeadline(time.Now().Add(2 * time.Second))
+	var err error
+	for {
+		if _, _, err = ws.ReadMessage(); err != nil {
+			break
+		}
+	}
+	require.Error(t, err)
+	require.True(t, isTimeout(err), "connection should stay open and time out; got %v", err)
+}
+
+func isTimeout(err error) bool {
+	var netErr net.Error
+	if errors.As(err, &netErr) {
+		return netErr.Timeout()
+	}
+	return false
+}

test/webhook_test.go

@@ -0,0 +1,242 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/protocol/auth"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils/guid"
+	"github.com/livekit/protocol/utils/protojson"
+	"github.com/livekit/protocol/webhook"
+
+	"github.com/livekit/livekit-server/pkg/config"
+	"github.com/livekit/livekit-server/pkg/routing"
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/livekit-server/pkg/service"
+	"github.com/livekit/livekit-server/pkg/testutils"
+)
+
+func TestWebhooks(t *testing.T) {
+	server, ts, finish, err := setupServerWithWebhook()
+	require.NoError(t, err)
+	defer finish()
+
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1)
+			testutils.WithTimeout(t, func() string {
+				if ts.GetEvent(webhook.EventRoomStarted) == nil {
+					return "did not receive RoomStarted"
+				}
+				if ts.GetEvent(webhook.EventParticipantJoined) == nil {
+					return "did not receive ParticipantJoined"
+				}
+				return ""
+			})
+
+			// first participant join should have started the room
+			started := ts.GetEvent(webhook.EventRoomStarted)
+			require.Equal(t, testRoom, started.Room.Name)
+			require.NotEmpty(t, started.Id)
+			require.Greater(t, started.CreatedAt, time.Now().Unix()-100)
+			require.GreaterOrEqual(t, time.Now().Unix(), started.CreatedAt)
+			joined := ts.GetEvent(webhook.EventParticipantJoined)
+			require.Equal(t, "c1", joined.Participant.Identity)
+			ts.ClearEvents()
+
+			// another participant joins
+			c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c2)
+			defer c2.Stop()
+			testutils.WithTimeout(t, func() string {
+				if ts.GetEvent(webhook.EventParticipantJoined) == nil {
+					return "did not receive ParticipantJoined"
+				}
+				return ""
+			})
+			joined = ts.GetEvent(webhook.EventParticipantJoined)
+			require.Equal(t, "c2", joined.Participant.Identity)
+			ts.ClearEvents()
+
+			// track published
+			writers := publishTracksForClients(t, c1)
+			defer stopWriters(writers...)
+			testutils.WithTimeout(t, func() string {
+				ev := ts.GetEvent(webhook.EventTrackPublished)
+				if ev == nil {
+					return "did not receive TrackPublished"
+				}
+				require.NotNil(t, ev.Track, "TrackPublished did not include trackInfo")
+				require.Equal(t, string(c1.ID()), ev.Participant.Sid)
+				return ""
+			})
+			ts.ClearEvents()
+
+			// first participant leaves
+			c1.Stop()
+			testutils.WithTimeout(t, func() string {
+				if ts.GetEvent(webhook.EventParticipantLeft) == nil {
+					return "did not receive ParticipantLeft"
+				}
+				return ""
+			})
+			left := ts.GetEvent(webhook.EventParticipantLeft)
+			require.Equal(t, "c1", left.Participant.Identity)
+			ts.ClearEvents()
+
+			// room closed
+			rm := server.RoomManager().GetRoom(context.Background(), testRoom)
+			rm.Close(types.RoomCloseReasonAPIDelete)
+			testutils.WithTimeout(t, func() string {
+				if ts.GetEvent(webhook.EventRoomFinished) == nil {
+					return "did not receive RoomFinished"
+				}
+				return ""
+			})
+			require.Equal(t, testRoom, ts.GetEvent(webhook.EventRoomFinished).Room.Name)
+			require.Equal(t, livekit.RoomEndReason_ROOM_END_API_DELETE, ts.GetEvent(webhook.EventRoomFinished).RoomEndReason)
+		})
+	}
+}
+
+func setupServerWithWebhook() (server *service.LivekitServer, testServer *webhookTestServer, finishFunc func(), err error) {
+	conf, err := config.NewConfig("", true, nil, nil)
+	if err != nil {
+		panic(fmt.Sprintf("could not create config: %v", err))
+	}
+	conf.WebHook.URLs = []string{"http://localhost:7890"}
+	conf.WebHook.APIKey = testApiKey
+	conf.Keys = map[string]string{testApiKey: testApiSecret}
+
+	testServer = newTestServer(":7890")
+	if err = testServer.Start(); err != nil {
+		return
+	}
+
+	currentNode, err := routing.NewLocalNode(conf)
+	if err != nil {
+		return
+	}
+	currentNode.SetNodeID(livekit.NodeID(guid.New(nodeID1)))
+
+	server, err = service.InitializeServer(conf, currentNode)
+	if err != nil {
+		return
+	}
+
+	go func() {
+		if err := server.Start(); err != nil {
+			logger.Errorw("server returned error", err)
+		}
+	}()
+
+	waitForServerToStart(server)
+
+	finishFunc = func() {
+		server.Stop(true)
+		testServer.Stop()
+	}
+	return
+}
+
+type webhookTestServer struct {
+	server   *http.Server
+	events   map[string]*livekit.WebhookEvent
+	lock     sync.Mutex
+	provider auth.KeyProvider
+}
+
+func newTestServer(addr string) *webhookTestServer {
+	s := &webhookTestServer{
+		events:   make(map[string]*livekit.WebhookEvent),
+		provider: auth.NewFileBasedKeyProviderFromMap(map[string]string{testApiKey: testApiSecret}),
+	}
+	s.server = &http.Server{
+		Addr:    addr,
+		Handler: s,
+	}
+	return s
+}
+
+func (s *webhookTestServer) ServeHTTP(_ http.ResponseWriter, r *http.Request) {
+	data, err := webhook.Receive(r, s.provider)
+	if err != nil {
+		logger.Errorw("could not receive webhook", err)
+		return
+	}
+
+	event := livekit.WebhookEvent{}
+	if err = protojson.Unmarshal(data, &event); err != nil {
+		logger.Errorw("could not unmarshal event", err)
+		return
+	}
+
+	s.lock.Lock()
+	s.events[event.Event] = &event
+	s.lock.Unlock()
+}
+
+func (s *webhookTestServer) GetEvent(name string) *livekit.WebhookEvent {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	return s.events[name]
+}
+
+func (s *webhookTestServer) ClearEvents() {
+	s.lock.Lock()
+	s.events = make(map[string]*livekit.WebhookEvent)
+	s.lock.Unlock()
+}
+
+func (s *webhookTestServer) Start() error {
+	l, err := net.Listen("tcp", s.server.Addr)
+	if err != nil {
+		return err
+	}
+	go s.server.Serve(l)
+
+	// wait for webhook server to start
+	ctx, cancel := context.WithTimeout(context.Background(), testutils.ConnectTimeout)
+	defer cancel()
+	for {
+		select {
+		case <-ctx.Done():
+			return errors.New("could not start webhook server after timeout")
+		case <-time.After(10 * time.Millisecond):
+			// ensure we can connect to it
+			res, err := http.Get(fmt.Sprintf("http://localhost%s", s.server.Addr))
+			if err == nil && res.StatusCode == http.StatusOK {
+				return nil
+			}
+		}
+	}
+}
+
+func (s *webhookTestServer) Stop() {
+	_ = s.server.Shutdown(context.Background())
+}