snapshot of https://github.com/livekit/livekit.git @ 08c5433449c106943231037d0b892895f6e92d3b (2026-09-15)

pkg/sfu/NOTICE

@@ -0,0 +1,16 @@
+Portions of this package originated from ion-sfu: https://github.com/pion/ion-sfu.
+
+MIT License
+
+Copyright (c) 2019
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+----------------------------------------------------------------------------------------

pkg/sfu/audio/audiolevel.go

@@ -0,0 +1,197 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package audio
+
+import (
+	"math"
+	"sync"
+)
+
+const (
+	silentAudioLevel = 127
+	negInv20         = -1.0 / 20
+)
+
+// --------------------------------------
+
+type AudioLevelConfig struct {
+	// minimum level to be considered active, 0-127, where 0 is loudest
+	ActiveLevel uint8 `yaml:"active_level,omitempty"`
+	// percentile to measure, a participant is considered active if it has exceeded the ActiveLevel more than
+	// MinPercentile% of the time
+	MinPercentile uint8 `yaml:"min_percentile,omitempty"`
+	// interval to update clients, in ms
+	UpdateInterval uint32 `yaml:"update_interval,omitempty"`
+	// smoothing for audioLevel values sent to the client.
+	// audioLevel will be an average of `smooth_intervals`, 0 to disable
+	SmoothIntervals uint32 `yaml:"smooth_intervals,omitempty"`
+}
+
+var (
+	DefaultAudioLevelConfig = AudioLevelConfig{
+		ActiveLevel:     35, // -35dBov
+		MinPercentile:   40,
+		UpdateInterval:  400,
+		SmoothIntervals: 2,
+	}
+)
+
+// --------------------------------------
+
+type AudioLevelParams struct {
+	ClockRate uint32
+}
+
+// keeps track of audio level for a participant
+type AudioLevel struct {
+	params AudioLevelParams
+
+	config AudioLevelConfig
+
+	// min duration within an observe duration window to be considered active
+	minActiveDuration uint32
+	smoothFactor      float64
+	activeThreshold   float64
+
+	lock          sync.Mutex
+	smoothedLevel float64
+
+	loudestObservedLevel uint8
+	activeDuration       uint32 // ms
+	observedDuration     uint32 // ms
+	lastObservedAt       int64
+
+	highestRTPTimestamp            uint32
+	highestRTPTimestampInitialized bool
+}
+
+func NewAudioLevel(params AudioLevelParams) *AudioLevel {
+	return &AudioLevel{
+		params:               params,
+		smoothFactor:         1,
+		loudestObservedLevel: silentAudioLevel,
+	}
+}
+
+func (l *AudioLevel) SetConfig(config AudioLevelConfig) {
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	l.config = config
+	l.minActiveDuration = uint32(l.config.MinPercentile) * l.config.UpdateInterval / 100
+	if l.config.SmoothIntervals > 0 {
+		// exponential moving average (EMA), same center of mass with simple moving average (SMA)
+		l.smoothFactor = float64(2) / (float64(l.config.SmoothIntervals + 1))
+	} else {
+		l.smoothFactor = 1
+	}
+	l.activeThreshold = ConvertAudioLevel(float64(l.config.ActiveLevel))
+}
+
+// Observes a new frame
+func (l *AudioLevel) Observe(level uint8, durationMs uint32, arrivalTime int64) {
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	l.observeLocked(level, durationMs, arrivalTime)
+}
+
+func (l *AudioLevel) observeLocked(level uint8, durationMs uint32, arrivalTime int64) {
+	if l.config.UpdateInterval == 0 {
+		return
+	}
+
+	l.lastObservedAt = arrivalTime
+
+	l.observedDuration += durationMs
+
+	if level <= l.config.ActiveLevel {
+		l.activeDuration += durationMs
+		if l.loudestObservedLevel > level {
+			l.loudestObservedLevel = level
+		}
+	}
+
+	if l.observedDuration >= l.config.UpdateInterval {
+		smoothedLevel := float64(0.0)
+		// compute and reset
+		if l.activeDuration >= l.minActiveDuration {
+			// adjust loudest observed level by how much of the window was active.
+			// Weight will be 0 if active the entire duration
+			// > 0 if active for longer than observe duration
+			// < 0 if active for less than observe duration
+			activityWeight := 20 * math.Log10(float64(l.activeDuration)/float64(l.config.UpdateInterval))
+			adjustedLevel := float64(l.loudestObservedLevel) - activityWeight
+			linearLevel := ConvertAudioLevel(adjustedLevel)
+
+			// exponential smoothing to dampen transients
+			smoothedLevel = l.smoothedLevel + (linearLevel-l.smoothedLevel)*l.smoothFactor
+		}
+		l.resetLocked(smoothedLevel)
+	}
+}
+
+func (l *AudioLevel) ObserveWithRTPTimestamp(level uint8, ts uint32, arrivalTime int64) {
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	if !l.highestRTPTimestampInitialized {
+		l.highestRTPTimestampInitialized = true
+		l.highestRTPTimestamp = ts
+	}
+
+	if (ts - l.highestRTPTimestamp) < (1 << 31) {
+		durationMs := (ts - l.highestRTPTimestamp) * 1e3 / l.params.ClockRate
+		l.observeLocked(level, durationMs, arrivalTime)
+
+		l.highestRTPTimestamp = ts
+	}
+}
+
+// returns current smoothed audio level
+func (l *AudioLevel) GetLevel(now int64) (float64, bool) {
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	if l.config.UpdateInterval == 0 {
+		return 0.0, false
+	}
+
+	l.resetIfStaleLocked(now)
+
+	return l.smoothedLevel, l.smoothedLevel >= l.activeThreshold
+}
+
+func (l *AudioLevel) resetIfStaleLocked(arrivalTime int64) {
+	if (arrivalTime-l.lastObservedAt)/1e6 < int64(2*l.config.UpdateInterval) {
+		return
+	}
+
+	l.resetLocked(0.0)
+}
+
+func (l *AudioLevel) resetLocked(smoothedLevel float64) {
+	l.smoothedLevel = smoothedLevel
+	l.loudestObservedLevel = silentAudioLevel
+	l.activeDuration = 0
+	l.observedDuration = 0
+}
+
+// ---------------------------------------------------
+
+// convert decibel back to linear
+func ConvertAudioLevel(level float64) float64 {
+	return math.Pow(10, level*negInv20)
+}

pkg/sfu/audio/audiolevel_test.go

@@ -0,0 +1,158 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package audio
+
+import (
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	samplesPerBatch    = 25
+	defaultActiveLevel = 30
+	// requires two noisy samples to count
+	defaultPercentile      = 10
+	defaultObserveDuration = 500 // ms
+)
+
+func TestAudioLevel(t *testing.T) {
+	t.Run("initially to return not noisy, within a few samples", func(t *testing.T) {
+		clock := time.Now()
+		a := createAudioLevel(defaultActiveLevel, defaultPercentile, defaultObserveDuration)
+
+		_, noisy := a.GetLevel(clock.UnixNano())
+		require.False(t, noisy)
+
+		observeSamples(a, 28, 5, clock)
+		clock = clock.Add(5 * 20 * time.Millisecond)
+
+		_, noisy = a.GetLevel(clock.UnixNano())
+		require.False(t, noisy)
+	})
+
+	t.Run("not noisy when all samples are below threshold", func(t *testing.T) {
+		clock := time.Now()
+		a := createAudioLevel(defaultActiveLevel, defaultPercentile, defaultObserveDuration)
+
+		observeSamples(a, 35, 100, clock)
+		clock = clock.Add(100 * 20 * time.Millisecond)
+
+		_, noisy := a.GetLevel(clock.UnixNano())
+		require.False(t, noisy)
+	})
+
+	t.Run("not noisy when less than percentile samples are above threshold", func(t *testing.T) {
+		clock := time.Now()
+		a := createAudioLevel(defaultActiveLevel, defaultPercentile, defaultObserveDuration)
+
+		observeSamples(a, 35, samplesPerBatch-2, clock)
+		clock = clock.Add((samplesPerBatch - 2) * 20 * time.Millisecond)
+		observeSamples(a, 25, 1, clock)
+		clock = clock.Add(20 * time.Millisecond)
+		observeSamples(a, 35, 1, clock)
+		clock = clock.Add(20 * time.Millisecond)
+
+		_, noisy := a.GetLevel(clock.UnixNano())
+		require.False(t, noisy)
+	})
+
+	t.Run("noisy when higher than percentile samples are above threshold", func(t *testing.T) {
+		clock := time.Now()
+		a := createAudioLevel(defaultActiveLevel, defaultPercentile, defaultObserveDuration)
+
+		observeSamples(a, 35, samplesPerBatch-16, clock)
+		clock = clock.Add((samplesPerBatch - 16) * 20 * time.Millisecond)
+		observeSamples(a, 25, 8, clock)
+		clock = clock.Add(8 * 20 * time.Millisecond)
+		observeSamples(a, 29, 8, clock)
+		clock = clock.Add(8 * 20 * time.Millisecond)
+
+		level, noisy := a.GetLevel(clock.UnixNano())
+		require.True(t, noisy)
+		require.Greater(t, level, ConvertAudioLevel(float64(defaultActiveLevel)))
+		require.Less(t, level, ConvertAudioLevel(float64(25)))
+	})
+
+	t.Run("not noisy when samples are stale", func(t *testing.T) {
+		clock := time.Now()
+		a := createAudioLevel(defaultActiveLevel, defaultPercentile, defaultObserveDuration)
+
+		observeSamples(a, 25, 100, clock)
+		clock = clock.Add(100 * 20 * time.Millisecond)
+		level, noisy := a.GetLevel(clock.UnixNano())
+		require.True(t, noisy)
+		require.Greater(t, level, ConvertAudioLevel(float64(defaultActiveLevel)))
+		require.Less(t, level, ConvertAudioLevel(float64(20)))
+
+		// let enough time pass to make the samples stale
+		clock = clock.Add(1500 * time.Millisecond)
+		level, noisy = a.GetLevel(clock.UnixNano())
+		require.Equal(t, float64(0.0), level)
+		require.False(t, noisy)
+	})
+
+	t.Run("not noisy when samples are stale - with RTP timestamp", func(t *testing.T) {
+		clock := time.Now()
+		a := createAudioLevel(defaultActiveLevel, defaultPercentile, defaultObserveDuration)
+
+		observeSamplesWithRTPTimestamp(a, 25, 100, clock)
+		clock = clock.Add(100 * 20 * time.Millisecond)
+		level, noisy := a.GetLevel(clock.UnixNano())
+		require.True(t, noisy)
+		require.Greater(t, level, ConvertAudioLevel(float64(defaultActiveLevel)))
+		require.Less(t, level, ConvertAudioLevel(float64(20)))
+
+		// let enough time pass to make the samples stale
+		clock = clock.Add(1500 * time.Millisecond)
+		level, noisy = a.GetLevel(clock.UnixNano())
+		require.Equal(t, float64(0.0), level)
+		require.False(t, noisy)
+	})
+}
+
+func createAudioLevel(activeLevel uint8, minPercentile uint8, observeDuration uint32) *AudioLevel {
+	al := NewAudioLevel(AudioLevelParams{
+		ClockRate: 48000,
+	})
+	al.SetConfig(AudioLevelConfig{
+		ActiveLevel:    activeLevel,
+		MinPercentile:  minPercentile,
+		UpdateInterval: observeDuration,
+	})
+	return al
+}
+
+func observeSamples(a *AudioLevel, level uint8, count int, baseTime time.Time) {
+	for i := range count {
+		a.Observe(level, 20, baseTime.Add(time.Duration(i*20)*time.Millisecond).UnixNano())
+	}
+}
+
+func observeSamplesWithRTPTimestamp(a *AudioLevel, level uint8, count int, baseTime time.Time) {
+	sampleTS := uint32(rand.Intn(1 << 20))
+	sampleTime := baseTime
+	for i := range count {
+		if (i % 5) == 0 {
+			// out-of-order sample
+			a.ObserveWithRTPTimestamp(level, sampleTS-1920, sampleTime.UnixNano())
+		}
+		a.ObserveWithRTPTimestamp(level, sampleTS, sampleTime.UnixNano())
+		sampleTS += 960 // 20 ms at 48 kHz
+		sampleTime = sampleTime.Add(20 * time.Millisecond)
+	}
+}

pkg/sfu/buffer/buffer.go

@@ -0,0 +1,540 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+
+	"github.com/pion/rtcp"
+	"github.com/pion/rtp"
+	"github.com/pion/webrtc/v4"
+
+	sutils "github.com/livekit/livekit-server/pkg/utils"
+	"github.com/livekit/mediatransportutil/pkg/bucket"
+	"github.com/livekit/mediatransportutil/pkg/twcc"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/utils/mono"
+)
+
+const (
+	rtcpReceiverReportDelta = 1e9
+
+	InitPacketBufferSizeVideo = 300
+	InitPacketBufferSizeAudio = 70
+)
+
+var (
+	errInvalidCodec = errors.New("invalid codec")
+)
+
+var _ BufferProvider = (*Buffer)(nil)
+
+type pendingPacket struct {
+	arrivalTime int64
+	packet      []byte
+}
+
+// Buffer contains all packets
+type Buffer struct {
+	*BufferBase
+
+	pPackets     []pendingPacket
+	lastReportAt int64
+	isBound      bool
+
+	twcc      *twcc.Responder
+	twccExtID uint8
+
+	enableAudioLossProxying  bool
+	lastFractionLostToReport uint8 // Last fraction lost from subscribers, should report to publisher; Audio only
+
+	lastPacketRead int
+
+	// callbacks
+	onClose         func()
+	onRtcpFeedback  func([]rtcp.Packet)
+	onFinalRtpStats func(*livekit.RTPStats)
+	onNotifyRTX     func(uint32, uint32, string)
+
+	primaryBufferForRTX *Buffer
+	rtxPktBuf           []byte
+
+	streamInfoProbe       *StreamInfoProbe
+	warnedPendingOverflow bool
+}
+
+// StreamInfoProbe identifies a stream from the mid/rid/rsid header extensions of its
+// packets. It runs on the write path, i. e. as SRTP pushes into this buffer, because
+// nothing reads remote streams through pion's interceptor chain.
+type StreamInfoProbe struct {
+	MidExtID  uint8
+	RidExtID  uint8
+	RsidExtID uint8
+
+	// Tries bounds how many packets are inspected before giving up.
+	Tries int
+
+	// OnFound is called at most once, in a goroutine, as it can re-enter this buffer.
+	OnFound func(ssrc uint32, mid, rid, rsid string)
+}
+
+func NewBuffer(ssrc uint32, maxVideoPkts, maxAudioPkts int) *Buffer {
+	b := &Buffer{}
+	b.BufferBase = NewBufferBase(BufferBaseParams{
+		SSRC:               ssrc,
+		MaxVideoPkts:       maxVideoPkts,
+		MaxAudioPkts:       maxAudioPkts,
+		LoggerComponents:   []string{sutils.ComponentPub, sutils.ComponentSFU},
+		SendPLI:            b.sendPLI,
+		IsReportingEnabled: true,
+	})
+	return b
+}
+
+func (b *Buffer) SetTWCCAndExtID(twcc *twcc.Responder, extID uint8) {
+	b.Lock()
+	defer b.Unlock()
+
+	b.twcc = twcc
+	b.twccExtID = extID
+}
+
+func (b *Buffer) SetAudioLossProxying(enable bool) {
+	b.Lock()
+	defer b.Unlock()
+
+	b.enableAudioLossProxying = enable
+}
+
+func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapability, bitrates int) error {
+	b.Lock()
+	if b.isBound {
+		b.Unlock()
+		return nil
+	}
+
+	if err := b.BufferBase.BindLocked(params, codec, bitrates); err != nil {
+		b.Unlock()
+		return err
+	}
+
+	b.lastReportAt = mono.UnixNano()
+
+	if len(b.pPackets) != 0 {
+		b.logger.Debugw("releasing queued packets on bind", "count", len(b.pPackets))
+	}
+	var rtcpPackets []rtcp.Packet
+	for _, pp := range b.pPackets {
+		rtcpPackets = append(rtcpPackets, b.calc(pp.packet, nil, pp.arrivalTime, true, false)...)
+	}
+	b.pPackets = nil
+
+	b.isBound = true
+	b.Unlock()
+
+	if len(rtcpPackets) != 0 {
+		if cb := b.getOnRtcpFeedback(); cb != nil {
+			cb(rtcpPackets)
+		}
+	}
+
+	return nil
+}
+
+// Write adds an RTP Packet, ordering is not guaranteed, newer packets may arrive later
+func (b *Buffer) Write(pkt []byte) (n int, err error) {
+	var rtpPacket rtp.Packet
+	err = rtpPacket.Unmarshal(pkt)
+	if err != nil {
+		return
+	}
+
+	b.Lock()
+	if b.BufferBase.IsClosed() {
+		b.Unlock()
+		err = io.EOF
+		return
+	}
+
+	now := mono.UnixNano()
+	if b.twcc != nil && b.twccExtID != 0 {
+		if ext := rtpPacket.GetExtension(b.twccExtID); len(ext) >= 2 {
+			b.twcc.Push(rtpPacket.SSRC, binary.BigEndian.Uint16(ext[0:2]), now, rtpPacket.Marker)
+		}
+	}
+
+	// libwebrtc will use 0 ssrc for probing, don't push the packet to pending queue to avoid memory increasing since
+	// the Bind will not be called to consume the pending packets. More details in https://github.com/pion/webrtc/pull/2816
+	if rtpPacket.SSRC == 0 {
+		b.Unlock()
+		return
+	}
+
+	if b.streamInfoProbe != nil {
+		b.probeStreamInfoLocked(&rtpPacket)
+	}
+
+	// handle RTX packet
+	if pb := b.primaryBufferForRTX; pb != nil {
+		b.Unlock()
+
+		// skip padding only packets
+		if rtpPacket.Padding && len(rtpPacket.Payload) == 0 {
+			return
+		}
+
+		pb.writeRTX(&rtpPacket, now)
+		return
+	}
+
+	if !b.isBound {
+		packet := make([]byte, len(pkt))
+		copy(packet, pkt)
+
+		if len(b.pPackets) == 0 {
+			b.logger.Debugw("received first packet")
+		}
+
+		startIdx := 0
+		overflow := len(b.pPackets) - max(b.BufferBase.MaxVideoPkts(), b.BufferBase.MaxAudioPkts())
+		if overflow > 0 {
+			startIdx = overflow
+
+			// a stream that keeps arriving but never binds drops every packet from here
+			// on; for an RTX stream it means the pairing was never established
+			if !b.warnedPendingOverflow {
+				b.warnedPendingOverflow = true
+				b.logger.Warnw(
+					"unbound buffer overflowing, dropping packets", nil,
+					"ssrc", b.BufferBase.SSRC(),
+					"pending", len(b.pPackets),
+				)
+			}
+		}
+		b.pPackets = append(b.pPackets[startIdx:], pendingPacket{
+			packet:      packet,
+			arrivalTime: now,
+		})
+
+		b.BufferBase.NotifyRead()
+		b.Unlock()
+		return
+	}
+
+	rtcpPackets := b.calc(pkt, &rtpPacket, now, false, false)
+	b.Unlock()
+
+	if len(rtcpPackets) != 0 {
+		if cb := b.getOnRtcpFeedback(); cb != nil {
+			cb(rtcpPackets)
+		}
+	}
+	return
+}
+
+// SetStreamInfoProbe installs probe and runs it over packets already queued.
+func (b *Buffer) SetStreamInfoProbe(probe *StreamInfoProbe) {
+	b.Lock()
+	defer b.Unlock()
+
+	b.streamInfoProbe = probe
+
+	for _, pp := range b.pPackets {
+		if b.streamInfoProbe == nil {
+			return
+		}
+
+		var rtpPacket rtp.Packet
+		if err := rtpPacket.Unmarshal(pp.packet); err != nil {
+			continue
+		}
+		b.probeStreamInfoLocked(&rtpPacket)
+	}
+}
+
+// probeStreamInfoLocked inspects one packet, clearing the probe once the stream is
+// identified or the try budget runs out.
+func (b *Buffer) probeStreamInfoLocked(rtpPacket *rtp.Packet) {
+	probe := b.streamInfoProbe
+
+	var mid, rid, rsid string
+	if ext := rtpPacket.GetExtension(probe.MidExtID); ext != nil {
+		mid = string(ext)
+	}
+	if ext := rtpPacket.GetExtension(probe.RidExtID); ext != nil {
+		rid = string(ext)
+	}
+	if ext := rtpPacket.GetExtension(probe.RsidExtID); ext != nil {
+		rsid = string(ext)
+	}
+
+	if mid != "" && (rid != "" || rsid != "") {
+		b.streamInfoProbe = nil
+		b.logger.Debugw("stream found", "ssrc", rtpPacket.SSRC, "mid", mid, "rid", rid, "rsid", rsid)
+		go probe.OnFound(rtpPacket.SSRC, mid, rid, rsid)
+		return
+	}
+
+	// ignore padding only packets for probe count
+	if rtpPacket.Padding && len(rtpPacket.Payload) == 0 {
+		return
+	}
+
+	if probe.Tries--; probe.Tries <= 0 {
+		b.streamInfoProbe = nil
+	}
+}
+
+func (b *Buffer) SetPrimaryBufferForRTX(primaryBuffer *Buffer) {
+	b.Lock()
+	b.primaryBufferForRTX = primaryBuffer
+	pkts := b.pPackets
+	b.pPackets = nil
+	b.Unlock()
+
+	for _, pp := range pkts {
+		var rtpPacket rtp.Packet
+		err := rtpPacket.Unmarshal(pp.packet)
+		if err != nil {
+			continue
+		}
+		if rtpPacket.Padding && len(rtpPacket.Payload) == 0 {
+			continue
+		}
+		primaryBuffer.writeRTX(&rtpPacket, pp.arrivalTime)
+	}
+}
+
+func (b *Buffer) NotifyRTX(ssrc uint32, repairSSRC uint32, rsid string) {
+	if onNotifyRTX := b.getOnNotifyRTX(); onNotifyRTX != nil {
+		onNotifyRTX(ssrc, repairSSRC, rsid)
+	}
+}
+
+func (b *Buffer) writeRTX(rtxPkt *rtp.Packet, arrivalTime int64) {
+	b.Lock()
+	defer b.Unlock()
+	if !b.isBound {
+		return
+	}
+
+	if rtxPkt.PayloadType != b.rtxPayloadType {
+		b.logger.Debugw("unexpected rtx payload type", "expected", b.rtxPayloadType, "actual", rtxPkt.PayloadType)
+		return
+	}
+
+	if b.rtxPktBuf == nil {
+		b.rtxPktBuf = make([]byte, bucket.RTPMaxPktSize)
+	}
+
+	if len(rtxPkt.Payload) < 2 {
+		b.logger.Warnw("rtx payload too short", nil, "size", len(rtxPkt.Payload))
+		return
+	}
+
+	repairedPkt := *rtxPkt
+	repairedPkt.PayloadType = b.payloadType
+	repairedPkt.SequenceNumber = binary.BigEndian.Uint16(rtxPkt.Payload[:2])
+	repairedPkt.SSRC = b.BufferBase.SSRC()
+	repairedPkt.Payload = rtxPkt.Payload[2:]
+	n, err := repairedPkt.MarshalTo(b.rtxPktBuf)
+	if err != nil {
+		b.logger.Errorw("could not marshal repaired packet", err, "ssrc", b.BufferBase.SSRC(), "sn", repairedPkt.SequenceNumber)
+		return
+	}
+
+	b.calc(b.rtxPktBuf[:n], &repairedPkt, arrivalTime, false, true)
+}
+
+func (b *Buffer) Read(buff []byte) (n int, err error) {
+	b.Lock()
+	for {
+		if b.BufferBase.IsClosed() {
+			b.Unlock()
+			return 0, io.EOF
+		}
+
+		if b.pPackets != nil && len(b.pPackets) > b.lastPacketRead {
+			if len(buff) < len(b.pPackets[b.lastPacketRead].packet) {
+				b.Unlock()
+				return 0, bucket.ErrBufferTooSmall
+			}
+
+			n = copy(buff, b.pPackets[b.lastPacketRead].packet)
+			b.lastPacketRead++
+			b.Unlock()
+			return
+		}
+		b.BufferBase.WaitRead()
+	}
+}
+
+func (b *Buffer) Close() error {
+	stats, err := b.BufferBase.CloseWithReason("close")
+	if err != nil {
+		return err
+	}
+
+	if stats != nil {
+		if cb := b.getOnFinalRtpStats(); cb != nil {
+			cb(stats)
+		}
+	}
+
+	if cb := b.getOnClose(); cb != nil {
+		cb()
+	}
+
+	return nil
+}
+
+func (b *Buffer) OnClose(fn func()) {
+	b.Lock()
+	b.onClose = fn
+	b.Unlock()
+}
+
+func (b *Buffer) getOnClose() func() {
+	b.RLock()
+	defer b.RUnlock()
+
+	return b.onClose
+}
+
+func (b *Buffer) sendPLI() {
+	ssrc := b.BufferBase.SSRC()
+	if ssrc == 0 {
+		return
+	}
+
+	b.logger.Debugw("send pli", "mediaSSRC", ssrc)
+	pli := []rtcp.Packet{
+		&rtcp.PictureLossIndication{
+			SenderSSRC: ssrc,
+			MediaSSRC:  ssrc,
+		},
+	}
+
+	if cb := b.getOnRtcpFeedback(); cb != nil {
+		cb(pli)
+	}
+}
+
+func (b *Buffer) calc(rawPkt []byte, rtpPacket *rtp.Packet, arrivalTime int64, isBuffered bool, isRTX bool) []rtcp.Packet {
+	b.BufferBase.HandleIncomingPacketLocked(
+		rawPkt,
+		rtpPacket,
+		arrivalTime,
+		isBuffered,
+		isRTX,
+		nil,
+		0,
+	)
+
+	return b.getRTCPPackets(arrivalTime)
+}
+
+func (b *Buffer) getRTCPPackets(arrivalTime int64) []rtcp.Packet {
+	var pkts []rtcp.Packet
+	if nackPkt := b.getNACKPacket(); nackPkt != nil {
+		pkts = append(pkts, nackPkt)
+	}
+	if receiverReport := b.getRTCPReceiverReport(arrivalTime); receiverReport != nil {
+		pkts = append(pkts, receiverReport)
+	}
+
+	return pkts
+}
+
+func (b *Buffer) getNACKPacket() *rtcp.TransportLayerNack {
+	if nacks := b.BufferBase.GetNACKPairsLocked(); len(nacks) > 0 {
+		ssrc := b.BufferBase.SSRC()
+		return &rtcp.TransportLayerNack{
+			SenderSSRC: ssrc,
+			MediaSSRC:  ssrc,
+			Nacks:      nacks,
+		}
+	}
+	return nil
+}
+
+func (b *Buffer) getRTCPReceiverReport(arrivalTime int64) *rtcp.ReceiverReport {
+	if arrivalTime-b.lastReportAt < rtcpReceiverReportDelta {
+		return nil
+	}
+	b.lastReportAt = arrivalTime
+
+	proxyLoss := b.lastFractionLostToReport
+	if b.codecType == webrtc.RTPCodecTypeAudio && !b.enableAudioLossProxying {
+		proxyLoss = 0
+	}
+
+	if receptionReport := b.BufferBase.GetRtcpReceptionReportLocked(proxyLoss); receptionReport != nil {
+		return &rtcp.ReceiverReport{
+			SSRC:    b.BufferBase.SSRC(),
+			Reports: []rtcp.ReceptionReport{*receptionReport},
+		}
+	}
+
+	return nil
+}
+
+func (b *Buffer) SetLastFractionLostReport(lost uint8) {
+	b.Lock()
+	defer b.Unlock()
+
+	b.lastFractionLostToReport = lost
+}
+
+func (b *Buffer) OnRtcpFeedback(fn func(fb []rtcp.Packet)) {
+	b.Lock()
+	b.onRtcpFeedback = fn
+	b.Unlock()
+}
+
+func (b *Buffer) getOnRtcpFeedback() func(fb []rtcp.Packet) {
+	b.RLock()
+	defer b.RUnlock()
+
+	return b.onRtcpFeedback
+}
+
+func (b *Buffer) OnFinalRtpStats(fn func(*livekit.RTPStats)) {
+	b.Lock()
+	b.onFinalRtpStats = fn
+	b.Unlock()
+}
+
+func (b *Buffer) getOnFinalRtpStats() func(*livekit.RTPStats) {
+	b.RLock()
+	defer b.RUnlock()
+
+	return b.onFinalRtpStats
+}
+
+func (b *Buffer) OnNotifyRTX(fn func(ssrc uint32, repairSSRC uint32, rsid string)) {
+	b.Lock()
+	b.onNotifyRTX = fn
+	b.Unlock()
+}
+
+func (b *Buffer) getOnNotifyRTX() func(ssrc uint32, repairSSRC uint32, rsid string) {
+	b.RLock()
+	defer b.RUnlock()
+
+	return b.onNotifyRTX
+}

pkg/sfu/buffer/buffer_test.go

@@ -0,0 +1,479 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"fmt"
+	"math"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/pion/rtcp"
+	"github.com/pion/rtp"
+	"github.com/pion/webrtc/v4"
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/mediatransportutil/pkg/nack"
+)
+
+var h265Codec = webrtc.RTPCodecParameters{
+	RTPCodecCapability: webrtc.RTPCodecCapability{
+		MimeType:  "video/h265",
+		ClockRate: 90000,
+		RTCPFeedback: []webrtc.RTCPFeedback{{
+			Type: "nack",
+		}},
+	},
+	PayloadType: 116,
+}
+
+var vp8Codec = webrtc.RTPCodecParameters{
+	RTPCodecCapability: webrtc.RTPCodecCapability{
+		MimeType:  "video/vp8",
+		ClockRate: 90000,
+		RTCPFeedback: []webrtc.RTCPFeedback{{
+			Type: "nack",
+		}},
+	},
+	PayloadType: 96,
+}
+
+var opusCodec = webrtc.RTPCodecParameters{
+	RTPCodecCapability: webrtc.RTPCodecCapability{
+		MimeType:  "audio/opus",
+		ClockRate: 48000,
+	},
+	PayloadType: 111,
+}
+
+func TestNack(t *testing.T) {
+	t.Run("nack normal", func(t *testing.T) {
+		buff := NewBuffer(123, 1, 1)
+		buff.codecType = webrtc.RTPCodecTypeVideo
+		require.NotNil(t, buff)
+		var wg sync.WaitGroup
+		// 5 tries
+		wg.Add(5)
+		buff.OnRtcpFeedback(func(fb []rtcp.Packet) {
+			for _, pkt := range fb {
+				switch p := pkt.(type) {
+				case *rtcp.TransportLayerNack:
+					if p.Nacks[0].PacketList()[0] == 1 && p.MediaSSRC == 123 {
+						wg.Done()
+					}
+				}
+			}
+		})
+		buff.Bind(webrtc.RTPParameters{
+			HeaderExtensions: nil,
+			Codecs:           []webrtc.RTPCodecParameters{vp8Codec},
+		}, vp8Codec.RTPCodecCapability, 0)
+		rtt := uint32(20)
+		buff.nacker.SetRTT(rtt)
+		for i := range 15 {
+			if i == 1 {
+				continue
+			}
+			if i < 14 {
+				time.Sleep(time.Duration(float64(rtt)*math.Pow(nack.NackQueueParamsDefault.BackoffFactor, float64(i))+10) * time.Millisecond)
+			} else {
+				time.Sleep(500 * time.Millisecond) // even a long wait should not exceed max retries
+			}
+			pkt := rtp.Packet{
+				Header: rtp.Header{
+					Version:        2,
+					PayloadType:    96,
+					SequenceNumber: uint16(i),
+					Timestamp:      uint32(i),
+					SSRC:           123,
+				},
+				Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
+			}
+			b, err := pkt.Marshal()
+			require.NoError(t, err)
+			_, err = buff.Write(b)
+			require.NoError(t, err)
+		}
+		wg.Wait()
+
+	})
+
+	t.Run("nack with seq wrap", func(t *testing.T) {
+		buff := NewBuffer(123, 1, 1)
+		buff.codecType = webrtc.RTPCodecTypeVideo
+		require.NotNil(t, buff)
+		var wg sync.WaitGroup
+		expects := map[uint16]int{
+			65534: 0,
+			65535: 0,
+			0:     0,
+			1:     0,
+		}
+		wg.Add(5 * len(expects)) // retry 5 times
+		buff.OnRtcpFeedback(func(fb []rtcp.Packet) {
+			for _, pkt := range fb {
+				switch p := pkt.(type) {
+				case *rtcp.TransportLayerNack:
+					if p.MediaSSRC == 123 {
+						for _, v := range p.Nacks {
+							v.Range(func(seq uint16) bool {
+								if _, ok := expects[seq]; ok {
+									wg.Done()
+								} else {
+									require.Fail(t, "unexpected nack seq ", seq)
+								}
+								return true
+							})
+						}
+					}
+				}
+			}
+		})
+		buff.Bind(webrtc.RTPParameters{
+			HeaderExtensions: nil,
+			Codecs:           []webrtc.RTPCodecParameters{vp8Codec},
+		}, vp8Codec.RTPCodecCapability, 0)
+		rtt := uint32(30)
+		buff.nacker.SetRTT(rtt)
+		for i := range 15 {
+			if i > 0 && i < 5 {
+				continue
+			}
+			if i < 14 {
+				time.Sleep(time.Duration(float64(rtt)*math.Pow(nack.NackQueueParamsDefault.BackoffFactor, float64(i))+10) * time.Millisecond)
+			} else {
+				time.Sleep(500 * time.Millisecond) // even a long wait should not exceed max retries
+			}
+			pkt := rtp.Packet{
+				Header: rtp.Header{
+					Version:        2,
+					PayloadType:    96,
+					SequenceNumber: uint16(i + 65533),
+					Timestamp:      uint32(i),
+					SSRC:           123,
+				},
+				Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
+			}
+			b, err := pkt.Marshal()
+			require.NoError(t, err)
+			_, err = buff.Write(b)
+			require.NoError(t, err)
+		}
+		wg.Wait()
+
+	})
+}
+
+func TestNewBuffer(t *testing.T) {
+	tests := []struct {
+		name string
+	}{
+		{
+			name: "Must not be nil and add packets in sequence",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var TestPackets = []*rtp.Packet{
+				{
+					Header: rtp.Header{
+						Version:        2,
+						PayloadType:    96,
+						SequenceNumber: 65533,
+						SSRC:           123,
+					},
+				},
+				{
+					Header: rtp.Header{
+						Version:        2,
+						PayloadType:    96,
+						SequenceNumber: 65534,
+						SSRC:           123,
+					},
+					Payload: []byte{1},
+				},
+				{
+					Header: rtp.Header{
+						Version:        2,
+						PayloadType:    96,
+						SequenceNumber: 2,
+						SSRC:           123,
+					},
+				},
+				{
+					Header: rtp.Header{
+						Version:        2,
+						PayloadType:    96,
+						SequenceNumber: 65535,
+						SSRC:           123,
+					},
+				},
+			}
+			buff := NewBuffer(123, 1, 1)
+			buff.codecType = webrtc.RTPCodecTypeVideo
+			require.NotNil(t, buff)
+			buff.OnRtcpFeedback(func(_ []rtcp.Packet) {})
+			buff.Bind(webrtc.RTPParameters{
+				HeaderExtensions: nil,
+				Codecs:           []webrtc.RTPCodecParameters{vp8Codec},
+			}, vp8Codec.RTPCodecCapability, 0)
+
+			for _, p := range TestPackets {
+				buf, _ := p.Marshal()
+				_, _ = buff.Write(buf)
+			}
+			require.Equal(t, uint16(2), buff.rtpStats.HighestSequenceNumber())
+			require.Equal(t, uint64(65536+2), buff.rtpStats.ExtendedHighestSequenceNumber())
+		})
+	}
+}
+
+func TestFractionLostReport(t *testing.T) {
+	buff := NewBuffer(123, 1, 1)
+	require.NotNil(t, buff)
+
+	var wg sync.WaitGroup
+
+	// with loss proxying
+	wg.Add(1)
+	buff.SetAudioLossProxying(true)
+	buff.SetLastFractionLostReport(55)
+	buff.OnRtcpFeedback(func(fb []rtcp.Packet) {
+		for _, pkt := range fb {
+			switch p := pkt.(type) {
+			case *rtcp.ReceiverReport:
+				for _, v := range p.Reports {
+					require.EqualValues(t, 55, v.FractionLost)
+				}
+				wg.Done()
+			}
+		}
+	})
+	buff.Bind(webrtc.RTPParameters{
+		HeaderExtensions: nil,
+		Codecs:           []webrtc.RTPCodecParameters{opusCodec},
+	}, opusCodec.RTPCodecCapability, 0)
+	for i := range 15 {
+		pkt := rtp.Packet{
+			Header: rtp.Header{
+				Version:        2,
+				PayloadType:    111,
+				SequenceNumber: uint16(i),
+				Timestamp:      uint32(i),
+				SSRC:           123,
+			},
+			Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
+		}
+		b, err := pkt.Marshal()
+		require.NoError(t, err)
+		if i == 1 {
+			time.Sleep(1 * time.Second)
+		}
+		_, err = buff.Write(b)
+		require.NoError(t, err)
+	}
+	wg.Wait()
+
+	wg.Add(1)
+	buff.SetAudioLossProxying(false)
+	buff.OnRtcpFeedback(func(fb []rtcp.Packet) {
+		for _, pkt := range fb {
+			switch p := pkt.(type) {
+			case *rtcp.ReceiverReport:
+				for _, v := range p.Reports {
+					require.EqualValues(t, 0, v.FractionLost)
+				}
+				wg.Done()
+			}
+		}
+	})
+	buff.Bind(webrtc.RTPParameters{
+		HeaderExtensions: nil,
+		Codecs:           []webrtc.RTPCodecParameters{opusCodec},
+	}, opusCodec.RTPCodecCapability, 0)
+	for i := range 15 {
+		pkt := rtp.Packet{
+			Header: rtp.Header{
+				Version:        2,
+				PayloadType:    111,
+				SequenceNumber: uint16(i),
+				Timestamp:      uint32(i),
+				SSRC:           123,
+			},
+			Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
+		}
+		b, err := pkt.Marshal()
+		require.NoError(t, err)
+		if i == 1 {
+			time.Sleep(1 * time.Second)
+		}
+		_, err = buff.Write(b)
+		require.NoError(t, err)
+	}
+	wg.Wait()
+}
+
+func TestCodecChange(t *testing.T) {
+	// codec change before bind
+	buff := NewBuffer(123, 1, 1)
+	require.NotNil(t, buff)
+	changedCodec := make(chan webrtc.RTPCodecParameters, 1)
+	buff.OnCodecChange(func(rp webrtc.RTPCodecParameters) {
+		select {
+		case changedCodec <- rp:
+		default:
+			t.Fatalf("codec change not consumed")
+		}
+	})
+	buff.OnStreamRestart(func(reason string) {
+		require.Equal(t, "codec-change", reason)
+
+		// read once to clear pending restart
+		var buf [1500]byte
+		extPkt, err := buff.ReadExtended(buf[:])
+		require.NoError(t, err)
+		require.Nil(t, extPkt)
+	})
+
+	h265Pkt := rtp.Packet{
+		Header: rtp.Header{
+			Version:        2,
+			PayloadType:    116,
+			SequenceNumber: 1,
+			Timestamp:      1,
+			SSRC:           123,
+		},
+		Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
+	}
+	buf, err := h265Pkt.Marshal()
+	require.NoError(t, err)
+	_, err = buff.Write(buf)
+	require.NoError(t, err)
+
+	select {
+	case <-changedCodec:
+		t.Fatalf("unexpected codec change")
+	case <-time.After(100 * time.Millisecond):
+	}
+
+	// Bind sets up VP8 as expected codec,
+	// packet written to the buffer above before Bind is H.265,
+	// that should trigger a codec change and a stream restart
+	// when the queued packets from Write before Bind get flushed
+	buff.Bind(
+		webrtc.RTPParameters{
+			HeaderExtensions: nil,
+			Codecs:           []webrtc.RTPCodecParameters{vp8Codec, h265Codec},
+		},
+		vp8Codec.RTPCodecCapability,
+		0,
+	)
+
+	select {
+	case c := <-changedCodec:
+		require.Equal(t, h265Codec, c)
+	case <-time.After(1 * time.Second):
+		t.Fatalf("expected codec change")
+	}
+
+	// second codec change - writing VP8 packet after Bind should trigger another codec change
+	vp8Pkt := rtp.Packet{
+		Header: rtp.Header{
+			Version:        2,
+			PayloadType:    96,
+			SequenceNumber: 3,
+			Timestamp:      3,
+			SSRC:           123,
+		},
+		Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
+	}
+	buf, err = vp8Pkt.Marshal()
+	require.NoError(t, err)
+	_, err = buff.Write(buf)
+	require.NoError(t, err)
+
+	select {
+	case c := <-changedCodec:
+		require.Equal(t, vp8Codec, c)
+	case <-time.After(1 * time.Second):
+		t.Fatalf("expected codec change")
+	}
+	fmt.Printf("done second codec change\n") // REMOVE
+
+	// out of order pkts can't cause codec change
+	// rewrite the VP8 packet to start the sequence after a stream restart
+	_, err = buff.Write(buf)
+	require.NoError(t, err)
+
+	h265Pkt.SequenceNumber = 2
+	h265Pkt.Timestamp = 2
+	buf, err = h265Pkt.Marshal()
+	require.NoError(t, err)
+	_, err = buff.Write(buf)
+	require.NoError(t, err)
+	select {
+	case <-changedCodec:
+		t.Fatalf("unexpected codec change")
+	case <-time.After(100 * time.Millisecond):
+	}
+
+	// unknown codec should not cause change even if it is in order
+	h265Pkt.SequenceNumber = 4
+	h265Pkt.Timestamp = 4
+	h265Pkt.PayloadType = 117
+	buf, err = h265Pkt.Marshal()
+	require.NoError(t, err)
+	_, err = buff.Write(buf)
+	require.NoError(t, err)
+	select {
+	case <-changedCodec:
+		t.Fatalf("unexpected codec change")
+	case <-time.After(100 * time.Millisecond):
+	}
+
+	// an in-order packet should change codec again
+	h265Pkt.SequenceNumber = 5
+	h265Pkt.Timestamp = 5
+	h265Pkt.PayloadType = 116
+	buf, err = h265Pkt.Marshal()
+	require.NoError(t, err)
+	_, err = buff.Write(buf)
+	require.NoError(t, err)
+	select {
+	case c := <-changedCodec:
+		require.Equal(t, h265Codec, c)
+	case <-time.After(1 * time.Second):
+		t.Fatalf("expected codec change")
+	}
+}
+
+func BenchmarkMemcpu(b *testing.B) {
+	buf := make([]byte, 1500*1500*10)
+	buf2 := make([]byte, 1500*1500*20)
+
+	for b.Loop() {
+		copy(buf2, buf)
+	}
+}
+
+func BenchmarkExtPacketFactory(b *testing.B) {
+
+	for b.Loop() {
+		extPkt := ExtPacketFactory.Get().(*ExtPacket)
+		*extPkt = ExtPacket{}
+		ExtPacketFactory.Put(extPkt)
+	}
+}

pkg/sfu/buffer/datastats.go

@@ -0,0 +1,102 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"sync"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	"github.com/livekit/protocol/livekit"
+)
+
+type DataStatsParam struct {
+	WindowDuration time.Duration
+}
+
+type DataStats struct {
+	params      DataStatsParam
+	lock        sync.RWMutex
+	totalBytes  int64
+	startTime   time.Time
+	endTime     time.Time
+	windowStart int64
+	windowBytes int64
+}
+
+func NewDataStats(params DataStatsParam) *DataStats {
+	return &DataStats{
+		params:      params,
+		startTime:   time.Now(),
+		windowStart: time.Now().UnixNano(),
+	}
+}
+
+func (s *DataStats) Update(bytes int, time int64) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	s.totalBytes += int64(bytes)
+
+	if s.params.WindowDuration > 0 && time-s.windowStart > s.params.WindowDuration.Nanoseconds() {
+		s.windowBytes = 0
+		s.windowStart = time
+	}
+	s.windowBytes += int64(bytes)
+}
+
+func (s *DataStats) ToProtoActive() *livekit.RTPStats {
+	if s.params.WindowDuration == 0 {
+		return &livekit.RTPStats{}
+	}
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+	now := time.Now().UnixNano()
+	duration := now - s.windowStart
+	if duration > s.params.WindowDuration.Nanoseconds() {
+		return &livekit.RTPStats{}
+	}
+
+	return &livekit.RTPStats{
+		StartTime: timestamppb.New(time.Unix(s.windowStart/1e9, s.windowStart%1e9)),
+		EndTime:   timestamppb.New(time.Unix(0, now)),
+		Duration:  float64(duration / 1e9),
+		Bytes:     uint64(s.windowBytes),
+		Bitrate:   float64(s.windowBytes) * 8 / float64(duration) / 1e9,
+	}
+}
+
+func (s *DataStats) Stop() {
+	s.lock.Lock()
+	s.endTime = time.Now()
+	s.lock.Unlock()
+}
+
+func (s *DataStats) ToProtoAggregateOnly() *livekit.RTPStats {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+
+	end := s.endTime
+	if end.IsZero() {
+		end = time.Now()
+	}
+	return &livekit.RTPStats{
+		StartTime: timestamppb.New(s.startTime),
+		EndTime:   timestamppb.New(end),
+		Duration:  end.Sub(s.startTime).Seconds(),
+		Bytes:     uint64(s.totalBytes),
+		Bitrate:   float64(s.totalBytes) * 8 / end.Sub(s.startTime).Seconds(),
+	}
+}

pkg/sfu/buffer/datastats_test.go

@@ -0,0 +1,53 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/livekit/protocol/livekit"
+)
+
+func TestDataStats(t *testing.T) {
+	stats := NewDataStats(DataStatsParam{WindowDuration: time.Second})
+
+	time.Sleep(time.Millisecond)
+	r := stats.ToProtoAggregateOnly()
+	require.Equal(t, r.StartTime.AsTime().UnixNano(), stats.startTime.UnixNano())
+	require.NotZero(t, r.EndTime)
+	require.NotZero(t, r.Duration)
+	r.StartTime = nil
+	r.EndTime = nil
+	r.Duration = 0
+	require.True(t, proto.Equal(r, &livekit.RTPStats{}))
+
+	stats.Update(100, time.Now().UnixNano())
+	r = stats.ToProtoActive()
+	require.EqualValues(t, 100, r.Bytes)
+	require.NotZero(t, r.Bitrate)
+
+	// wait for window duration
+	time.Sleep(time.Second)
+	r = stats.ToProtoActive()
+	require.True(t, proto.Equal(r, &livekit.RTPStats{}))
+	stats.Stop()
+	r = stats.ToProtoAggregateOnly()
+	require.EqualValues(t, 100, r.Bytes)
+	require.NotZero(t, r.Bitrate)
+}

pkg/sfu/buffer/dependencydescriptorparser.go

@@ -0,0 +1,364 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"fmt"
+	"slices"
+	"sync"
+	"time"
+
+	"github.com/pion/rtp"
+	"go.uber.org/atomic"
+
+	dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
+	"github.com/livekit/mediatransportutil/pkg/codec"
+	"github.com/livekit/mediatransportutil/pkg/utils"
+	"github.com/livekit/protocol/logger"
+)
+
+var (
+	ExtDependencyDescriptorFactory = &sync.Pool{
+		New: func() any {
+			return &ExtDependencyDescriptor{}
+		},
+	}
+)
+
+// --------------------------------------
+
+const (
+	ddRestartThreshold = 30 * time.Second
+
+	// frame integrity check 2 seconds for L3T3 30fps video
+	integrityCheckFrame = 180
+	integrityCheckPkt   = 1024
+)
+
+var (
+	ErrFrameEarlierThanKeyFrame            = fmt.Errorf("frame is earlier than current keyframe")
+	ErrDDStructureAttachedToNonFirstPacket = fmt.Errorf("dependency descriptor structure is attached to non-first packet of a frame")
+	ErrDDExtentionNotFound                 = fmt.Errorf("dependency descriptor extension not found")
+)
+
+type DependencyDescriptorParser struct {
+	structure         *dd.FrameDependencyStructure
+	ddExtID           uint8
+	logger            logger.Logger
+	onMaxLayerChanged func(int32, int32)
+	decodeTargets     []DependencyDescriptorDecodeTarget
+
+	seqWrapAround        *utils.WrapAround[uint16, uint64]
+	frameWrapAround      *utils.WrapAround[uint16, uint64]
+	structureExtFrameNum uint64
+	// drop threshold for frames belonging to a previous dependency structure.
+	// advanced only when the structure id actually changes: structureExtFrameNum
+	// advances on every structure-bearing key frame, so with frequent key frames
+	// repeating the same structure (e. g. screen content), valid late/retransmitted
+	// frames would be dropped as "earlier than current structure".
+	structureChangeExtFrameNum uint64
+	activeDecodeTargetsExtSeq  uint64
+	activeDecodeTargetsMask    uint32
+	frameChecker               *FrameIntegrityChecker
+
+	ddNotFoundCount atomic.Uint32
+
+	// restart detection
+	restartGeneration int
+	enableRestart     bool
+	lastPacketAt      time.Time
+}
+
+func NewDependencyDescriptorParser(ddExtID uint8, logger logger.Logger, onMaxLayerChanged func(int32, int32), enableRestart bool) *DependencyDescriptorParser {
+	return &DependencyDescriptorParser{
+		ddExtID:           ddExtID,
+		logger:            logger,
+		onMaxLayerChanged: onMaxLayerChanged,
+		seqWrapAround:     utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
+		frameWrapAround:   utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
+		frameChecker:      NewFrameIntegrityChecker(integrityCheckFrame, integrityCheckPkt),
+		enableRestart:     enableRestart,
+	}
+}
+
+type ExtDependencyDescriptor struct {
+	Descriptor *dd.DependencyDescriptor
+
+	DecodeTargets              []DependencyDescriptorDecodeTarget
+	StructureUpdated           bool
+	ActiveDecodeTargetsUpdated bool
+	Integrity                  bool
+	ExtFrameNum                uint64
+	// the frame number of the keyframe which the current frame depends on
+	ExtKeyFrameNum uint64
+
+	// increase when the stream restarts, clear and reinitialize all dd state includes
+	// attached structure, frame chain, decode target.
+	RestartGeneration int
+}
+
+func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescriptor, VideoLayer, error) {
+	var videoLayer VideoLayer
+	ddBuf := pkt.GetExtension(r.ddExtID)
+	if ddBuf == nil {
+		ddNotFoundCount := r.ddNotFoundCount.Inc()
+		if ddNotFoundCount%100 == 0 {
+			r.logger.Warnw("dependency descriptor extension is not present", nil, "seq", pkt.SequenceNumber, "count", ddNotFoundCount)
+		}
+		return nil, videoLayer, ErrDDExtentionNotFound
+	}
+
+	var ddVal dd.DependencyDescriptor
+	ext := &dd.DependencyDescriptorExtension{
+		Descriptor: &ddVal,
+		Structure:  r.structure,
+	}
+	_, err := ext.Unmarshal(ddBuf)
+	if err != nil {
+		if err != dd.ErrDDReaderNoStructure && err != dd.ErrDDReaderInvalidTemplateIndex {
+			r.logger.Infow("failed to parse generic dependency descriptor", err, "payload", pkt.PayloadType, "ddbufLen", len(ddBuf))
+		}
+		return nil, videoLayer, err
+	}
+
+	var restart bool
+	if r.enableRestart {
+		if !r.lastPacketAt.IsZero() && time.Since(r.lastPacketAt) > ddRestartThreshold {
+			r.restart()
+			restart = true
+			r.logger.Debugw(
+				"dependency descriptor parser restart stream",
+				"generation", r.restartGeneration,
+				"lastPacketAt", r.lastPacketAt,
+				"sinceLast", time.Since(r.lastPacketAt),
+				"frameWrapAround", r.frameWrapAround,
+			)
+		}
+		r.lastPacketAt = time.Now()
+	}
+
+	extSeq := r.seqWrapAround.Update(pkt.SequenceNumber).ExtendedVal
+
+	if ddVal.FrameDependencies != nil {
+		videoLayer.Spatial, videoLayer.Temporal = int32(ddVal.FrameDependencies.SpatialId), int32(ddVal.FrameDependencies.TemporalId)
+	}
+
+	// assume the packet is in-order when stream restarting
+	unwrapped := r.frameWrapAround.UpdateWithOrderKnown(ddVal.FrameNumber, restart)
+	extFN := unwrapped.ExtendedVal
+
+	if extFN < r.structureChangeExtFrameNum {
+		r.logger.Debugw(
+			"drop frame which is earlier than current structure",
+			"fn", ddVal.FrameNumber,
+			"extFN", extFN,
+			"structureChangeExtFrameNum", r.structureChangeExtFrameNum,
+			"unwrappedFN", unwrapped,
+			"frameWrapAround", r.frameWrapAround,
+		)
+		return nil, videoLayer, ErrFrameEarlierThanKeyFrame
+	}
+
+	r.frameChecker.AddPacket(extSeq, extFN, &ddVal)
+
+	extDD := ExtDependencyDescriptorFactory.Get().(*ExtDependencyDescriptor)
+	*extDD = ExtDependencyDescriptor{
+		Descriptor:        &ddVal,
+		ExtFrameNum:       extFN,
+		Integrity:         r.frameChecker.FrameIntegrity(extFN),
+		RestartGeneration: r.restartGeneration,
+	}
+
+	if ddVal.AttachedStructure != nil {
+		if !ddVal.FirstPacketInFrame {
+			r.logger.Warnw(
+				"attached structure is not the first packet in frame", nil,
+				"sn", pkt.SequenceNumber,
+				"extSeq", extSeq,
+				"fn", ddVal.FrameNumber,
+				"extFN", extFN,
+			)
+			ReleaseExtDependencyDescriptor(extDD)
+			return nil, videoLayer, ErrDDStructureAttachedToNonFirstPacket
+		}
+
+		if extFN < r.structureExtFrameNum {
+			// out-of-order key frame repeating the current structure: accepting it
+			// would regress structureExtFrameNum (ExtKeyFrameNum) and replay a stale
+			// structure update, confusing the downtrack's dependency descriptor
+			// selector.
+			r.logger.Debugw(
+				"drop out-of-order key frame",
+				"extFN", extFN,
+				"structureExtFrameNum", r.structureExtFrameNum,
+			)
+			ReleaseExtDependencyDescriptor(extDD)
+			return nil, videoLayer, ErrFrameEarlierThanKeyFrame
+		}
+
+		if r.structure == nil || ddVal.AttachedStructure.StructureId != r.structure.StructureId {
+			// structure actually changed (or first structure): advance the drop threshold
+			// so that only frames preceding this structure are dropped.
+			r.structureChangeExtFrameNum = extFN
+			r.logger.Debugw(
+				"structure updated",
+				"structureID", ddVal.AttachedStructure.StructureId,
+				"sn", pkt.SequenceNumber,
+				"extSeq", extSeq,
+				"fn", ddVal.FrameNumber,
+				"extFN", extFN,
+				"descriptor", ddVal.String(),
+				"unwrappedFN", unwrapped,
+				"frameWrapAround", r.frameWrapAround,
+			)
+		}
+		r.structure = ddVal.AttachedStructure
+		r.decodeTargets = ProcessFrameDependencyStructure(ddVal.AttachedStructure)
+		if extFN > unwrapped.PreExtendedHighest && extFN-unwrapped.PreExtendedHighest > 1000 {
+			r.logger.Debugw(
+				"large frame number jump on structure updating",
+				"fn", ddVal.FrameNumber,
+				"extFN", extFN,
+				"preExtendedHighest", unwrapped.PreExtendedHighest,
+				"structureExtFrameNum", r.structureExtFrameNum,
+				"unwrappedFN", unwrapped,
+				"frameWrapAround", r.frameWrapAround,
+			)
+		}
+		r.structureExtFrameNum = extFN
+		extDD.StructureUpdated = true
+		extDD.ActiveDecodeTargetsUpdated = true
+		// The dependency descriptor reader will always set ActiveDecodeTargetsBitmask for TemplateDependencyStructure is present,
+		// so don't need to notify max layer change here.
+	}
+
+	if mask := ddVal.ActiveDecodeTargetsBitmask; mask != nil && extSeq > r.activeDecodeTargetsExtSeq {
+		r.activeDecodeTargetsExtSeq = extSeq
+		if *mask != r.activeDecodeTargetsMask {
+			r.activeDecodeTargetsMask = *mask
+			extDD.ActiveDecodeTargetsUpdated = true
+			var maxSpatial, maxTemporal int32
+			for _, dt := range r.decodeTargets {
+				if *mask&(1<<dt.Target) != uint32(dd.DecodeTargetNotPresent) {
+					if maxSpatial < dt.Layer.Spatial {
+						maxSpatial = dt.Layer.Spatial
+					}
+					if maxTemporal < dt.Layer.Temporal {
+						maxTemporal = dt.Layer.Temporal
+					}
+				}
+			}
+			r.logger.Debugw("max layer changed", "maxSpatial", maxSpatial, "maxTemporal", maxTemporal)
+			r.onMaxLayerChanged(maxSpatial, maxTemporal)
+		}
+	}
+
+	extDD.DecodeTargets = r.decodeTargets
+	extDD.ExtKeyFrameNum = r.structureExtFrameNum
+
+	return extDD, videoLayer, nil
+}
+
+func (r *DependencyDescriptorParser) restart() {
+	r.frameChecker = NewFrameIntegrityChecker(integrityCheckFrame, integrityCheckPkt)
+	r.structure = nil
+	r.structureExtFrameNum = 0
+	r.structureChangeExtFrameNum = 0
+	r.activeDecodeTargetsExtSeq = 0
+	r.activeDecodeTargetsMask = 0
+	r.decodeTargets = r.decodeTargets[:0]
+	r.restartGeneration++
+}
+
+// ------------------------------------------------------------------------------
+
+type DependencyDescriptorDecodeTarget struct {
+	Target int
+	Layer  VideoLayer
+}
+
+func (dt *DependencyDescriptorDecodeTarget) String() string {
+	return fmt.Sprintf("DecodeTarget{t: %d, l: %+v}", dt.Target, dt.Layer)
+}
+
+// ------------------------------------------------------------------------------
+
+func ProcessFrameDependencyStructure(structure *dd.FrameDependencyStructure) []DependencyDescriptorDecodeTarget {
+	decodeTargets := make([]DependencyDescriptorDecodeTarget, 0, structure.NumDecodeTargets)
+	for target := 0; target < structure.NumDecodeTargets; target++ {
+		layer := VideoLayer{Spatial: 0, Temporal: 0}
+		for _, t := range structure.Templates {
+			if t.DecodeTargetIndications[target] != dd.DecodeTargetNotPresent {
+				if layer.Spatial < int32(t.SpatialId) {
+					layer.Spatial = int32(t.SpatialId)
+				}
+				if layer.Temporal < int32(t.TemporalId) {
+					layer.Temporal = int32(t.TemporalId)
+				}
+			}
+		}
+		decodeTargets = append(decodeTargets, DependencyDescriptorDecodeTarget{target, layer})
+	}
+
+	// sort decode target layer by spatial and temporal from high to low
+	slices.SortFunc(decodeTargets, func(a, b DependencyDescriptorDecodeTarget) int {
+		if a.Layer.GreaterThan(b.Layer) {
+			return -1
+		}
+		if b.Layer.GreaterThan(a.Layer) {
+			return 1
+		}
+		return 0
+	})
+
+	return decodeTargets
+}
+
+func GetActiveDecodeTargetBitmask(layer VideoLayer, decodeTargets []DependencyDescriptorDecodeTarget) *uint32 {
+	activeBitMask := uint32(0)
+	for _, dt := range decodeTargets {
+		if dt.Layer.Spatial <= layer.Spatial && dt.Layer.Temporal <= layer.Temporal {
+			activeBitMask |= 1 << dt.Target
+		}
+	}
+
+	return &activeBitMask
+}
+
+// ------------------------------------------------------------------------------
+
+func ExtractDependencyDescriptorVideoSize(dd *dd.DependencyDescriptor) []codec.VideoSize {
+	if dd.AttachedStructure == nil {
+		return nil
+	}
+
+	videoSizes := make([]codec.VideoSize, 0, len(dd.AttachedStructure.Resolutions))
+	for _, res := range dd.AttachedStructure.Resolutions {
+		videoSizes = append(videoSizes, codec.VideoSize{Width: uint32(res.Width), Height: uint32(res.Height)})
+	}
+
+	return videoSizes
+}
+
+// ------------------------------------------------------------------------------
+
+func ReleaseExtDependencyDescriptor(extDD *ExtDependencyDescriptor) {
+	if extDD == nil {
+		return
+	}
+
+	*extDD = ExtDependencyDescriptor{}
+	ExtDependencyDescriptorFactory.Put(extDD)
+}

pkg/sfu/buffer/dependencydescriptorparser_test.go

@@ -0,0 +1,188 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"testing"
+
+	"github.com/pion/rtp"
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/protocol/logger"
+
+	dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
+)
+
+const ddTestExtID = uint8(1)
+
+// single spatial layer, single temporal layer, one decode target, one chain
+func newL1T1Structure(structureID int) *dd.FrameDependencyStructure {
+	return &dd.FrameDependencyStructure{
+		StructureId:                  structureID,
+		NumDecodeTargets:             1,
+		NumChains:                    1,
+		DecodeTargetProtectedByChain: []int{0},
+		Templates: []*dd.FrameDependencyTemplate{
+			{ // key frame
+				DecodeTargetIndications: []dd.DecodeTargetIndication{dd.DecodeTargetSwitch},
+				ChainDiffs:              []int{0},
+			},
+			{ // delta frame
+				DecodeTargetIndications: []dd.DecodeTargetIndication{dd.DecodeTargetRequired},
+				FrameDiffs:              []int{1},
+				ChainDiffs:              []int{1},
+			},
+		},
+	}
+}
+
+type ddTestFeeder struct {
+	t      *testing.T
+	parser *DependencyDescriptorParser
+	// structure the writer marshals against, i. e. what the publisher last sent
+	structure *dd.FrameDependencyStructure
+	seq       uint16
+}
+
+func newDDTestFeeder(t *testing.T) *ddTestFeeder {
+	return &ddTestFeeder{
+		t:      t,
+		parser: NewDependencyDescriptorParser(ddTestExtID, logger.GetLogger(), func(int32, int32) {}, false),
+	}
+}
+
+// keyFrame sends a key frame carrying `structure`. Passing the same structure id as
+// the previous key frame models a publisher that repeats an unchanged structure,
+// which is what screen content with frequent key frames does.
+func (f *ddTestFeeder) keyFrame(frameNumber uint16, structure *dd.FrameDependencyStructure) (*ExtDependencyDescriptor, error) {
+	f.structure = structure
+	return f.feed(frameNumber, structure.Templates[0], structure)
+}
+
+func (f *ddTestFeeder) deltaFrame(frameNumber uint16) (*ExtDependencyDescriptor, error) {
+	return f.feed(frameNumber, f.structure.Templates[1], nil)
+}
+
+func (f *ddTestFeeder) feed(
+	frameNumber uint16,
+	template *dd.FrameDependencyTemplate,
+	attachedStructure *dd.FrameDependencyStructure,
+) (*ExtDependencyDescriptor, error) {
+	f.t.Helper()
+
+	ddVal := &dd.DependencyDescriptor{
+		FirstPacketInFrame: true,
+		LastPacketInFrame:  true,
+		FrameNumber:        frameNumber,
+		FrameDependencies:  template,
+		AttachedStructure:  attachedStructure,
+	}
+	buf, err := (&dd.DependencyDescriptorExtension{Descriptor: ddVal, Structure: f.structure}).Marshal()
+	require.NoError(f.t, err)
+
+	f.seq++
+	pkt := &rtp.Packet{Header: rtp.Header{SequenceNumber: f.seq}}
+	require.NoError(f.t, pkt.SetExtension(ddTestExtID, buf))
+
+	extDD, _, err := f.parser.Parse(pkt)
+	return extDD, err
+}
+
+// A key frame that repeats the current structure must not start dropping frames that
+// precede it. Upstream advanced the drop threshold on every structure bearing key
+// frame, so with frequent key frames (screen content) a late or retransmitted frame
+// arriving after one was discarded as "earlier than current structure".
+func TestDependencyDescriptorParserLateFrameAfterRepeatedStructure(t *testing.T) {
+	f := newDDTestFeeder(t)
+	structure := newL1T1Structure(0)
+
+	_, err := f.keyFrame(0, structure)
+	require.NoError(t, err)
+	_, err = f.deltaFrame(1)
+	require.NoError(t, err)
+	_, err = f.deltaFrame(3)
+	require.NoError(t, err)
+
+	// key frame repeating the same structure id
+	_, err = f.keyFrame(4, structure)
+	require.NoError(t, err)
+
+	// frame 2 finally shows up (reordered or retransmitted). The structure has not
+	// changed, so it is still decodable and must be forwarded.
+	extDD, err := f.deltaFrame(2)
+	require.NoError(t, err)
+	require.NotNil(t, extDD)
+	require.EqualValues(t, 2, extDD.ExtFrameNum)
+}
+
+// A key frame carrying a *different* structure does invalidate everything before it:
+// earlier frames reference templates that no longer exist.
+func TestDependencyDescriptorParserLateFrameAfterStructureChange(t *testing.T) {
+	f := newDDTestFeeder(t)
+
+	_, err := f.keyFrame(0, newL1T1Structure(0))
+	require.NoError(t, err)
+	_, err = f.deltaFrame(1)
+	require.NoError(t, err)
+	_, err = f.deltaFrame(3)
+	require.NoError(t, err)
+
+	_, err = f.keyFrame(4, newL1T1Structure(1))
+	require.NoError(t, err)
+
+	_, err = f.deltaFrame(2)
+	require.ErrorIs(t, err, ErrFrameEarlierThanKeyFrame)
+}
+
+// An out-of-order key frame must still be dropped: accepting it would regress
+// structureExtFrameNum (ExtKeyFrameNum) and replay a stale structure update.
+func TestDependencyDescriptorParserOutOfOrderKeyFrame(t *testing.T) {
+	f := newDDTestFeeder(t)
+	structure := newL1T1Structure(0)
+
+	_, err := f.keyFrame(0, structure)
+	require.NoError(t, err)
+	_, err = f.deltaFrame(1)
+	require.NoError(t, err)
+	_, err = f.keyFrame(4, structure)
+	require.NoError(t, err)
+
+	_, err = f.keyFrame(2, structure)
+	require.ErrorIs(t, err, ErrFrameEarlierThanKeyFrame)
+}
+
+// ExtKeyFrameNum keeps tracking every structure bearing key frame, unchanged by the
+// split of the drop threshold into its own field.
+func TestDependencyDescriptorParserExtKeyFrameNum(t *testing.T) {
+	f := newDDTestFeeder(t)
+	structure := newL1T1Structure(0)
+
+	extDD, err := f.keyFrame(0, structure)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, extDD.ExtKeyFrameNum)
+
+	extDD, err = f.deltaFrame(1)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, extDD.ExtKeyFrameNum)
+
+	// repeated structure still advances ExtKeyFrameNum
+	extDD, err = f.keyFrame(4, structure)
+	require.NoError(t, err)
+	require.EqualValues(t, 4, extDD.ExtKeyFrameNum)
+
+	extDD, err = f.deltaFrame(5)
+	require.NoError(t, err)
+	require.EqualValues(t, 4, extDD.ExtKeyFrameNum)
+}

pkg/sfu/buffer/factory.go

@@ -0,0 +1,149 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"io"
+	"sync"
+
+	"github.com/pion/transport/v4/packetio"
+)
+
+type FactoryOfBufferFactory struct {
+	trackingPacketsVideo int
+	trackingPacketsAudio int
+}
+
+func NewFactoryOfBufferFactory(trackingPacketsVideo int, trackingPacketsAudio int) *FactoryOfBufferFactory {
+	return &FactoryOfBufferFactory{
+		trackingPacketsVideo: trackingPacketsVideo,
+		trackingPacketsAudio: trackingPacketsAudio,
+	}
+}
+
+func (f *FactoryOfBufferFactory) CreateBufferFactory() *Factory {
+	return &Factory{
+		trackingPacketsVideo: f.trackingPacketsVideo,
+		trackingPacketsAudio: f.trackingPacketsAudio,
+		rtpBuffers:           make(map[uint32]*Buffer),
+		rtcpReaders:          make(map[uint32]*RTCPReader),
+		rtxPair:              make(map[uint32]uint32),
+	}
+}
+
+type Factory struct {
+	sync.RWMutex
+	trackingPacketsVideo int
+	trackingPacketsAudio int
+	rtpBuffers           map[uint32]*Buffer
+	rtcpReaders          map[uint32]*RTCPReader
+	rtxPair              map[uint32]uint32 // repair -> base
+}
+
+func (f *Factory) GetOrNew(packetType packetio.BufferPacketType, ssrc uint32) io.ReadWriteCloser {
+	f.Lock()
+	defer f.Unlock()
+	switch packetType {
+	case packetio.RTCPBufferPacket:
+		if reader, ok := f.rtcpReaders[ssrc]; ok {
+			return reader
+		}
+		reader := NewRTCPReader(ssrc)
+		f.rtcpReaders[ssrc] = reader
+		reader.OnClose(func() {
+			f.Lock()
+			delete(f.rtcpReaders, ssrc)
+			f.Unlock()
+		})
+		return reader
+	case packetio.RTPBufferPacket:
+		if reader, ok := f.rtpBuffers[ssrc]; ok {
+			return reader
+		}
+		buffer := NewBuffer(ssrc, f.trackingPacketsVideo, f.trackingPacketsAudio)
+		f.rtpBuffers[ssrc] = buffer
+		for repair, base := range f.rtxPair {
+			if repair == ssrc {
+				baseBuffer, ok := f.rtpBuffers[base]
+				if ok {
+					buffer.SetPrimaryBufferForRTX(baseBuffer)
+				}
+				break
+			} else if base == ssrc {
+				repairBuffer, ok := f.rtpBuffers[repair]
+				if ok {
+					repairBuffer.SetPrimaryBufferForRTX(buffer)
+				}
+				break
+			}
+		}
+		buffer.OnClose(func() {
+			f.Lock()
+			delete(f.rtpBuffers, ssrc)
+			delete(f.rtxPair, ssrc)
+			f.Unlock()
+		})
+		return buffer
+	}
+	return nil
+}
+
+func (f *Factory) GetBufferPair(ssrc uint32) (*Buffer, *RTCPReader) {
+	f.RLock()
+	defer f.RUnlock()
+	return f.rtpBuffers[ssrc], f.rtcpReaders[ssrc]
+}
+
+func (f *Factory) GetBuffer(ssrc uint32) *Buffer {
+	f.RLock()
+	defer f.RUnlock()
+	return f.rtpBuffers[ssrc]
+}
+
+func (f *Factory) GetRTCPReader(ssrc uint32) *RTCPReader {
+	f.RLock()
+	defer f.RUnlock()
+	return f.rtcpReaders[ssrc]
+}
+
+// SetStreamInfoProbe installs probe on the buffer of ssrc, reporting whether that
+// buffer exists. False means the stream can never be identified.
+func (f *Factory) SetStreamInfoProbe(ssrc uint32, probe *StreamInfoProbe) bool {
+	f.RLock()
+	buffer := f.rtpBuffers[ssrc]
+	f.RUnlock()
+
+	if buffer == nil {
+		return false
+	}
+
+	buffer.SetStreamInfoProbe(probe)
+	return true
+}
+
+func (f *Factory) SetRTXPair(repair, base uint32, rsid string) {
+	f.Lock()
+	repairBuffer, baseBuffer := f.rtpBuffers[repair], f.rtpBuffers[base]
+	if repairBuffer == nil || baseBuffer == nil {
+		f.rtxPair[repair] = base
+	}
+	f.Unlock()
+	if repairBuffer != nil && baseBuffer != nil {
+		repairBuffer.SetPrimaryBufferForRTX(baseBuffer)
+		if rsid != "" {
+			baseBuffer.NotifyRTX(base, repair, rsid)
+		}
+	}
+}

pkg/sfu/buffer/fps.go

@@ -0,0 +1,750 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"container/list"
+
+	"github.com/pion/rtp/codecs"
+
+	"github.com/livekit/mediatransportutil/pkg/codec"
+	"github.com/livekit/protocol/logger"
+)
+
+var minFramesForCalculation = [...]int{8, 15, 40, 60}
+
+type frameInfo struct {
+	startSeq  uint16
+	endSeq    uint16
+	ts        uint32
+	fn        uint16
+	spatial   int32
+	temporal  int32
+	frameDiff []int
+}
+
+type FrameRateCalculator interface {
+	RecvPacket(ep *ExtPacket) bool
+	GetFrameRate() []float32
+	Completed() bool
+}
+
+// -----------------------------
+
+// FrameRateCalculator based on PictureID in VPx
+type frameRateCalculatorVPx struct {
+	frameRates   [DefaultMaxLayerTemporal + 1]float32
+	clockRate    uint32
+	logger       logger.Logger
+	firstFrames  [DefaultMaxLayerTemporal + 1]*frameInfo
+	secondFrames [DefaultMaxLayerTemporal + 1]*frameInfo
+	fnReceived   [64]*frameInfo
+	baseFrame    *frameInfo
+	completed    bool
+}
+
+func newFrameRateCalculatorVPx(clockRate uint32, logger logger.Logger) *frameRateCalculatorVPx {
+	return &frameRateCalculatorVPx{
+		clockRate: clockRate,
+		logger:    logger,
+	}
+}
+
+func (f *frameRateCalculatorVPx) Completed() bool {
+	return f.completed
+}
+
+func (f *frameRateCalculatorVPx) RecvPacket(ep *ExtPacket, fn uint16) bool {
+	if f.completed {
+		return true
+	}
+
+	if ep.Temporal >= int32(len(f.frameRates)) {
+		f.logger.Warnw("invalid temporal layer", nil, "temporal", ep.Temporal)
+		return false
+	}
+
+	temporal := ep.Temporal
+	if temporal < 0 {
+		temporal = 0
+	}
+
+	if f.baseFrame == nil {
+		f.baseFrame = &frameInfo{ts: ep.Packet.Timestamp, fn: fn}
+		f.fnReceived[0] = f.baseFrame
+		f.firstFrames[temporal] = f.baseFrame
+		return false
+	}
+
+	baseDiff := fn - f.baseFrame.fn
+	if baseDiff == 0 || baseDiff > 0x4000 {
+		return false
+	}
+
+	if baseDiff >= uint16(len(f.fnReceived)) {
+		// frame number is not continuous, reset
+		f.reset()
+
+		return false
+	}
+
+	if f.fnReceived[baseDiff] != nil {
+		return false
+	}
+
+	fi := &frameInfo{
+		ts:       ep.Packet.Timestamp,
+		fn:       fn,
+		temporal: temporal,
+	}
+	f.fnReceived[baseDiff] = fi
+
+	firstFrame := f.firstFrames[temporal]
+	secondFrame := f.secondFrames[temporal]
+	if firstFrame == nil {
+		f.firstFrames[temporal] = fi
+		firstFrame = fi
+	} else {
+		if (secondFrame == nil || secondFrame.fn < fn) && fn != firstFrame.fn && (fn-firstFrame.fn) < 0x4000 {
+			f.secondFrames[temporal] = fi
+		}
+	}
+
+	return f.calc()
+}
+
+func (f *frameRateCalculatorVPx) calc() bool {
+	var rateCounter int
+	for currentTemporal := int32(0); currentTemporal <= DefaultMaxLayerTemporal; currentTemporal++ {
+		if f.frameRates[currentTemporal] > 0 {
+			rateCounter++
+			continue
+		}
+
+		ff := f.firstFrames[currentTemporal]
+		sf := f.secondFrames[currentTemporal]
+		// lower temporal layer has been calculated, but higher layer has not received any frames, it should not exist
+		if rateCounter > 0 && ff == nil {
+			rateCounter++
+			continue
+		}
+		if ff == nil || sf == nil {
+			continue
+		}
+
+		var frameCount int
+		lastTs := ff.ts
+		for j := ff.fn - f.baseFrame.fn + 1; j < sf.fn-f.baseFrame.fn+1; j++ {
+			if f := f.fnReceived[j]; f == nil {
+				break
+			} else if f.temporal <= currentTemporal {
+				frameCount++
+				lastTs = f.ts
+			}
+		}
+		if frameCount >= minFramesForCalculation[currentTemporal] {
+			f.frameRates[currentTemporal] = float32(f.clockRate) / float32(lastTs-ff.ts) * float32(frameCount)
+			rateCounter++
+		}
+	}
+
+	if rateCounter == len(f.frameRates) {
+		f.completed = true
+
+		// normalize frame rates, Microsoft Edge use 3 temporal layers for vp8 but the middle layer has chance to
+		// get a very low frame rate, so we need to normalize the frame rate(use fixed ration 1:2 of highest layer for that layer)
+		if f.frameRates[2] > 0 && f.frameRates[2] > f.frameRates[1]*3 {
+			f.frameRates[1] = f.frameRates[2] / 2
+		}
+		f.reset()
+		return true
+	}
+	return false
+}
+
+func (f *frameRateCalculatorVPx) reset() {
+	for i := range f.firstFrames {
+		f.firstFrames[i] = nil
+		f.secondFrames[i] = nil
+	}
+
+	for i := range f.fnReceived {
+		f.fnReceived[i] = nil
+	}
+	f.baseFrame = nil
+}
+
+func (f *frameRateCalculatorVPx) GetFrameRate() []float32 {
+	return f.frameRates[:]
+}
+
+// -----------------------------
+
+// FrameRateCalculator based on PictureID in VP8
+type FrameRateCalculatorVP8 struct {
+	*frameRateCalculatorVPx
+	logger logger.Logger
+}
+
+func NewFrameRateCalculatorVP8(clockRate uint32, logger logger.Logger) *FrameRateCalculatorVP8 {
+	return &FrameRateCalculatorVP8{
+		frameRateCalculatorVPx: newFrameRateCalculatorVPx(clockRate, logger),
+		logger:                 logger,
+	}
+}
+
+func (f *FrameRateCalculatorVP8) RecvPacket(ep *ExtPacket) bool {
+	if f.frameRateCalculatorVPx.Completed() {
+		return true
+	}
+
+	vp8, ok := ep.Payload.(codec.VP8)
+	if !ok {
+		f.logger.Debugw("no vp8 payload", "sn", ep.Packet.SequenceNumber)
+		return false
+	}
+	if !vp8.I {
+		f.logger.Debugw("no vp8 pictureID", "sn", ep.Packet.SequenceNumber)
+		return false
+	}
+	success := f.frameRateCalculatorVPx.RecvPacket(ep, vp8.PictureID)
+
+	if f.frameRateCalculatorVPx.Completed() {
+		f.logger.Debugw("frame rate calculated", "rate", f.frameRateCalculatorVPx.GetFrameRate())
+	}
+
+	return success
+}
+
+// -----------------------------
+
+// FrameRateCalculator based on PictureID in VP9
+type FrameRateCalculatorVP9 struct {
+	logger    logger.Logger
+	completed bool
+
+	// VP9-TODO - this is assuming three spatial layers. As `completed` marker relies on all layers being finished, have to assume this. FIX.
+	//            Maybe look at number of layers in livekit.TrackInfo and declare completed once advertised layers are measured
+	frameRateCalculatorsVPx [DefaultMaxLayerSpatial + 1]*frameRateCalculatorVPx
+}
+
+func NewFrameRateCalculatorVP9(clockRate uint32, logger logger.Logger) *FrameRateCalculatorVP9 {
+	f := &FrameRateCalculatorVP9{
+		logger: logger,
+	}
+
+	for i := range f.frameRateCalculatorsVPx {
+		f.frameRateCalculatorsVPx[i] = newFrameRateCalculatorVPx(clockRate, logger)
+	}
+
+	return f
+}
+
+func (f *FrameRateCalculatorVP9) Completed() bool {
+	return f.completed
+}
+
+func (f *FrameRateCalculatorVP9) RecvPacket(ep *ExtPacket) bool {
+	if f.completed {
+		return true
+	}
+
+	vp9, ok := ep.Payload.(codecs.VP9Packet)
+	if !ok {
+		f.logger.Debugw("no vp9 payload", "sn", ep.Packet.SequenceNumber)
+		return false
+	}
+	if !vp9.I {
+		f.logger.Debugw("no vp9 pictureID", "sn", ep.Packet.SequenceNumber)
+		return false
+	}
+	if ep.Spatial < 0 || ep.Spatial >= int32(len(f.frameRateCalculatorsVPx)) || f.frameRateCalculatorsVPx[ep.Spatial] == nil {
+		f.logger.Debugw("invalid spatial layer", "sn", ep.Packet.SequenceNumber, "spatial", ep.Spatial)
+		return false
+	}
+
+	success := f.frameRateCalculatorsVPx[ep.Spatial].RecvPacket(ep, vp9.PictureID)
+
+	completed := true
+	for _, frc := range f.frameRateCalculatorsVPx {
+		if !frc.Completed() {
+			completed = false
+			break
+		}
+	}
+
+	if completed {
+		f.completed = true
+
+		var frameRates [DefaultMaxLayerSpatial + 1][]float32
+		for i := range f.frameRateCalculatorsVPx {
+			frameRates[i] = f.frameRateCalculatorsVPx[i].GetFrameRate()
+		}
+		f.logger.Debugw("frame rate calculated", "rate", frameRates)
+	}
+
+	return success
+}
+
+func (f *FrameRateCalculatorVP9) GetFrameRateForSpatial(spatial int32) []float32 {
+	if spatial < 0 || spatial >= int32(len(f.frameRateCalculatorsVPx)) || f.frameRateCalculatorsVPx[spatial] == nil {
+		return nil
+	}
+	return f.frameRateCalculatorsVPx[spatial].GetFrameRate()
+}
+
+func (f *FrameRateCalculatorVP9) GetFrameRateCalculatorForSpatial(spatial int32) *FrameRateCalculatorForVP9Layer {
+	return &FrameRateCalculatorForVP9Layer{
+		FrameRateCalculatorVP9: f,
+		spatial:                spatial,
+	}
+}
+
+// -----------------------------
+
+type FrameRateCalculatorForVP9Layer struct {
+	*FrameRateCalculatorVP9
+	spatial int32
+}
+
+func (f *FrameRateCalculatorForVP9Layer) GetFrameRate() []float32 {
+	return f.FrameRateCalculatorVP9.GetFrameRateForSpatial(f.spatial)
+}
+
+// -----------------------------------------------
+
+// FrameRateCalculator based on Dependency descriptor
+type FrameRateCalculatorDD struct {
+	frameRates   [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]float32
+	clockRate    uint32
+	logger       logger.Logger
+	firstFrames  [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]*frameInfo
+	secondFrames [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]*frameInfo
+	fnReceived   [256]*frameInfo
+	baseFrame    *frameInfo
+	completed    bool
+
+	// frames for each decode target
+	targetFrames [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]list.List
+
+	maxSpatial, maxTemporal int32
+}
+
+func NewFrameRateCalculatorDD(clockRate uint32, logger logger.Logger) *FrameRateCalculatorDD {
+	return &FrameRateCalculatorDD{
+		clockRate:   clockRate,
+		logger:      logger,
+		maxSpatial:  DefaultMaxLayerSpatial,
+		maxTemporal: DefaultMaxLayerTemporal,
+	}
+}
+
+func (f *FrameRateCalculatorDD) Completed() bool {
+	return f.completed
+}
+
+func (f *FrameRateCalculatorDD) SetMaxLayer(spatial, temporal int32) {
+	if spatial <= DefaultMaxLayerSpatial {
+		f.maxSpatial = spatial
+	} else {
+		f.logger.Warnw("max spatial layer out of range", nil, "spatial", spatial)
+	}
+
+	if temporal <= DefaultMaxLayerTemporal {
+		f.maxTemporal = temporal
+	} else {
+		f.logger.Warnw("max temporal layer out of range", nil, "temporal", temporal)
+	}
+}
+
+func (f *FrameRateCalculatorDD) RecvPacket(ep *ExtPacket) bool {
+	if f.completed {
+		return true
+	}
+
+	if ep.DependencyDescriptor == nil {
+		f.logger.Debugw("dependency descriptor is nil")
+		return false
+	}
+
+	spatial := ep.Spatial
+	// non-SVC codec will set spatial to -1
+	if spatial < 0 {
+		spatial = 0
+	}
+	temporal := ep.Temporal
+	if temporal < 0 || temporal > DefaultMaxLayerTemporal || spatial > DefaultMaxLayerSpatial {
+		f.logger.Warnw("invalid spatial or temporal", nil, "spatial", spatial, "temporal", temporal, "sn", ep.Packet.SequenceNumber)
+		return false
+	}
+
+	fn := ep.DependencyDescriptor.Descriptor.FrameNumber
+	if f.baseFrame == nil {
+		f.baseFrame = &frameInfo{ts: ep.Packet.Timestamp, fn: fn}
+		f.fnReceived[0] = f.baseFrame
+		f.firstFrames[spatial][temporal] = f.baseFrame
+		f.secondFrames[spatial][temporal] = f.baseFrame
+		return false
+	}
+
+	baseDiff := fn - f.baseFrame.fn
+	if baseDiff == 0 || baseDiff > 0x8000 {
+		return false
+	}
+
+	if baseDiff >= uint16(len(f.fnReceived)) {
+		// frame number is not continuous, reset
+		f.baseFrame = nil
+		for i := range f.firstFrames {
+			for j := range f.firstFrames[i] {
+				f.firstFrames[i][j] = nil
+				f.secondFrames[i][j] = nil
+				f.targetFrames[i][j].Init()
+			}
+		}
+		for i := range f.fnReceived {
+			f.fnReceived[i] = nil
+		}
+		return false
+	}
+
+	if f.fnReceived[baseDiff] != nil {
+		return false
+	}
+
+	fi := &frameInfo{
+		ts:        ep.Packet.Timestamp,
+		fn:        fn,
+		temporal:  temporal,
+		spatial:   spatial,
+		frameDiff: ep.DependencyDescriptor.Descriptor.FrameDependencies.FrameDiffs,
+	}
+	f.fnReceived[baseDiff] = fi
+
+	if f.firstFrames[spatial][temporal] == nil {
+		f.firstFrames[spatial][temporal] = fi
+		f.secondFrames[spatial][temporal] = fi
+		return false
+	}
+
+	chain := &f.targetFrames[spatial][temporal]
+	if chain.Len() == 0 {
+		chain.PushBack(fn)
+	}
+	for _, fdiff := range ep.DependencyDescriptor.Descriptor.FrameDependencies.FrameDiffs {
+		dependFrame := fn - uint16(fdiff)
+		// frame too old, ignore
+		if dependFrame-f.secondFrames[spatial][temporal].fn > 0x8000 {
+			continue
+		}
+
+	insertFrame:
+		for e := chain.Back(); e != nil; e = e.Prev() {
+			val := e.Value.(uint16)
+			switch {
+			case val == dependFrame:
+				break insertFrame
+			case sn16LT(val, dependFrame):
+				chain.InsertAfter(dependFrame, e)
+				break insertFrame
+			default:
+				if e == chain.Front() {
+					chain.PushFront(dependFrame)
+					break insertFrame
+				}
+			}
+		}
+	}
+	return f.calc()
+}
+
+func (f *FrameRateCalculatorDD) calc() bool {
+	var rateCounter int
+	for currentSpatial := int32(0); currentSpatial <= f.maxSpatial; currentSpatial++ {
+		var currentSpatialRateCounter int
+		for currentTemporal := int32(0); currentTemporal <= f.maxTemporal; currentTemporal++ {
+			if f.frameRates[currentSpatial][currentTemporal] > 0 {
+				rateCounter++
+				currentSpatialRateCounter++
+				continue
+			}
+
+			firstFrame := f.firstFrames[currentSpatial][currentTemporal]
+			// lower temporal layer has been calculated, but higher layer has not received any frames, it should not exist
+			if currentSpatialRateCounter > 0 && firstFrame == nil {
+				currentSpatialRateCounter++
+				rateCounter++
+				continue
+			}
+
+			chain := &f.targetFrames[currentSpatial][currentTemporal]
+
+			// find last decodable frame (no dependency frame is lost)
+			var lastFrame *frameInfo
+			for e := chain.Front(); e != nil; e = e.Next() {
+				diff := e.Value.(uint16) - f.baseFrame.fn
+				if diff >= uint16(len(f.fnReceived)) {
+					continue
+				}
+
+				fi := f.fnReceived[diff]
+				if fi == nil {
+					break
+				} else {
+					lastFrame = fi
+					if firstFrame == nil && fi.spatial == currentSpatial && fi.temporal == currentTemporal {
+						firstFrame = fi
+					}
+				}
+			}
+
+			if lastFrame != nil && lastFrame.fn > f.secondFrames[currentSpatial][currentTemporal].fn {
+				f.secondFrames[currentSpatial][currentTemporal] = lastFrame
+			} else {
+				continue
+			}
+
+			frameCount := 0
+			for i := firstFrame.fn - f.baseFrame.fn; i <= lastFrame.fn-f.baseFrame.fn; i++ {
+				fi := f.fnReceived[i]
+				if fi == nil {
+					continue
+				}
+				if fi.spatial == currentSpatial && fi.temporal <= currentTemporal {
+					frameCount++
+				}
+			}
+
+			if frameCount >= minFramesForCalculation[currentTemporal] && lastFrame.ts > firstFrame.ts {
+				f.frameRates[currentSpatial][currentTemporal] = float32(f.clockRate) / float32(lastFrame.ts-firstFrame.ts) * float32(frameCount)
+				rateCounter++
+			}
+		}
+	}
+
+	if rateCounter == int(f.maxSpatial+1)*int(f.maxTemporal+1) {
+		f.completed = true
+		f.close()
+
+		f.logger.Debugw("frame rate calculated", "rate", f.frameRates)
+		return true
+	}
+	return false
+}
+
+func (f *FrameRateCalculatorDD) GetFrameRateForSpatial(spatial int32) []float32 {
+	if spatial < 0 || spatial >= int32(len(f.frameRates)) {
+		return nil
+	}
+	return f.frameRates[spatial][:]
+}
+
+func (f *FrameRateCalculatorDD) close() {
+	f.baseFrame = nil
+	for i := range f.firstFrames {
+		for j := range f.firstFrames[i] {
+			f.firstFrames[i][j] = nil
+			f.secondFrames[i][j] = nil
+		}
+	}
+
+	for i := range f.fnReceived {
+		f.fnReceived[i] = nil
+	}
+	for i := range f.targetFrames {
+		for j := range f.targetFrames[i] {
+			f.targetFrames[i][j].Init()
+		}
+	}
+}
+
+func (f *FrameRateCalculatorDD) GetFrameRateCalculatorForSpatial(spatial int32) *FrameRateCalculatorForDDLayer {
+	return &FrameRateCalculatorForDDLayer{
+		FrameRateCalculatorDD: f,
+		spatial:               spatial,
+	}
+}
+
+// -----------------------------------------------
+
+type FrameRateCalculatorForDDLayer struct {
+	*FrameRateCalculatorDD
+	spatial int32
+}
+
+func (f *FrameRateCalculatorForDDLayer) GetFrameRate() []float32 {
+	return f.FrameRateCalculatorDD.GetFrameRateForSpatial(f.spatial)
+}
+
+// -----------------------------------------------
+
+type FrameRateCalculatorH26x struct {
+	frameRates [DefaultMaxLayerTemporal + 1]float32
+	clockRate  uint32
+	logger     logger.Logger
+	fnReceived *list.List
+	baseFrame  *frameInfo
+	completed  bool
+}
+
+func NewFrameRateCalculatorH26x(clockRate uint32, logger logger.Logger) *FrameRateCalculatorH26x {
+	return &FrameRateCalculatorH26x{
+		clockRate: clockRate,
+		logger:    logger,
+	}
+}
+
+func (f *FrameRateCalculatorH26x) Completed() bool {
+	return f.completed
+}
+
+func (f *FrameRateCalculatorH26x) RecvPacket(ep *ExtPacket) bool {
+	if f.completed {
+		return true
+	}
+
+	if ep.Temporal >= int32(len(f.frameRates)) {
+		f.logger.Warnw("invalid temporal layer", nil, "temporal", ep.Temporal)
+		return false
+	}
+
+	temporal := ep.Temporal
+	if temporal < 0 {
+		temporal = 0
+	}
+
+	if f.baseFrame == nil {
+		f.baseFrame = &frameInfo{
+			startSeq: ep.Packet.SequenceNumber,
+			endSeq:   ep.Packet.SequenceNumber,
+			ts:       ep.Packet.Timestamp,
+			temporal: temporal,
+		}
+		f.fnReceived = list.New()
+		f.fnReceived.PushBack(f.baseFrame)
+		return false
+	}
+
+	if sn16LTOrEqual(ep.Packet.SequenceNumber, f.baseFrame.startSeq) {
+		return false
+	}
+
+insertFrame:
+	for e := f.fnReceived.Back(); e != nil; e = e.Prev() {
+		frame := e.Value.(*frameInfo)
+		switch {
+		case frame.ts == ep.Packet.Timestamp:
+			if sn16LT(frame.endSeq, ep.Packet.SequenceNumber) {
+				frame.endSeq = ep.Packet.SequenceNumber
+			}
+			if sn16LT(ep.Packet.SequenceNumber, frame.startSeq) {
+				frame.startSeq = ep.Packet.SequenceNumber
+			}
+			break insertFrame
+		case sn32LT(frame.ts, ep.Packet.Timestamp):
+			f.fnReceived.InsertAfter(&frameInfo{
+				startSeq: ep.Packet.SequenceNumber,
+				endSeq:   ep.Packet.SequenceNumber,
+				ts:       ep.Packet.Timestamp,
+				temporal: temporal,
+			}, e)
+			break insertFrame
+		default:
+			if e == f.fnReceived.Front() {
+				f.fnReceived.PushFront(&frameInfo{
+					startSeq: ep.Packet.SequenceNumber,
+					endSeq:   ep.Packet.SequenceNumber,
+					ts:       ep.Packet.Timestamp,
+					temporal: temporal,
+				})
+				break insertFrame
+			}
+		}
+	}
+
+	return f.calc()
+}
+
+func (f *FrameRateCalculatorH26x) calc() bool {
+	frameCounts := make([]int, DefaultMaxLayerTemporal+1)
+	var totalFrameCount int
+	var tsDuration int
+	cur := f.fnReceived.Front()
+	for {
+		next := cur.Next()
+		if next == nil {
+			break
+		}
+		ff := cur.Value.(*frameInfo)
+		nf := next.Value.(*frameInfo)
+		if nf.startSeq-ff.endSeq == 1 {
+			totalFrameCount++
+			tsDuration += int(nf.ts - ff.ts)
+			for i := int(nf.temporal); i < len(frameCounts); i++ {
+				frameCounts[i]++
+			}
+		} else {
+			// reset to find continuous frames
+			totalFrameCount = 0
+			for i := range frameCounts {
+				frameCounts[i] = 0
+			}
+			tsDuration = 0
+		}
+
+		// received enough continuous frames, calculate fps
+		if totalFrameCount >= minFramesForCalculation[DefaultMaxLayerTemporal] {
+			for currentTemporal := int32(0); currentTemporal <= DefaultMaxLayerTemporal; currentTemporal++ {
+				count := frameCounts[currentTemporal]
+				if currentTemporal > 0 && count == frameCounts[currentTemporal-1] {
+					// no frames for this temporal layer
+					f.frameRates[currentTemporal] = 0
+				} else {
+					f.frameRates[currentTemporal] = float32(f.clockRate) / float32(tsDuration) * float32(count)
+				}
+			}
+			f.logger.Debugw("fps changed", "fps", f.GetFrameRate())
+			f.completed = true
+			f.reset()
+			return true
+		}
+
+		cur = next
+	}
+
+	return false
+}
+
+func (f *FrameRateCalculatorH26x) reset() {
+	f.fnReceived.Init()
+	f.baseFrame = nil
+}
+
+func (f *FrameRateCalculatorH26x) GetFrameRate() []float32 {
+	return f.frameRates[:]
+}
+
+// -----------------------------------------------
+func sn16LT(a, b uint16) bool {
+	return a-b > 0x8000
+}
+
+func sn16LTOrEqual(a, b uint16) bool {
+	return a == b || a-b > 0x8000
+}
+
+func sn32LT(a, b uint32) bool {
+	return a-b > 0x80000000
+}

pkg/sfu/buffer/fps_test.go

@@ -0,0 +1,439 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"testing"
+
+	"github.com/pion/rtp"
+	"github.com/stretchr/testify/require"
+
+	dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
+	"github.com/livekit/mediatransportutil/pkg/codec"
+	"github.com/livekit/protocol/logger"
+)
+
+type testFrameInfo struct {
+	header      rtp.Header
+	framenumber uint16
+	spatial     int
+	temporal    int
+	frameDiff   []int
+}
+
+func (f *testFrameInfo) toVP8() *ExtPacket {
+	return &ExtPacket{
+		Packet: &rtp.Packet{Header: f.header},
+		Payload: codec.VP8{
+			I:         true,
+			PictureID: f.framenumber,
+		},
+		VideoLayer: VideoLayer{Spatial: InvalidLayerSpatial, Temporal: int32(f.temporal)},
+	}
+}
+
+func (f *testFrameInfo) toDD() *ExtPacket {
+	return &ExtPacket{
+		Packet: &rtp.Packet{Header: f.header},
+		DependencyDescriptor: &ExtDependencyDescriptor{
+			Descriptor: &dd.DependencyDescriptor{
+				FrameNumber: f.framenumber,
+				FrameDependencies: &dd.FrameDependencyTemplate{
+					FrameDiffs: f.frameDiff,
+				},
+			},
+		},
+		VideoLayer: VideoLayer{Spatial: int32(f.spatial), Temporal: int32(f.temporal)},
+	}
+}
+
+func (f *testFrameInfo) toH26x() *ExtPacket {
+	return &ExtPacket{
+		Packet:     &rtp.Packet{Header: f.header},
+		VideoLayer: VideoLayer{Spatial: InvalidLayerSpatial, Temporal: int32(f.temporal)},
+	}
+}
+
+func createFrames(startFrameNumber uint16, startTs uint32, startSeq uint16, totalFramesPerSpatial int, fps [][]float32, spatialDependency bool) [][]*testFrameInfo {
+	spatials := len(fps)
+	temporals := len(fps[0])
+	frames := make([][]*testFrameInfo, spatials)
+	for s := 0; s < spatials; s++ {
+		frames[s] = make([]*testFrameInfo, 0, totalFramesPerSpatial)
+	}
+	fn := startFrameNumber
+
+	nextTs := make([][]uint32, spatials)
+	tsStep := make([][]uint32, spatials)
+	for i := range spatials {
+		nextTs[i] = make([]uint32, temporals)
+		tsStep[i] = make([]uint32, temporals)
+		for j := 0; j < temporals; j++ {
+			nextTs[i][j] = startTs
+			tsStep[i][j] = uint32(90000 / fps[i][j])
+		}
+	}
+
+	currentTs := make([]uint32, spatials)
+	for i := range spatials {
+		currentTs[i] = startTs
+	}
+	for range totalFramesPerSpatial {
+		for s := range spatials {
+			frame := &testFrameInfo{
+				header:      rtp.Header{Timestamp: currentTs[s], SequenceNumber: startSeq},
+				framenumber: fn,
+				spatial:     s,
+			}
+			for t := 0; t < temporals; t++ {
+				if currentTs[s] >= nextTs[s][t] {
+					frame.temporal = t
+					for nt := t; nt < temporals; nt++ {
+						nextTs[s][nt] += tsStep[s][nt]
+					}
+					break
+				}
+			}
+			currentTs[s] += tsStep[s][temporals-1]
+			frames[s] = append(frames[s], frame)
+			fn++
+			startSeq++
+
+			for fidx := len(frames[s]) - 1; fidx >= 0; fidx-- {
+				cf := frames[s][fidx]
+				if cf.header.Timestamp-frame.header.Timestamp > 0x80000000 {
+					frame.frameDiff = append(frame.frameDiff, int(frame.framenumber-cf.framenumber))
+					break
+				}
+			}
+
+			if spatialDependency && frame.spatial > 0 {
+				for fidx := len(frames[frame.spatial-1]) - 1; fidx >= 0; fidx-- {
+					cf := frames[frame.spatial-1][fidx]
+					if cf.header.Timestamp == frame.header.Timestamp {
+						frame.frameDiff = append(frame.frameDiff, int(frame.framenumber-cf.framenumber))
+						break
+					}
+				}
+			}
+		}
+	}
+
+	return frames
+}
+
+func verifyFps(t *testing.T, expect, got []float32) {
+	require.Equal(t, len(expect), len(got))
+	for i := range expect {
+		require.GreaterOrEqual(t, got[i], expect[i]*0.9, "expect %v, got %v", expect, got)
+		require.LessOrEqual(t, got[i], expect[i]*1.1, "expect %v, got %v", expect, got)
+	}
+}
+
+type testcase struct {
+	startTs           uint32
+	startSeq          uint16
+	startFrameNumber  uint16
+	fps               [][]float32
+	spatialDependency bool
+}
+
+func TestFpsVP8(t *testing.T) {
+	cases := map[string]testcase{
+		"normal": {
+			startTs:          12345678,
+			startFrameNumber: 100,
+			fps:              [][]float32{{5, 10, 15}, {5, 10, 15}, {7.5, 15, 30}},
+		},
+		"frame number and timestamp wrap": {
+			startTs:          (uint32(1) << 31) - 10,
+			startFrameNumber: (uint16(1) << 15) - 10,
+			fps:              [][]float32{{5, 10, 15}, {5, 10, 15}, {7.5, 15, 30}},
+		},
+		"2 temporal layers": {
+			startTs:          12345678,
+			startFrameNumber: 100,
+			fps:              [][]float32{{7.5, 15}, {7.5, 15}, {15, 30}},
+		},
+	}
+
+	for name, c := range cases {
+		testCase := c
+		t.Run(name, func(t *testing.T) {
+			fps := testCase.fps
+			frames := make([][]*testFrameInfo, 0)
+			vp8calcs := make([]*FrameRateCalculatorVP8, len(fps))
+			for i := range vp8calcs {
+				vp8calcs[i] = NewFrameRateCalculatorVP8(90000, logger.GetLogger())
+				frames = append(frames, createFrames(c.startFrameNumber, c.startTs, 10, 200, [][]float32{fps[i]}, false)[0])
+			}
+
+			var frameratesGot bool
+			for s, fs := range frames {
+				for _, f := range fs {
+					if vp8calcs[s].RecvPacket(f.toVP8()) {
+						frameratesGot = true
+						for _, calc := range vp8calcs {
+							if !calc.Completed() {
+								frameratesGot = false
+								break
+							}
+						}
+					}
+				}
+			}
+			require.True(t, frameratesGot)
+			for i, calc := range vp8calcs {
+				fpsExpected := fps[i]
+				fpsGot := calc.GetFrameRate()
+				verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
+			}
+		})
+	}
+	t.Run("packet lost and duplicate", func(t *testing.T) {
+		fps := [][]float32{{7.5, 15}, {7.5, 15}, {15, 30}}
+		frames := make([][]*testFrameInfo, 0)
+		vp8calcs := make([]*FrameRateCalculatorVP8, len(fps))
+		for i := range vp8calcs {
+			vp8calcs[i] = NewFrameRateCalculatorVP8(90000, logger.GetLogger())
+			frames = append(frames, createFrames(100, 12345678, 10, 300, [][]float32{fps[i]}, false)[0])
+			for j := 5; j < 130; j++ {
+				if j%2 == 0 {
+					frames[i][j] = frames[i][j-1]
+				}
+			}
+		}
+
+		var frameratesGot bool
+		for s, fs := range frames {
+			for _, f := range fs {
+				if vp8calcs[s].RecvPacket(f.toVP8()) {
+					frameratesGot = true
+					for _, calc := range vp8calcs {
+						if !calc.Completed() {
+							frameratesGot = false
+							break
+						}
+					}
+				}
+			}
+		}
+		require.True(t, frameratesGot)
+		for i, calc := range vp8calcs {
+			fpsExpected := fps[i]
+			fpsGot := calc.GetFrameRate()
+			verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
+		}
+	})
+}
+
+func TestFpsDD(t *testing.T) {
+	cases := map[string]testcase{
+		"normal": {
+			startTs:           12345678,
+			startFrameNumber:  100,
+			fps:               [][]float32{{5.1, 10.1, 16}, {5.1, 10.1, 16}, {8, 15, 30.1}},
+			spatialDependency: true,
+		},
+		"frame number and timestamp wrap": {
+			startTs:           (uint32(1) << 31) - 10,
+			startFrameNumber:  (uint16(1) << 15) - 10,
+			fps:               [][]float32{{7.5, 15, 30}, {7.5, 15, 30}, {7.5, 15, 30}},
+			spatialDependency: true,
+		},
+		"vp8": {
+			startTs:           12345678,
+			startFrameNumber:  100,
+			fps:               [][]float32{{7.5, 15}, {7.5, 15}, {15, 30}},
+			spatialDependency: false,
+		},
+	}
+
+	for name, c := range cases {
+		testCase := c
+		t.Run(name, func(t *testing.T) {
+			fps := testCase.fps
+			frames := createFrames(c.startFrameNumber, c.startTs, 10, 500, fps, testCase.spatialDependency)
+			ddcalc := NewFrameRateCalculatorDD(90000, logger.GetLogger())
+			ddcalc.SetMaxLayer(int32(len(fps)-1), int32(len(fps[0])-1))
+			ddcalcs := make([]FrameRateCalculator, len(fps))
+			for i := range fps {
+				ddcalcs[i] = ddcalc.GetFrameRateCalculatorForSpatial(int32(i))
+			}
+
+			var frameratesGot bool
+			for s, fs := range frames {
+				for _, f := range fs {
+					if ddcalcs[s].RecvPacket(f.toDD()) {
+						frameratesGot = true
+						for _, calc := range ddcalcs {
+							if !calc.Completed() {
+								frameratesGot = false
+								break
+							}
+						}
+					}
+				}
+			}
+			require.True(t, frameratesGot)
+			for i, calc := range ddcalcs {
+				fpsExpected := fps[i]
+				fpsGot := calc.GetFrameRate()
+				verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
+			}
+		})
+	}
+
+	t.Run("packet lost and duplicate", func(t *testing.T) {
+		fps := [][]float32{{7.5, 15, 30}, {7.5, 15, 30}, {7.5, 15, 30}}
+		frames := createFrames(100, 12345678, 10, 500, fps, true)
+		ddcalc := NewFrameRateCalculatorDD(90000, logger.GetLogger())
+		ddcalc.SetMaxLayer(int32(len(fps)-1), int32(len(fps[0])-1))
+		ddcalcs := make([]FrameRateCalculator, len(fps))
+		for i := range fps {
+			ddcalcs[i] = ddcalc.GetFrameRateCalculatorForSpatial(int32(i))
+			for j := 5; j < 130; j++ {
+				if j%2 == 0 {
+					frames[i][j] = frames[i][j-1]
+				}
+			}
+		}
+
+		var frameratesGot bool
+		for s, fs := range frames {
+			for _, f := range fs {
+				if ddcalcs[s].RecvPacket(f.toDD()) {
+					frameratesGot = true
+					for _, calc := range ddcalcs {
+						if !calc.Completed() {
+							frameratesGot = false
+							break
+						}
+					}
+				}
+			}
+		}
+		require.True(t, frameratesGot)
+		for i, calc := range ddcalcs {
+			fpsExpected := fps[i]
+			fpsGot := calc.GetFrameRate()
+			verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
+		}
+	})
+}
+
+func TestFpsH26x(t *testing.T) {
+	cases := map[string]testcase{
+		"normal": {
+			startTs:          12345678,
+			startSeq:         100,
+			startFrameNumber: 100,
+			fps:              [][]float32{{5, 10, 15}, {5, 10, 15}, {7.5, 15, 30}},
+		},
+		"frame number and timestamp wrap": {
+			startTs:          (uint32(1) << 31) - 10,
+			startSeq:         (uint16(1) << 15) - 10,
+			startFrameNumber: (uint16(1) << 15) - 10,
+			fps:              [][]float32{{5, 10, 15}, {5, 10, 15}, {7.5, 15, 30}},
+		},
+		"2 temporal layers": {
+			startTs:          12345678,
+			startFrameNumber: 100,
+			fps:              [][]float32{{7.5, 15}, {7.5, 15}, {15, 30}},
+		},
+	}
+
+	for name, c := range cases {
+		testCase := c
+		t.Run(name, func(t *testing.T) {
+			fps := testCase.fps
+			frames := make([][]*testFrameInfo, 0)
+			h26xcalcs := make([]*FrameRateCalculatorH26x, len(fps))
+			for i := range h26xcalcs {
+				h26xcalcs[i] = NewFrameRateCalculatorH26x(90000, logger.GetLogger())
+				frames = append(frames, createFrames(c.startFrameNumber, c.startTs, c.startSeq, 200, [][]float32{fps[i]}, false)[0])
+			}
+
+			var frameratesGot bool
+			for s, fs := range frames {
+				for _, f := range fs {
+					if h26xcalcs[s].RecvPacket(f.toH26x()) {
+						frameratesGot = true
+						for _, calc := range h26xcalcs {
+							if !calc.Completed() {
+								frameratesGot = false
+								break
+							}
+						}
+					}
+				}
+			}
+			require.True(t, frameratesGot)
+			for i, calc := range h26xcalcs {
+				fpsExpected := fps[i]
+				fpsGot := calc.GetFrameRate()
+				verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
+			}
+		})
+	}
+
+	t.Run("packet lost and duplicate", func(t *testing.T) {
+		fps := [][]float32{{7.5, 15, 30}, {7.5, 15, 30}, {7.5, 15, 30}}
+		frames := make([][]*testFrameInfo, 0, len(fps))
+		h26xcalcs := make([]FrameRateCalculator, len(fps))
+		for i := range fps {
+			frames = append(frames, createFrames(100, 12345678, 10, 500, [][]float32{fps[i]}, false)[0])
+			h26xcalcs[i] = NewFrameRateCalculatorH26x(90000, logger.GetLogger())
+			for j := 5; j < 130; j++ {
+				if j%2 == 0 {
+					frames[i][j] = frames[i][j-1]
+				}
+			}
+			for j := 130; j < 230; j++ {
+				if j%3 == 0 {
+					frames[i][j] = nil
+				}
+			}
+			for j := 230; j < 330; j++ {
+				if j%2 == 0 {
+					frames[i][j], frames[i][j-1] = frames[i][j-1], frames[i][j]
+				}
+			}
+		}
+		var frameratesGot bool
+		for s, fs := range frames {
+			for _, f := range fs {
+				if f == nil {
+					continue
+				}
+				if h26xcalcs[s].RecvPacket(f.toH26x()) {
+					frameratesGot = true
+					for _, calc := range h26xcalcs {
+						if !calc.Completed() {
+							frameratesGot = false
+							break
+						}
+					}
+				}
+			}
+		}
+		require.True(t, frameratesGot)
+		for i, calc := range h26xcalcs {
+			fpsExpected := fps[i]
+			fpsGot := calc.GetFrameRate()
+			verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
+		}
+	})
+}

pkg/sfu/buffer/frameintegrity.go

@@ -0,0 +1,238 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
+)
+
+type FrameEntity struct {
+	startSeq  *uint64
+	endSeq    *uint64
+	integrity bool
+
+	pktHistory *PacketHistory
+}
+
+func (fe *FrameEntity) AddPacket(extSeq uint64, ddVal *dd.DependencyDescriptor) {
+	// duplicate packet
+	if fe.integrity {
+		return
+	}
+
+	if fe.startSeq == nil && ddVal.FirstPacketInFrame {
+		fe.startSeq = &extSeq
+	}
+	if fe.endSeq == nil && ddVal.LastPacketInFrame {
+		fe.endSeq = &extSeq
+	}
+
+	if fe.startSeq != nil && fe.endSeq != nil {
+		if fe.pktHistory.PacketsConsecutive(*fe.startSeq, *fe.endSeq) {
+			fe.integrity = true
+		}
+	}
+}
+
+func (fe *FrameEntity) Reset() {
+	fe.integrity = false
+	fe.startSeq, fe.endSeq = nil, nil
+}
+
+func (fe *FrameEntity) Integrity() bool {
+	return fe.integrity
+}
+
+// ------------------------------
+
+type PacketHistory struct {
+	base        uint64
+	last        uint64
+	bits        []uint64
+	packetCount int
+	inited      bool
+}
+
+func NewPacketHistory(packetCount int) *PacketHistory {
+	packetCount = (packetCount + 63) / 64 * 64
+	return &PacketHistory{
+		bits:        make([]uint64, packetCount/64),
+		packetCount: packetCount,
+	}
+}
+
+func (ph *PacketHistory) AddPacket(extSeq uint64) {
+	if !ph.inited {
+		ph.inited = true
+		ph.base = uint64(extSeq)
+		// set base to extSeq-100 to avoid out-of-order packets belongs to first frame to be dropped
+		if ph.base > 100 {
+			ph.base -= 100
+		} else {
+			ph.base = 0
+		}
+		ph.last = uint64(extSeq)
+		ph.set(extSeq, true)
+		return
+	}
+
+	if extSeq <= ph.base {
+		// too old
+		return
+	}
+
+	if extSeq <= ph.last {
+		if ph.last-extSeq < uint64(ph.packetCount) {
+			ph.set(extSeq, true)
+		}
+		return
+	}
+
+	// A forward jump only needs at most packetCount slots cleared; anything older aliases
+	// into the same ring and would be overwritten anyway. Cap the loop so a single crafted
+	// sequence number jump (up to ~32k by the extension wrap-around heuristic) cannot force
+	// a large per-packet iteration count.
+	start := ph.last + 1
+	if extSeq-start > uint64(ph.packetCount) {
+		start = extSeq - uint64(ph.packetCount)
+	}
+	for i := start; i < extSeq; i++ {
+		ph.set(i, false)
+	}
+
+	ph.set(extSeq, true)
+	ph.last = extSeq
+}
+
+func (ph *PacketHistory) getPos(seq uint64) (index, offset int) {
+	idx := (seq - ph.base) % uint64(ph.packetCount)
+	return int(idx >> 6), int(idx % 64)
+}
+
+func (ph *PacketHistory) set(seq uint64, received bool) {
+	idx, offset := ph.getPos(seq)
+	if !received {
+		ph.bits[idx] &= ^(1 << offset)
+	} else {
+		ph.bits[idx] |= 1 << (offset)
+	}
+}
+
+func (ph *PacketHistory) PacketsConsecutive(start, end uint64) bool {
+	if start > end {
+		return false
+	}
+
+	if end-start >= uint64(ph.packetCount) {
+		return false
+	}
+
+	startIndex, startOffset := ph.getPos(start)
+	endIndex, endOffset := ph.getPos(end)
+
+	if startIndex == endIndex && end-start <= 64 {
+		testBits := uint64((1<<(endOffset-startOffset+1))-1) << startOffset
+		return ph.bits[startIndex]&testBits == testBits
+	}
+
+	if (ph.bits[startIndex]>>(startOffset))+1 != 1<<(64-startOffset) {
+		return false
+	}
+
+	for i := startIndex + 1; i != endIndex; i++ {
+		if i == len(ph.bits) {
+			i = 0
+			if i == endIndex {
+				break
+			}
+		}
+		if ph.bits[i]+1 != 0 {
+			return false
+		}
+	}
+
+	testBits := uint64((1 << (endOffset + 1)) - 1)
+	return ph.bits[endIndex]&testBits == testBits
+}
+
+// ------------------------------
+
+type FrameIntegrityChecker struct {
+	frameCount int
+	frames     []FrameEntity
+	base       uint64
+	last       uint64
+
+	pktHistory *PacketHistory
+	inited     bool
+}
+
+func NewFrameIntegrityChecker(frameCount, packetCount int) *FrameIntegrityChecker {
+	fc := &FrameIntegrityChecker{
+		frames:     make([]FrameEntity, frameCount),
+		pktHistory: NewPacketHistory(packetCount),
+		frameCount: frameCount,
+	}
+
+	for i := range fc.frames {
+		fc.frames[i].pktHistory = fc.pktHistory
+		fc.frames[i].Reset()
+	}
+	return fc
+}
+
+func (fc *FrameIntegrityChecker) AddPacket(extSeq uint64, extFrameNum uint64, ddVal *dd.DependencyDescriptor) {
+	fc.pktHistory.AddPacket(extSeq)
+
+	if !fc.inited {
+		fc.inited = true
+		fc.base = extFrameNum
+		fc.last = extFrameNum
+	}
+
+	if extFrameNum < fc.base {
+		// frame too old
+		return
+	}
+
+	if extFrameNum <= fc.last {
+		if fc.last-extFrameNum >= uint64(fc.frameCount) {
+			// frame too old
+			return
+		}
+		fc.frames[int(extFrameNum-fc.base)%fc.frameCount].AddPacket(extSeq, ddVal)
+		return
+	}
+
+	// reset missing frames; cap to frameCount so a crafted frame-number jump cannot force a
+	// large loop (older frames alias into the same ring and get overwritten anyway).
+	start := fc.last + 1
+	if extFrameNum-fc.last > uint64(fc.frameCount) {
+		start = extFrameNum - uint64(fc.frameCount) + 1
+	}
+	for i := start; i <= extFrameNum; i++ {
+		fc.frames[int(i-fc.base)%fc.frameCount].Reset()
+	}
+	fc.frames[int(extFrameNum-fc.base)%fc.frameCount].AddPacket(extSeq, ddVal)
+	fc.last = extFrameNum
+}
+
+func (fc *FrameIntegrityChecker) FrameIntegrity(extFrameNum uint64) bool {
+	if extFrameNum < fc.base || extFrameNum > fc.last || fc.last-extFrameNum >= uint64(fc.frameCount) {
+		return false
+	}
+
+	return fc.frames[int(extFrameNum-fc.base)%fc.frameCount].Integrity()
+}

pkg/sfu/buffer/frameintegrity_test.go

@@ -0,0 +1,147 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"math/bits"
+	"math/rand"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
+)
+
+func TestFrameIntegrityChecker(t *testing.T) {
+	fc := NewFrameIntegrityChecker(100, 1000)
+
+	// first frame out of order
+	fc.AddPacket(10, 10, &dd.DependencyDescriptor{})
+	require.False(t, fc.FrameIntegrity(10))
+	fc.AddPacket(9, 10, &dd.DependencyDescriptor{FirstPacketInFrame: true})
+	require.False(t, fc.FrameIntegrity(10))
+	fc.AddPacket(11, 10, &dd.DependencyDescriptor{LastPacketInFrame: true})
+	require.True(t, fc.FrameIntegrity(10))
+
+	// single packet frame
+	fc.AddPacket(100, 100, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true})
+	require.True(t, fc.FrameIntegrity(100))
+	require.False(t, fc.FrameIntegrity(101))
+	require.False(t, fc.FrameIntegrity(99))
+
+	// frame too old than first frame
+	fc.AddPacket(99, 99, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true})
+
+	// multiple packet frame, out of order
+	fc.AddPacket(2001, 2001, &dd.DependencyDescriptor{})
+	require.False(t, fc.FrameIntegrity(2001))
+	require.False(t, fc.FrameIntegrity(1999))
+	// out of frame count(100)
+	require.False(t, fc.FrameIntegrity(100))
+	require.False(t, fc.FrameIntegrity(1900))
+
+	fc.AddPacket(2000, 2001, &dd.DependencyDescriptor{FirstPacketInFrame: true})
+	require.False(t, fc.FrameIntegrity(2001))
+	fc.AddPacket(2002, 2001, &dd.DependencyDescriptor{LastPacketInFrame: true})
+	require.True(t, fc.FrameIntegrity(2001))
+	// duplicate packet
+	fc.AddPacket(2001, 2001, &dd.DependencyDescriptor{})
+	require.True(t, fc.FrameIntegrity(2001))
+
+	// frame too old
+	fc.AddPacket(900, 1900, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true})
+	require.False(t, fc.FrameIntegrity(1900))
+
+	for frame := uint64(2002); frame < 2102; frame++ {
+		// large frame (1000 packets) out of order / retransmitted
+		firstFrame := uint64(3000 + (frame-2002)*1000)
+		lastFrame := uint64(3999 + (frame-2002)*1000)
+		frames := make([]uint64, 0, lastFrame-firstFrame+1)
+		for i := firstFrame; i <= lastFrame; i++ {
+			frames = append(frames, i)
+		}
+		require.False(t, fc.FrameIntegrity(frame))
+		rng := rand.New(rand.NewSource(int64(frame)))
+		rng.Shuffle(len(frames), func(i, j int) { frames[i], frames[j] = frames[j], frames[i] })
+		for i, f := range frames {
+			fc.AddPacket(f, frame, &dd.DependencyDescriptor{
+				FirstPacketInFrame: f == firstFrame,
+				LastPacketInFrame:  f == lastFrame,
+			})
+			require.Equal(t, i == len(frames)-1, fc.FrameIntegrity(frame), i)
+		}
+		require.True(t, fc.FrameIntegrity(frame))
+	}
+}
+
+func countSetBits(ph *PacketHistory) int {
+	n := 0
+	for _, w := range ph.bits {
+		n += bits.OnesCount64(w)
+	}
+	return n
+}
+
+// A forward sequence-number jump much larger than the ring must clear the whole ring,
+// leaving only the newly received sequence number set.
+func TestPacketHistoryLargeForwardJump(t *testing.T) {
+	ph := NewPacketHistory(1000) // rounds up to a multiple of 64
+	require.Equal(t, 1024, ph.packetCount)
+
+	// Fill the entire ring so every slot holds a "received" bit.
+	base := uint64(100000)
+	ph.AddPacket(base)
+	for i := base + 1; i <= base+2000; i++ {
+		ph.AddPacket(i)
+	}
+	require.Equal(t, ph.packetCount, countSetBits(ph))
+	last := base + 2000
+
+	// Forward jump well beyond both the ring and the ~32k extension wrap-around cap. The ring
+	// must end up fully cleared, with only newSeq marked received.
+	newSeq := last + 40000
+	ph.AddPacket(newSeq)
+
+	// If the cap under-cleared, stale bits from the pre-jump fill would survive here.
+	require.Equal(t, 1, countSetBits(ph))
+	require.True(t, ph.PacketsConsecutive(newSeq, newSeq))
+	require.False(t, ph.PacketsConsecutive(newSeq-5, newSeq))
+
+	// The window just below newSeq was cleared and can be refilled normally.
+	for i := newSeq - 5; i < newSeq; i++ {
+		ph.AddPacket(i)
+	}
+	require.True(t, ph.PacketsConsecutive(newSeq-5, newSeq))
+}
+
+// A forward frame-number jump much larger than frameCount must reset the whole frame ring,
+// so no frame that aliases an old slot inherits stale integrity.
+func TestFrameIntegrityCheckerLargeFrameJump(t *testing.T) {
+	fc := NewFrameIntegrityChecker(100, 1000)
+
+	// Populate every ring slot with an integral single-packet frame.
+	for f := uint64(200); f <= 399; f++ {
+		fc.AddPacket(f, f, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true})
+	}
+	require.True(t, fc.FrameIntegrity(399))
+
+	// Jump far beyond frameCount. The capped reset loop must clear the entire frame ring; if it
+	// under-cleared, some aliased slot would still report a stale frame's integrity.
+	newFrame := uint64(399 + 5000)
+	fc.AddPacket(50000, newFrame, &dd.DependencyDescriptor{}) // incomplete frame, no first/last
+	for f := newFrame - uint64(fc.frameCount) + 1; f <= newFrame; f++ {
+		require.False(t, fc.FrameIntegrity(f), "frame %d should not be integral after jump", f)
+	}
+}

pkg/sfu/buffer/rtcpreader.go

@@ -0,0 +1,63 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"io"
+
+	"go.uber.org/atomic"
+)
+
+type RTCPReader struct {
+	ssrc     uint32
+	closed   atomic.Bool
+	onPacket atomic.Value // func([]byte)
+	onClose  func()
+}
+
+func NewRTCPReader(ssrc uint32) *RTCPReader {
+	return &RTCPReader{ssrc: ssrc}
+}
+
+//go:noinline
+func (r *RTCPReader) Write(p []byte) (n int, err error) {
+	if r.closed.Load() {
+		err = io.EOF
+		return
+	}
+	if f, ok := r.onPacket.Load().(func([]byte)); ok && f != nil {
+		f(p)
+	}
+	return
+}
+
+func (r *RTCPReader) OnClose(fn func()) {
+	r.onClose = fn
+}
+
+func (r *RTCPReader) Close() error {
+	if r.closed.Swap(true) {
+		return nil
+	}
+
+	r.onClose()
+	return nil
+}
+
+func (r *RTCPReader) OnPacket(f func([]byte)) {
+	r.onPacket.Store(f)
+}
+
+func (r *RTCPReader) Read(_ []byte) (n int, err error) { return }

pkg/sfu/buffer/streamstats.go

@@ -0,0 +1,24 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import "github.com/livekit/livekit-server/pkg/sfu/rtpstats"
+
+type StreamStatsWithLayers struct {
+	RTPStats *rtpstats.RTPDeltaInfo
+	Layers   map[int32]*rtpstats.RTPDeltaInfo
+
+	RTPStatsRemoteView *rtpstats.RTPDeltaInfo
+}

pkg/sfu/buffer/videolayer.go

@@ -0,0 +1,58 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import "fmt"
+
+const (
+	InvalidLayerSpatial  = int32(-1)
+	InvalidLayerTemporal = int32(-1)
+
+	DefaultMaxLayerSpatial  = int32(2)
+	DefaultMaxLayerTemporal = int32(3)
+)
+
+var (
+	InvalidLayer = VideoLayer{
+		Spatial:  InvalidLayerSpatial,
+		Temporal: InvalidLayerTemporal,
+	}
+
+	DefaultMaxLayer = VideoLayer{
+		Spatial:  DefaultMaxLayerSpatial,
+		Temporal: DefaultMaxLayerTemporal,
+	}
+)
+
+type VideoLayer struct {
+	Spatial  int32
+	Temporal int32
+}
+
+func (v VideoLayer) String() string {
+	return fmt.Sprintf("VideoLayer{s: %d, t: %d}", v.Spatial, v.Temporal)
+}
+
+func (v VideoLayer) GreaterThan(v2 VideoLayer) bool {
+	return v.Spatial > v2.Spatial || (v.Spatial == v2.Spatial && v.Temporal > v2.Temporal)
+}
+
+func (v VideoLayer) SpatialGreaterThanOrEqual(v2 VideoLayer) bool {
+	return v.Spatial >= v2.Spatial
+}
+
+func (v VideoLayer) IsValid() bool {
+	return v.Spatial != InvalidLayerSpatial && v.Temporal != InvalidLayerTemporal
+}

pkg/sfu/buffer/videolayerutils.go

@@ -0,0 +1,520 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"slices"
+
+	"github.com/livekit/protocol/codecs/mime"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+)
+
+const (
+	quarterResolutionQ = "q"
+	halfResolutionH    = "h"
+	fullResolutionF    = "f"
+
+	quarterResolution2 = "2"
+	halfResolution1    = "1"
+	fullResolution0    = "0"
+)
+
+type VideoLayersRid [DefaultMaxLayerSpatial + 1]string
+
+var (
+	videoLayersRidQHF     = VideoLayersRid{quarterResolutionQ, halfResolutionH, fullResolutionF}
+	videoLayersRid210     = VideoLayersRid{quarterResolution2, halfResolution1, fullResolution0}
+	DefaultVideoLayersRid = videoLayersRidQHF
+)
+
+func LayerPresenceFromTrackInfo(mimeType mime.MimeType, trackInfo *livekit.TrackInfo) *[livekit.VideoQuality_HIGH + 1]bool {
+	if trackInfo == nil {
+		return nil
+	}
+
+	layers := GetVideoLayersForMimeType(mimeType, trackInfo)
+	if len(layers) == 0 {
+		return nil
+	}
+
+	var layerPresence [livekit.VideoQuality_HIGH + 1]bool
+	for _, layer := range layers {
+		// WARNING: comparing protobuf enum
+		if layer.Quality <= livekit.VideoQuality_HIGH {
+			layerPresence[layer.Quality] = true
+		} else {
+			logger.Warnw("unexpected quality in track info", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
+		}
+	}
+
+	return &layerPresence
+}
+
+func RidToSpatialLayer(mimeType mime.MimeType, rid string, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) int32 {
+	lp := LayerPresenceFromTrackInfo(mimeType, trackInfo)
+	if lp == nil {
+		switch rid {
+		case quarterResolutionQ:
+			return 0
+		case halfResolutionH:
+			return 1
+		case fullResolutionF:
+			return 2
+		default:
+			return 0
+		}
+	}
+
+	switch rid {
+	case ridSpace[0]:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return 0
+
+		default:
+			// only one quality published, could be any
+			return 0
+		}
+
+	case ridSpace[1]:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return 1
+
+		default:
+			// only one quality published, could be any
+			return 0
+		}
+
+	case ridSpace[2]:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return 2
+
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			logger.Warnw("unexpected rid with only two qualities, low and medium", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo), "rid", ridSpace[2])
+			return 1
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			logger.Warnw("unexpected rid with only two qualities, low and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo), "rid", ridSpace[2])
+			return 1
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			logger.Warnw("unexpected rid with only two qualities, medium and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo), "rid", ridSpace[2])
+			return 1
+
+		default:
+			// only one quality published, could be any
+			return 0
+		}
+
+	default:
+		// no rid, should be single layer
+		return 0
+	}
+}
+
+func SpatialLayerToRid(mimeType mime.MimeType, layer int32, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) string {
+	lp := LayerPresenceFromTrackInfo(mimeType, trackInfo)
+	if lp == nil {
+		switch layer {
+		case 0:
+			return quarterResolutionQ
+		case 1:
+			return halfResolutionH
+		case 2:
+			return fullResolutionF
+		default:
+			return quarterResolutionQ
+		}
+	}
+
+	switch layer {
+	case 0:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return ridSpace[0]
+
+		default:
+			return ridSpace[0]
+		}
+
+	case 1:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return ridSpace[1]
+
+		default:
+			return ridSpace[0]
+		}
+
+	case 2:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return ridSpace[2]
+
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			logger.Warnw("unexpected layer 2 with only two qualities, low and medium", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
+			return ridSpace[1]
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			logger.Warnw("unexpected layer 2 with only two qualities, low and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
+			return ridSpace[1]
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			logger.Warnw("unexpected layer 2 with only two qualities, medium and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
+			return ridSpace[1]
+
+		default:
+			return ridSpace[0]
+		}
+
+	default:
+		return ridSpace[0]
+	}
+}
+
+func VideoQualityToRid(mimeType mime.MimeType, quality livekit.VideoQuality, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) string {
+	return SpatialLayerToRid(mimeType, VideoQualityToSpatialLayer(mimeType, quality, trackInfo), trackInfo, ridSpace)
+}
+
+func SpatialLayerToVideoQuality(mimeType mime.MimeType, layer int32, trackInfo *livekit.TrackInfo) livekit.VideoQuality {
+	lp := LayerPresenceFromTrackInfo(mimeType, trackInfo)
+	if lp == nil {
+		switch layer {
+		case 0:
+			return livekit.VideoQuality_LOW
+		case 1:
+			return livekit.VideoQuality_MEDIUM
+		case 2:
+			return livekit.VideoQuality_HIGH
+		default:
+			return livekit.VideoQuality_OFF
+		}
+	}
+
+	switch layer {
+	case 0:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW]:
+			return livekit.VideoQuality_LOW
+
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_MEDIUM]:
+			return livekit.VideoQuality_MEDIUM
+
+		default:
+			return livekit.VideoQuality_HIGH
+		}
+
+	case 1:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			return livekit.VideoQuality_MEDIUM
+
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return livekit.VideoQuality_HIGH
+
+		default:
+			logger.Errorw("invalid layer", nil, "trackID", trackInfo.Sid, "layer", layer, "trackInfo", logger.Proto(trackInfo))
+			return livekit.VideoQuality_HIGH
+		}
+
+	case 2:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return livekit.VideoQuality_HIGH
+
+		default:
+			logger.Errorw("invalid layer", nil, "trackID", trackInfo.Sid, "layer", layer, "trackInfo", logger.Proto(trackInfo))
+			return livekit.VideoQuality_HIGH
+		}
+	}
+
+	return livekit.VideoQuality_OFF
+}
+
+func VideoQualityToSpatialLayer(mimeType mime.MimeType, quality livekit.VideoQuality, trackInfo *livekit.TrackInfo) int32 {
+	lp := LayerPresenceFromTrackInfo(mimeType, trackInfo)
+	if lp == nil {
+		switch quality {
+		case livekit.VideoQuality_LOW:
+			return 0
+		case livekit.VideoQuality_MEDIUM:
+			return 1
+		case livekit.VideoQuality_HIGH:
+			return 2
+		default:
+			return InvalidLayerSpatial
+		}
+	}
+
+	switch quality {
+	case livekit.VideoQuality_LOW:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		default: // only one quality published, could be any
+			return 0
+		}
+
+	case livekit.VideoQuality_MEDIUM:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			return 1
+
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return 0
+
+		default: // only one quality published, could be any
+			return 0
+		}
+
+	case livekit.VideoQuality_HIGH:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return 2
+
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return 1
+
+		default: // only one quality published, could be any
+			return 0
+		}
+	}
+
+	return InvalidLayerSpatial
+}
+
+func GetVideoLayerModeForMimeType(mimeType mime.MimeType, ti *livekit.TrackInfo) livekit.VideoLayer_Mode {
+	if ti != nil {
+		for _, codec := range ti.Codecs {
+			if mime.NormalizeMimeType(codec.MimeType) == mimeType {
+				return codec.VideoLayerMode
+			}
+		}
+	}
+
+	return livekit.VideoLayer_MODE_UNUSED
+}
+
+func GetVideoLayersForMimeType(mimeType mime.MimeType, ti *livekit.TrackInfo) []*livekit.VideoLayer {
+	var layers []*livekit.VideoLayer
+	if ti != nil {
+		for _, codec := range ti.Codecs {
+			if mime.NormalizeMimeType(codec.MimeType) == mimeType {
+				layers = codec.Layers
+				break
+			}
+		}
+		if len(layers) == 0 {
+			layers = ti.Layers
+		}
+	}
+	return layers
+}
+
+func GetSpatialLayerForRid(mimeType mime.MimeType, rid string, ti *livekit.TrackInfo) int32 {
+	if ti == nil {
+		return InvalidLayerSpatial
+	}
+
+	if rid == "" {
+		// single layer without RID
+		return 0
+	}
+
+	layers := GetVideoLayersForMimeType(mimeType, ti)
+	for _, layer := range layers {
+		if layer.Rid == rid {
+			return layer.SpatialLayer
+		}
+	}
+
+	if len(layers) != 0 {
+		// RID present in codec, but may not be specified via signalling
+		// (happens with older browsers setting a rid for SVC codecs)
+		hasRid := false
+		for _, layer := range layers {
+			if layer.Rid != "" {
+				hasRid = true
+				break
+			}
+		}
+		if !hasRid {
+			return 0
+		}
+	}
+
+	// SIMULCAST-CODEC-TODO - ideally should return invalid, but there are
+	// VP9 publishers using rid = f, if there are only two layers
+	// in TrackInfo, that will be q;h and f will become invalid.
+	//
+	// Actually, there should be no rids for VP9 in SDP and hence
+	// the above check should take effect. However, as simulcast
+	// codec/back up codec does not update rids from SDP,
+	// the default rids are used when vp9 (primary codec)
+	// is published. Due to that the above check gets bypassed.
+	//
+	// The full proper sequence would be
+	// 1. For primary codec using SVC, there will be no rids.
+	//    The above check should take effect and it should
+	//    return 0 even if some publisher uses a rid like `f`.
+	// 2. When secondary codec is published, rids for the codec
+	//    corresponding to the back up codec mime type should
+	//    be updated in `TrackInfo`. This is a bit tricky
+	//    for a couple of cases
+	//    a. Browsers like Firefox use a different CID every time.
+	//       So, it cannot be matched between `AddTrack` and SDP.
+	//       One option is to look for a published track with
+	//       back up codec and apply it there. But, that becomes
+	//       a challenge if there are multiple published tracks
+	//       with pending back up codec.
+	//    b. The back up codec publish SDP will have the full
+	//       codec list. It should be okay to assume that the
+	//       codec that will be published is the back up codec,
+	//       but just something to be aware of.
+	// 3. Use of this function with proper mime so that proper
+	//    codec section can be looked up in `TrackInfo`.
+	// return InvalidLayerSpatial
+	logger.Infow(
+		"invalid layer for rid, returning default",
+		"trackID", ti.Sid,
+		"rid", rid,
+		"mimeType", mimeType,
+		"trackInfo", logger.Proto(ti),
+	)
+	return 0
+}
+
+func GetSpatialLayerForVideoQuality(mimeType mime.MimeType, quality livekit.VideoQuality, ti *livekit.TrackInfo) int32 {
+	if ti == nil || quality == livekit.VideoQuality_OFF {
+		return InvalidLayerSpatial
+	}
+
+	layers := GetVideoLayersForMimeType(mimeType, ti)
+	for _, layer := range layers {
+		if layer.Quality == quality {
+			return layer.SpatialLayer
+		}
+	}
+
+	if len(layers) == 0 {
+		// single layer
+		return 0
+	}
+
+	// requested quality is higher than available layers, return the highest available layer
+	return VideoQualityToSpatialLayer(mimeType, quality, ti)
+}
+
+func GetVideoQualityForSpatialLayer(mimeType mime.MimeType, spatialLayer int32, ti *livekit.TrackInfo) livekit.VideoQuality {
+	if spatialLayer == InvalidLayerSpatial || ti == nil {
+		return livekit.VideoQuality_OFF
+	}
+
+	layers := GetVideoLayersForMimeType(mimeType, ti)
+	for _, layer := range layers {
+		if layer.SpatialLayer == spatialLayer {
+			return layer.Quality
+		}
+	}
+
+	return livekit.VideoQuality_OFF
+}
+
+func isVideoLayersRidKnown(rids VideoLayersRid, knownRids VideoLayersRid) bool {
+	for _, rid := range rids {
+		if rid == "" {
+			continue
+		}
+
+		if !slices.Contains(knownRids[:], rid) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func NormalizeVideoLayersRid(rids VideoLayersRid) VideoLayersRid {
+	out := rids
+
+	normalize := func(knownRids VideoLayersRid) {
+		idx := 0
+		for _, known := range knownRids {
+			if slices.Contains(rids[:], known) {
+				out[idx] = known
+				idx++
+			}
+		}
+	}
+
+	if isVideoLayersRidKnown(rids, videoLayersRidQHF) {
+		normalize(videoLayersRidQHF)
+	}
+
+	if isVideoLayersRidKnown(rids, videoLayersRid210) {
+		normalize(videoLayersRid210)
+	}
+
+	return out
+}

pkg/sfu/buffer/videolayerutils_test.go

@@ -0,0 +1,892 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/protocol/codecs/mime"
+	"github.com/livekit/protocol/livekit"
+)
+
+func TestRidConversion(t *testing.T) {
+	type RidAndLayer struct {
+		rid   string
+		layer int32
+	}
+	tests := []struct {
+		name       string
+		trackInfo  *livekit.TrackInfo
+		mimeType   mime.MimeType
+		ridToLayer map[string]RidAndLayer
+	}{
+		{
+			"no track info",
+			nil,
+			mime.MimeTypeVP8,
+			map[string]RidAndLayer{
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: halfResolutionH, layer: 1},
+				fullResolutionF:    {rid: fullResolutionF, layer: 2},
+			},
+		},
+		{
+			"no layers",
+			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
+			map[string]RidAndLayer{
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: halfResolutionH, layer: 1},
+				fullResolutionF:    {rid: fullResolutionF, layer: 2},
+			},
+		},
+		{
+			"single layer, low",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]RidAndLayer{
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: quarterResolutionQ, layer: 0},
+				fullResolutionF:    {rid: quarterResolutionQ, layer: 0},
+			},
+		},
+		{
+			"single layer, medium",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]RidAndLayer{
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: quarterResolutionQ, layer: 0},
+				fullResolutionF:    {rid: quarterResolutionQ, layer: 0},
+			},
+		},
+		{
+			"single layer, high",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]RidAndLayer{
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: quarterResolutionQ, layer: 0},
+				fullResolutionF:    {rid: quarterResolutionQ, layer: 0},
+			},
+		},
+		{
+			"two layers, low and medium",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]RidAndLayer{
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: halfResolutionH, layer: 1},
+				fullResolutionF:    {rid: halfResolutionH, layer: 1},
+			},
+		},
+		{
+			"two layers, low and high",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]RidAndLayer{
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: halfResolutionH, layer: 1},
+				fullResolutionF:    {rid: halfResolutionH, layer: 1},
+			},
+		},
+		{
+			"two layers, medium and high",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]RidAndLayer{
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: halfResolutionH, layer: 1},
+				fullResolutionF:    {rid: halfResolutionH, layer: 1},
+			},
+		},
+		{
+			"three layers",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_MEDIUM},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]RidAndLayer{
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: halfResolutionH, layer: 1},
+				fullResolutionF:    {rid: fullResolutionF, layer: 2},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			for testRid, expectedResult := range test.ridToLayer {
+				actualLayer := RidToSpatialLayer(test.mimeType, testRid, test.trackInfo, DefaultVideoLayersRid)
+				require.Equal(t, expectedResult.layer, actualLayer)
+
+				actualRid := SpatialLayerToRid(test.mimeType, actualLayer, test.trackInfo, DefaultVideoLayersRid)
+				require.Equal(t, expectedResult.rid, actualRid)
+			}
+		})
+	}
+}
+
+func TestQualityConversion(t *testing.T) {
+	type QualityAndLayer struct {
+		quality livekit.VideoQuality
+		layer   int32
+	}
+	tests := []struct {
+		name           string
+		trackInfo      *livekit.TrackInfo
+		mimeType       mime.MimeType
+		qualityToLayer map[livekit.VideoQuality]QualityAndLayer
+	}{
+		{
+			"no track info",
+			nil,
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]QualityAndLayer{
+				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_LOW, layer: 0},
+				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
+				livekit.VideoQuality_HIGH:   {quality: livekit.VideoQuality_HIGH, layer: 2},
+			},
+		},
+		{
+			"no layers",
+			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]QualityAndLayer{
+				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_LOW, layer: 0},
+				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
+				livekit.VideoQuality_HIGH:   {quality: livekit.VideoQuality_HIGH, layer: 2},
+			},
+		},
+		{
+			"single layer, low",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]QualityAndLayer{
+				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_LOW, layer: 0},
+				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_LOW, layer: 0},
+				livekit.VideoQuality_HIGH:   {quality: livekit.VideoQuality_LOW, layer: 0},
+			},
+		},
+		{
+			"single layer, medium",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]QualityAndLayer{
+				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_MEDIUM, layer: 0},
+				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
+				livekit.VideoQuality_HIGH:   {quality: livekit.VideoQuality_MEDIUM, layer: 0},
+			},
+		},
+		{
+			"single layer, high",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]QualityAndLayer{
+				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_HIGH, layer: 0},
+				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 0},
+				livekit.VideoQuality_HIGH:   {quality: livekit.VideoQuality_HIGH, layer: 0},
+			},
+		},
+		{
+			"two layers, low and medium",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]QualityAndLayer{
+				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_LOW, layer: 0},
+				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
+				livekit.VideoQuality_HIGH:   {quality: livekit.VideoQuality_MEDIUM, layer: 1},
+			},
+		},
+		{
+			"two layers, low and high",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]QualityAndLayer{
+				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_LOW, layer: 0},
+				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 1},
+				livekit.VideoQuality_HIGH:   {quality: livekit.VideoQuality_HIGH, layer: 1},
+			},
+		},
+		{
+			"two layers, medium and high",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]QualityAndLayer{
+				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_MEDIUM, layer: 0},
+				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
+				livekit.VideoQuality_HIGH:   {quality: livekit.VideoQuality_HIGH, layer: 1},
+			},
+		},
+		{
+			"three layers",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_MEDIUM},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]QualityAndLayer{
+				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_LOW, layer: 0},
+				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
+				livekit.VideoQuality_HIGH:   {quality: livekit.VideoQuality_HIGH, layer: 2},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			for testQuality, expectedResult := range test.qualityToLayer {
+				actualLayer := VideoQualityToSpatialLayer(test.mimeType, testQuality, test.trackInfo)
+				require.Equal(t, expectedResult.layer, actualLayer)
+
+				actualQuality := SpatialLayerToVideoQuality(test.mimeType, actualLayer, test.trackInfo)
+				require.Equal(t, expectedResult.quality, actualQuality)
+			}
+		})
+	}
+}
+
+func TestVideoQualityToRidConversion(t *testing.T) {
+	tests := []struct {
+		name         string
+		trackInfo    *livekit.TrackInfo
+		mimeTye      mime.MimeType
+		qualityToRid map[livekit.VideoQuality]string
+	}{
+		{
+			"no track info",
+			nil,
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]string{
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: halfResolutionH,
+				livekit.VideoQuality_HIGH:   fullResolutionF,
+			},
+		},
+		{
+			"no layers",
+			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]string{
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: halfResolutionH,
+				livekit.VideoQuality_HIGH:   fullResolutionF,
+			},
+		},
+		{
+			"single layer, low",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]string{
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: quarterResolutionQ,
+				livekit.VideoQuality_HIGH:   quarterResolutionQ,
+			},
+		},
+		{
+			"single layer, medium",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]string{
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: quarterResolutionQ,
+				livekit.VideoQuality_HIGH:   quarterResolutionQ,
+			},
+		},
+		{
+			"single layer, high",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]string{
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: quarterResolutionQ,
+				livekit.VideoQuality_HIGH:   quarterResolutionQ,
+			},
+		},
+		{
+			"two layers, low and medium",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]string{
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: halfResolutionH,
+				livekit.VideoQuality_HIGH:   halfResolutionH,
+			},
+		},
+		{
+			"two layers, low and high",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]string{
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: halfResolutionH,
+				livekit.VideoQuality_HIGH:   halfResolutionH,
+			},
+		},
+		{
+			"two layers, medium and high",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]string{
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: quarterResolutionQ,
+				livekit.VideoQuality_HIGH:   halfResolutionH,
+			},
+		},
+		{
+			"three layers",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_MEDIUM},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]string{
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: halfResolutionH,
+				livekit.VideoQuality_HIGH:   fullResolutionF,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			for testQuality, expectedRid := range test.qualityToRid {
+				actualRid := VideoQualityToRid(test.mimeTye, testQuality, test.trackInfo, DefaultVideoLayersRid)
+				require.Equal(t, expectedRid, actualRid)
+			}
+		})
+	}
+}
+
+func TestGetSpatialLayerForRid(t *testing.T) {
+	tests := []struct {
+		name              string
+		trackInfo         *livekit.TrackInfo
+		mimeType          mime.MimeType
+		ridToSpatialLayer map[string]int32
+	}{
+		{
+			"no track info",
+			nil,
+			mime.MimeTypeVP8,
+			map[string]int32{
+				quarterResolutionQ: InvalidLayerSpatial,
+				halfResolutionH:    InvalidLayerSpatial,
+				fullResolutionF:    InvalidLayerSpatial,
+			},
+		},
+		{
+			"no layers",
+			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
+			map[string]int32{
+				// SIMULCAST-CODEC-TODO
+				// quarterResolutionQ: InvalidLayerSpatial,
+				// halfResolutionH:    InvalidLayerSpatial,
+				// fullResolutionF:    InvalidLayerSpatial,
+				quarterResolutionQ: 0,
+				halfResolutionH:    0,
+				fullResolutionF:    0,
+			},
+		},
+		{
+			"no rid",
+			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
+			map[string]int32{
+				"": 0,
+			},
+		},
+		{
+			"single layer",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]int32{
+				quarterResolutionQ: 0,
+				halfResolutionH:    0,
+				fullResolutionF:    0,
+			},
+		},
+		{
+			"layers",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ},
+							{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]int32{
+				quarterResolutionQ: 0,
+				halfResolutionH:    1,
+				// SIMULCAST-CODEC-TODO
+				// fullResolutionF:    InvalidLayerSpatial,
+				fullResolutionF: 0,
+			},
+		},
+		{
+			"layers - no rid",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0},
+							{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]int32{
+				quarterResolutionQ: 0,
+				halfResolutionH:    0,
+				fullResolutionF:    0,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			for testRid, expectedSpatialLayer := range test.ridToSpatialLayer {
+				actualSpatialLayer := GetSpatialLayerForRid(test.mimeType, testRid, test.trackInfo)
+				require.Equal(t, expectedSpatialLayer, actualSpatialLayer)
+			}
+		})
+	}
+}
+
+func TestGetSpatialLayerForVideoQuality(t *testing.T) {
+	tests := []struct {
+		name                       string
+		trackInfo                  *livekit.TrackInfo
+		mimeType                   mime.MimeType
+		videoQualityToSpatialLayer map[livekit.VideoQuality]int32
+	}{
+		{
+			"no track info",
+			nil,
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]int32{
+				livekit.VideoQuality_LOW:    InvalidLayerSpatial,
+				livekit.VideoQuality_MEDIUM: InvalidLayerSpatial,
+				livekit.VideoQuality_HIGH:   InvalidLayerSpatial,
+				livekit.VideoQuality_OFF:    InvalidLayerSpatial,
+			},
+		},
+		{
+			"no layers",
+			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]int32{
+				livekit.VideoQuality_LOW:    0,
+				livekit.VideoQuality_MEDIUM: 0,
+				livekit.VideoQuality_HIGH:   0,
+				livekit.VideoQuality_OFF:    InvalidLayerSpatial,
+			},
+		},
+		{
+			"not all layers",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ},
+							{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]int32{
+				livekit.VideoQuality_LOW:    0,
+				livekit.VideoQuality_MEDIUM: 1,
+				livekit.VideoQuality_HIGH:   1,
+				livekit.VideoQuality_OFF:    InvalidLayerSpatial,
+			},
+		},
+		{
+			"all layers",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ},
+							{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH},
+							{Quality: livekit.VideoQuality_HIGH, SpatialLayer: 2, Rid: fullResolutionF},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]int32{
+				livekit.VideoQuality_LOW:    0,
+				livekit.VideoQuality_MEDIUM: 1,
+				livekit.VideoQuality_HIGH:   2,
+				livekit.VideoQuality_OFF:    InvalidLayerSpatial,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			for testVideoQuality, expectedSpatialLayer := range test.videoQualityToSpatialLayer {
+				actualSpatialLayer := GetSpatialLayerForVideoQuality(test.mimeType, testVideoQuality, test.trackInfo)
+				require.Equal(t, expectedSpatialLayer, actualSpatialLayer)
+			}
+		})
+	}
+}
+
+func TestGetVideoQualityorSpatialLayer(t *testing.T) {
+	tests := []struct {
+		name                       string
+		trackInfo                  *livekit.TrackInfo
+		mimeType                   mime.MimeType
+		spatialLayerToVideoQuality map[int32]livekit.VideoQuality
+	}{
+		{
+			"no track info",
+			nil,
+			mime.MimeTypeVP8,
+			map[int32]livekit.VideoQuality{
+				InvalidLayerSpatial: livekit.VideoQuality_OFF,
+				0:                   livekit.VideoQuality_OFF,
+				1:                   livekit.VideoQuality_OFF,
+				2:                   livekit.VideoQuality_OFF,
+			},
+		},
+		{
+			"no layers",
+			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
+			map[int32]livekit.VideoQuality{
+				InvalidLayerSpatial: livekit.VideoQuality_OFF,
+				0:                   livekit.VideoQuality_OFF,
+				1:                   livekit.VideoQuality_OFF,
+				2:                   livekit.VideoQuality_OFF,
+			},
+		},
+		{
+			"layers",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ},
+							{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[int32]livekit.VideoQuality{
+				InvalidLayerSpatial: livekit.VideoQuality_OFF,
+				0:                   livekit.VideoQuality_LOW,
+				1:                   livekit.VideoQuality_MEDIUM,
+				2:                   livekit.VideoQuality_OFF,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			for testSpatialLayer, expectedVideoQuality := range test.spatialLayerToVideoQuality {
+				actualVideoQuality := GetVideoQualityForSpatialLayer(test.mimeType, testSpatialLayer, test.trackInfo)
+				require.Equal(t, expectedVideoQuality, actualVideoQuality)
+			}
+		})
+	}
+}
+
+func TestNormalizeVideoLayersRid(t *testing.T) {
+	tests := []struct {
+		name       string
+		rids       VideoLayersRid
+		normalized VideoLayersRid
+	}{
+		{
+			"empty",
+			VideoLayersRid{},
+			VideoLayersRid{},
+		},
+		{
+			"unknown pattern",
+			VideoLayersRid{"3", "2", "1"},
+			VideoLayersRid{"3", "2", "1"},
+		},
+		{
+			"qhf",
+			videoLayersRidQHF,
+			videoLayersRidQHF,
+		},
+		{
+			"scrambled qhf",
+			VideoLayersRid{"f", "h", "q"},
+			videoLayersRidQHF,
+		},
+		{
+			"partial qhf",
+			VideoLayersRid{"h", "q"},
+			VideoLayersRid{"q", "h", ""},
+		},
+		{
+			"210",
+			videoLayersRid210,
+			videoLayersRid210,
+		},
+		{
+			"scrambled 210",
+			VideoLayersRid{"2", "0", "1"},
+			videoLayersRid210,
+		},
+		{
+			"partial 210",
+			VideoLayersRid{"1", "2"},
+			VideoLayersRid{"2", "1", ""},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			normalizedRids := NormalizeVideoLayersRid(test.rids)
+			require.Equal(t, test.normalized, normalizedRids)
+		})
+	}
+}

pkg/sfu/bwe/bwe.go

@@ -0,0 +1,123 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bwe
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/livekit/livekit-server/pkg/sfu/ccutils"
+	"github.com/pion/rtcp"
+)
+
+// ------------------------------------------------
+
+const (
+	DefaultRTT         = float64(0.070) // 70 ms
+	RTTSmoothingFactor = float64(0.5)
+)
+
+// ------------------------------------------------
+
+type BWEType int
+
+const (
+	BWETypeNone BWEType = iota
+	BWETypeRemote
+	BWETypeSendSide
+)
+
+func (b BWEType) String() string {
+	switch b {
+	case BWETypeNone:
+		return "NONE"
+	case BWETypeRemote:
+		return "REMOTE"
+	case BWETypeSendSide:
+		return "SEND_SIDE"
+	default:
+		return fmt.Sprintf("%d", int(b))
+	}
+}
+
+// ------------------------------------------------
+
+type CongestionState int
+
+const (
+	CongestionStateNone CongestionState = iota
+	CongestionStateEarlyWarning
+	CongestionStateCongested
+)
+
+func (c CongestionState) String() string {
+	switch c {
+	case CongestionStateNone:
+		return "NONE"
+	case CongestionStateEarlyWarning:
+		return "EARLY_WARNING"
+	case CongestionStateCongested:
+		return "CONGESTED"
+	default:
+		return fmt.Sprintf("%d", int(c))
+	}
+}
+
+// ------------------------------------------------
+
+type BWE interface {
+	Type() BWEType
+
+	SetBWEListener(bweListner BWEListener)
+
+	Reset()
+
+	HandleREMB(
+		receivedEstimate int64,
+		expectedBandwidthUsage int64,
+		sentPackets uint32,
+		repeatedNacks uint32,
+	)
+
+	// TWCC sequence number
+	RecordPacketSendAndGetSequenceNumber(
+		atMicro int64,
+		size int,
+		isRTX bool,
+		probeClusterId ccutils.ProbeClusterId,
+		isProbe bool,
+	) uint16
+
+	HandleTWCCFeedback(report *rtcp.TransportLayerCC)
+
+	UpdateRTT(rtt float64)
+
+	CongestionState() CongestionState
+
+	CanProbe() bool
+	ProbeDuration() time.Duration
+	ProbeClusterStarting(pci ccutils.ProbeClusterInfo)
+	ProbeClusterDone(pci ccutils.ProbeClusterInfo)
+	ProbeClusterIsGoalReached() bool
+	ProbeClusterFinalize() (ccutils.ProbeSignal, int64, bool)
+}
+
+// ------------------------------------------------
+
+type BWEListener interface {
+	OnCongestionStateChange(fromState CongestionState, toState CongestionState, estimatedAvailableChannelCapacity int64)
+}
+
+// ------------------------------------------------

pkg/sfu/bwe/null_bwe.go

@@ -0,0 +1,77 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bwe
+
+import (
+	"time"
+
+	"github.com/livekit/livekit-server/pkg/sfu/ccutils"
+	"github.com/pion/rtcp"
+)
+
+type NullBWE struct {
+}
+
+func (n *NullBWE) SetBWEListener(_bweListener BWEListener) {}
+
+func (n *NullBWE) Reset() {}
+
+func (n *NullBWE) RecordPacketSendAndGetSequenceNumber(
+	_atMicro int64,
+	_size int,
+	_isRTX bool,
+	_probeClusterId ccutils.ProbeClusterId,
+	_isProbe bool,
+) uint16 {
+	return 0
+}
+
+func (n *NullBWE) HandleREMB(
+	_receivedEstimate int64,
+	_expectedBandwidthUsage int64,
+	_sentPackets uint32,
+	_repeatedNacks uint32,
+) {
+}
+
+func (n *NullBWE) HandleTWCCFeedback(_report *rtcp.TransportLayerCC) {}
+
+func (n *NullBWE) UpdateRTT(rtt float64) {}
+
+func (n *NullBWE) CongestionState() CongestionState {
+	return CongestionStateNone
+}
+
+func (n *NullBWE) CanProbe() bool {
+	return false
+}
+
+func (n *NullBWE) ProbeDuration() time.Duration {
+	return 0
+}
+
+func (n *NullBWE) ProbeClusterStarting(_pci ccutils.ProbeClusterInfo) {}
+
+func (n *NullBWE) ProbeClusterDone(_pci ccutils.ProbeClusterInfo) {}
+
+func (n *NullBWE) ProbeClusterIsGoalReached() bool {
+	return false
+}
+
+func (n *NullBWE) ProbeClusterFinalize() (ccutils.ProbeSignal, int64, bool) {
+	return ccutils.ProbeSignalInconclusive, 0, false
+}
+
+// ------------------------------------------------

pkg/sfu/bwe/remotebwe/channel_observer.go

@@ -0,0 +1,202 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package remotebwe
+
+import (
+	"fmt"
+	"time"
+
+	"go.uber.org/zap/zapcore"
+
+	"github.com/livekit/livekit-server/pkg/sfu/ccutils"
+	"github.com/livekit/protocol/logger"
+)
+
+// ------------------------------------------------
+
+type channelTrend int
+
+const (
+	channelTrendInconclusive channelTrend = iota
+	channelTrendClearing
+	channelTrendCongesting
+)
+
+func (c channelTrend) String() string {
+	switch c {
+	case channelTrendInconclusive:
+		return "INCONCLUSIVE"
+	case channelTrendClearing:
+		return "CLEARING"
+	case channelTrendCongesting:
+		return "CONGESTING"
+	default:
+		return fmt.Sprintf("%d", int(c))
+	}
+}
+
+// ------------------------------------------------
+
+type channelCongestionReason int
+
+const (
+	channelCongestionReasonNone channelCongestionReason = iota
+	channelCongestionReasonEstimate
+	channelCongestionReasonLoss
+)
+
+func (c channelCongestionReason) String() string {
+	switch c {
+	case channelCongestionReasonNone:
+		return "NONE"
+	case channelCongestionReasonEstimate:
+		return "ESTIMATE"
+	case channelCongestionReasonLoss:
+		return "LOSS"
+	default:
+		return fmt.Sprintf("%d", int(c))
+	}
+}
+
+// ------------------------------------------------
+
+type ChannelObserverConfig struct {
+	Estimate ccutils.TrendDetectorConfig `yaml:"estimate,omitempty"`
+	Nack     NackTrackerConfig           `yaml:"nack,omitempty"`
+}
+
+var (
+	defaultTrendDetectorConfigProbe = ccutils.TrendDetectorConfig{
+		RequiredSamples:        3,
+		RequiredSamplesMin:     3,
+		DownwardTrendThreshold: 0.0,
+		DownwardTrendMaxWait:   5 * time.Second,
+		CollapseThreshold:      0,
+		ValidityWindow:         10 * time.Second,
+	}
+
+	defaultChannelObserverConfigProbe = ChannelObserverConfig{
+		Estimate: defaultTrendDetectorConfigProbe,
+		Nack:     defaultNackTrackerConfigProbe,
+	}
+
+	defaultTrendDetectorConfigNonProbe = ccutils.TrendDetectorConfig{
+		RequiredSamples:        12,
+		RequiredSamplesMin:     8,
+		DownwardTrendThreshold: -0.6,
+		DownwardTrendMaxWait:   5 * time.Second,
+		CollapseThreshold:      500 * time.Millisecond,
+		ValidityWindow:         10 * time.Second,
+	}
+
+	defaultChannelObserverConfigNonProbe = ChannelObserverConfig{
+		Estimate: defaultTrendDetectorConfigNonProbe,
+		Nack:     defaultNackTrackerConfigNonProbe,
+	}
+)
+
+// ------------------------------------------------
+
+type channelObserverParams struct {
+	Name   string
+	Config ChannelObserverConfig
+}
+
+type channelObserver struct {
+	params channelObserverParams
+	logger logger.Logger
+
+	estimateTrend *ccutils.TrendDetector[int64]
+	nackTracker   *nackTracker
+}
+
+func newChannelObserver(params channelObserverParams, logger logger.Logger) *channelObserver {
+	return &channelObserver{
+		params: params,
+		logger: logger,
+		estimateTrend: ccutils.NewTrendDetector[int64](ccutils.TrendDetectorParams{
+			Name:   params.Name + "-estimate",
+			Logger: logger,
+			Config: params.Config.Estimate,
+		}),
+		nackTracker: newNackTracker(nackTrackerParams{
+			Name:   params.Name + "-nack",
+			Logger: logger,
+			Config: params.Config.Nack,
+		}),
+	}
+}
+
+func (c *channelObserver) SeedEstimate(estimate int64) {
+	c.estimateTrend.Seed(estimate)
+}
+
+func (c *channelObserver) AddEstimate(estimate int64) {
+	c.estimateTrend.AddValue(estimate)
+}
+
+func (c *channelObserver) AddNack(packets uint32, repeatedNacks uint32) {
+	c.nackTracker.Add(packets, repeatedNacks)
+}
+
+func (c *channelObserver) GetLowestEstimate() int64 {
+	return c.estimateTrend.GetLowest()
+}
+
+func (c *channelObserver) GetHighestEstimate() int64 {
+	return c.estimateTrend.GetHighest()
+}
+
+func (c *channelObserver) HasEnoughEstimateSamples() bool {
+	return c.estimateTrend.HasEnoughSamples()
+}
+
+func (c *channelObserver) GetNackRatio() float64 {
+	return c.nackTracker.GetRatio()
+}
+
+func (c *channelObserver) GetTrend() (channelTrend, channelCongestionReason) {
+	estimateDirection := c.estimateTrend.GetDirection()
+
+	switch {
+	case estimateDirection == ccutils.TrendDirectionDownward:
+		return channelTrendCongesting, channelCongestionReasonEstimate
+
+	case c.nackTracker.IsTriggered():
+		return channelTrendCongesting, channelCongestionReasonLoss
+
+	case estimateDirection == ccutils.TrendDirectionUpward:
+		return channelTrendClearing, channelCongestionReasonNone
+	}
+
+	return channelTrendInconclusive, channelCongestionReasonNone
+}
+
+func (c *channelObserver) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if c == nil {
+		return nil
+	}
+
+	e.AddString("name", c.params.Name)
+	e.AddObject("estimate", c.estimateTrend)
+	e.AddObject("nack", c.nackTracker)
+
+	channelTrend, channelCongestionReason := c.GetTrend()
+	e.AddString("channelTrend", channelTrend.String())
+	e.AddString("channelCongestionReason", channelCongestionReason.String())
+	return nil
+}
+
+// ------------------------------------------------

pkg/sfu/bwe/remotebwe/nack_tracker.go

@@ -0,0 +1,130 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package remotebwe
+
+import (
+	"time"
+
+	"go.uber.org/zap/zapcore"
+
+	"github.com/livekit/protocol/logger"
+)
+
+// ------------------------------------------------
+
+type NackTrackerConfig struct {
+	WindowMinDuration time.Duration `yaml:"window_min_duration,omitempty"`
+	WindowMaxDuration time.Duration `yaml:"window_max_duration,omitempty"`
+	RatioThreshold    float64       `yaml:"ratio_threshold,omitempty"`
+}
+
+var (
+	defaultNackTrackerConfigProbe = NackTrackerConfig{
+		WindowMinDuration: 500 * time.Millisecond,
+		WindowMaxDuration: 1 * time.Second,
+		RatioThreshold:    0.04,
+	}
+
+	defaultNackTrackerConfigNonProbe = NackTrackerConfig{
+		WindowMinDuration: 2 * time.Second,
+		WindowMaxDuration: 3 * time.Second,
+		RatioThreshold:    0.08,
+	}
+)
+
+// ------------------------------------------------
+
+type nackTrackerParams struct {
+	Name   string
+	Logger logger.Logger
+	Config NackTrackerConfig
+}
+
+type nackTracker struct {
+	params nackTrackerParams
+
+	windowStartTime time.Time
+	packets         uint32
+	repeatedNacks   uint32
+}
+
+func newNackTracker(params nackTrackerParams) *nackTracker {
+	return &nackTracker{
+		params: params,
+	}
+}
+
+func (n *nackTracker) Add(packets uint32, repeatedNacks uint32) {
+	if n.params.Config.WindowMaxDuration != 0 && !n.windowStartTime.IsZero() && time.Since(n.windowStartTime) > n.params.Config.WindowMaxDuration {
+		n.windowStartTime = time.Time{}
+		n.packets = 0
+		n.repeatedNacks = 0
+	}
+
+	//
+	// Start NACK monitoring window only when a repeated NACK happens.
+	// This allows locking tightly to when NACKs start happening and
+	// check if the NACKs keep adding up (potentially a sign of congestion)
+	// or isolated losses
+	//
+	if n.repeatedNacks == 0 && repeatedNacks != 0 {
+		n.windowStartTime = time.Now()
+	}
+
+	if !n.windowStartTime.IsZero() {
+		n.packets += packets
+		n.repeatedNacks += repeatedNacks
+	}
+}
+
+func (n *nackTracker) GetRatio() float64 {
+	ratio := 0.0
+	if n.packets != 0 {
+		ratio = float64(n.repeatedNacks) / float64(n.packets)
+		if ratio > 1.0 {
+			ratio = 1.0
+		}
+	}
+
+	return ratio
+}
+
+func (n *nackTracker) IsTriggered() bool {
+	if n.params.Config.WindowMinDuration != 0 && !n.windowStartTime.IsZero() && time.Since(n.windowStartTime) > n.params.Config.WindowMinDuration {
+		return n.GetRatio() > n.params.Config.RatioThreshold
+	}
+
+	return false
+}
+
+func (n *nackTracker) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if n == nil {
+		return nil
+	}
+
+	e.AddString("name", n.params.Name)
+	if n.windowStartTime.IsZero() {
+		e.AddString("window", "inactive")
+	} else {
+		e.AddTime("windowStartTime", n.windowStartTime)
+		e.AddDuration("windowDuration", time.Since(n.windowStartTime))
+		e.AddUint32("packets", n.packets)
+		e.AddUint32("repeatedNacks", n.repeatedNacks)
+		e.AddFloat64("nackRatio", n.GetRatio())
+	}
+	return nil
+}
+
+// ------------------------------------------------

pkg/sfu/bwe/remotebwe/probe_controller.go

@@ -0,0 +1,184 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package remotebwe
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/livekit/livekit-server/pkg/sfu/bwe"
+	"github.com/livekit/livekit-server/pkg/sfu/ccutils"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils/mono"
+)
+
+// ---------------------------------------------------------------------------
+
+type probeControllerState int
+
+const (
+	probeControllerStateNone probeControllerState = iota
+	probeControllerStateProbing
+	probeControllerStateHangover
+)
+
+func (p probeControllerState) String() string {
+	switch p {
+	case probeControllerStateNone:
+		return "NONE"
+	case probeControllerStateProbing:
+		return "PROBING"
+	case probeControllerStateHangover:
+		return "HANGOVER"
+	default:
+		return fmt.Sprintf("%d", int(p))
+	}
+}
+
+// ------------------------------------------------
+
+type ProbeControllerConfig struct {
+	ProbeRegulator ccutils.ProbeRegulatorConfig `yaml:"probe_regulator,omitempty"`
+
+	SettleWaitNumRTT uint32        `yaml:"settle_wait_num_rtt,omitempty"`
+	SettleWaitMin    time.Duration `yaml:"settle_wait_min,omitempty"`
+	SettleWaitMax    time.Duration `yaml:"settle_wait_max,omitempty"`
+}
+
+var (
+	DefaultProbeControllerConfig = ProbeControllerConfig{
+		ProbeRegulator: ccutils.DefaultProbeRegulatorConfig,
+
+		SettleWaitNumRTT: 5,
+		SettleWaitMin:    250 * time.Millisecond,
+		SettleWaitMax:    5 * time.Second,
+	}
+)
+
+// ---------------------------------------------------------------------------
+
+type probeControllerParams struct {
+	Config ProbeControllerConfig
+	Logger logger.Logger
+}
+
+type probeController struct {
+	params probeControllerParams
+
+	state           probeControllerState
+	stateSwitchedAt time.Time
+
+	pci ccutils.ProbeClusterInfo
+	rtt float64
+
+	*ccutils.ProbeRegulator
+}
+
+func newProbeController(params probeControllerParams) *probeController {
+	return &probeController{
+		params:          params,
+		state:           probeControllerStateNone,
+		stateSwitchedAt: mono.Now(),
+		pci:             ccutils.ProbeClusterInfoInvalid,
+		rtt:             bwe.DefaultRTT,
+		ProbeRegulator: ccutils.NewProbeRegulator(
+			ccutils.ProbeRegulatorParams{
+				Config: params.Config.ProbeRegulator,
+				Logger: params.Logger,
+			},
+		),
+	}
+}
+
+func (p *probeController) UpdateRTT(rtt float64) {
+	if rtt == 0 {
+		p.rtt = bwe.DefaultRTT
+	} else {
+		if p.rtt == 0 {
+			p.rtt = rtt
+		} else {
+			p.rtt = bwe.RTTSmoothingFactor*rtt + (1.0-bwe.RTTSmoothingFactor)*p.rtt
+		}
+	}
+}
+
+func (p *probeController) GetRTT() float64 {
+	return p.rtt
+}
+
+func (p *probeController) CanProbe() bool {
+	return p.state == probeControllerStateNone && p.ProbeRegulator.CanProbe()
+}
+
+func (p *probeController) IsInProbe() bool {
+	return p.state != probeControllerStateNone
+}
+
+func (p *probeController) ProbeClusterStarting(pci ccutils.ProbeClusterInfo) {
+	if p.state != probeControllerStateNone {
+		p.params.Logger.Warnw("unexpected probe controller state", nil, "state", p.state)
+	}
+
+	p.setState(probeControllerStateProbing)
+	p.pci = pci
+}
+
+func (p *probeController) ProbeClusterDone(pci ccutils.ProbeClusterInfo) {
+	if p.pci.Id != pci.Id {
+		return
+	}
+
+	p.pci.Result = pci.Result
+	p.setState(probeControllerStateHangover)
+}
+
+func (p *probeController) ProbeClusterIsGoalReached(estimate int64) bool {
+	if p.pci.Id == ccutils.ProbeClusterIdInvalid {
+		return false
+	}
+
+	return estimate > int64(p.pci.Goal.DesiredBps)
+}
+
+func (p *probeController) MaybeFinalizeProbe() (ccutils.ProbeClusterInfo, bool) {
+	if p.state != probeControllerStateHangover {
+		return ccutils.ProbeClusterInfoInvalid, false
+	}
+
+	settleWait := time.Duration(float64(p.params.Config.SettleWaitNumRTT) * p.rtt * float64(time.Second))
+	if settleWait < p.params.Config.SettleWaitMin {
+		settleWait = p.params.Config.SettleWaitMin
+	}
+	if settleWait > p.params.Config.SettleWaitMax {
+		settleWait = p.params.Config.SettleWaitMax
+	}
+	if time.Since(p.stateSwitchedAt) < settleWait {
+		return ccutils.ProbeClusterInfoInvalid, false
+	}
+
+	p.setState(probeControllerStateNone)
+	return p.pci, true
+}
+
+func (p *probeController) setState(state probeControllerState) {
+	if state == p.state {
+		return
+	}
+
+	p.state = state
+	p.stateSwitchedAt = mono.Now()
+}
+
+// ------------------------------------------------

pkg/sfu/bwe/remotebwe/remote_bwe.go

@@ -0,0 +1,359 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package remotebwe
+
+import (
+	"sync"
+	"time"
+
+	"github.com/livekit/livekit-server/pkg/sfu/bwe"
+	"github.com/livekit/livekit-server/pkg/sfu/ccutils"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils/mono"
+)
+
+var _ bwe.BWE = (*RemoteBWE)(nil)
+
+// ---------------------------------------------------------------------------
+
+type RemoteBWEConfig struct {
+	NackRatioAttenuator     float64               `yaml:"nack_ratio_attenuator,omitempty"`
+	ExpectedUsageThreshold  float64               `yaml:"expected_usage_threshold,omitempty"`
+	ChannelObserverProbe    ChannelObserverConfig `yaml:"channel_observer_probe,omitempty"`
+	ChannelObserverNonProbe ChannelObserverConfig `yaml:"channel_observer_non_probe,omitempty"`
+	ProbeController         ProbeControllerConfig `yaml:"probe_controller,omitempty"`
+}
+
+var (
+	DefaultRemoteBWEConfig = RemoteBWEConfig{
+		NackRatioAttenuator:     0.4,
+		ExpectedUsageThreshold:  0.95,
+		ChannelObserverProbe:    defaultChannelObserverConfigProbe,
+		ChannelObserverNonProbe: defaultChannelObserverConfigNonProbe,
+		ProbeController:         DefaultProbeControllerConfig,
+	}
+)
+
+// ---------------------------------------------------------------------------
+
+type RemoteBWEParams struct {
+	Config RemoteBWEConfig
+	Logger logger.Logger
+}
+
+type RemoteBWE struct {
+	bwe.NullBWE
+
+	params RemoteBWEParams
+
+	lock sync.RWMutex
+
+	lastReceivedEstimate       int64
+	lastExpectedBandwidthUsage int64
+	committedChannelCapacity   int64
+
+	probeController *probeController
+
+	channelObserver *channelObserver
+
+	congestionState           bwe.CongestionState
+	congestionStateSwitchedAt time.Time
+
+	bweListener bwe.BWEListener
+}
+
+func NewRemoteBWE(params RemoteBWEParams) *RemoteBWE {
+	r := &RemoteBWE{
+		params: params,
+	}
+
+	r.Reset()
+	return r
+}
+
+func (r *RemoteBWE) Type() bwe.BWEType {
+	return bwe.BWETypeRemote
+}
+
+func (r *RemoteBWE) SetBWEListener(bweListener bwe.BWEListener) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	r.bweListener = bweListener
+}
+
+func (r *RemoteBWE) getBWEListener() bwe.BWEListener {
+	r.lock.RLock()
+	defer r.lock.RUnlock()
+
+	return r.bweListener
+}
+
+func (r *RemoteBWE) Reset() {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	r.lastReceivedEstimate = 0
+	r.lastExpectedBandwidthUsage = 0
+	r.committedChannelCapacity = 100_000_000
+
+	r.congestionState = bwe.CongestionStateNone
+	r.congestionStateSwitchedAt = mono.Now()
+
+	r.probeController = newProbeController(probeControllerParams{
+		Config: r.params.Config.ProbeController,
+		Logger: r.params.Logger,
+	})
+
+	r.newChannelObserver()
+}
+
+func (r *RemoteBWE) HandleREMB(
+	receivedEstimate int64,
+	expectedBandwidthUsage int64,
+	sentPackets uint32,
+	repeatedNacks uint32,
+) {
+	r.lock.Lock()
+	r.lastReceivedEstimate = receivedEstimate
+	r.lastExpectedBandwidthUsage = expectedBandwidthUsage
+
+	// in probe, freeze channel observer state if probe causes congestion till the probe is done,
+	// this is to ensure that probe result is not marked a success,
+	// an unsuccessful probe will not up allocate any tracks
+	if r.congestionState != bwe.CongestionStateNone && r.probeController.IsInProbe() {
+		r.lock.Unlock()
+		return
+	}
+
+	r.channelObserver.AddEstimate(r.lastReceivedEstimate)
+	r.channelObserver.AddNack(sentPackets, repeatedNacks)
+
+	shouldNotify, fromState, toState, committedChannelCapacity := r.congestionDetectionStateMachine()
+	r.lock.Unlock()
+
+	if shouldNotify {
+		if bweListener := r.getBWEListener(); bweListener != nil {
+			bweListener.OnCongestionStateChange(fromState, toState, committedChannelCapacity)
+		}
+	}
+}
+
+func (r *RemoteBWE) UpdateRTT(rtt float64) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	r.probeController.UpdateRTT(rtt)
+}
+
+func (r *RemoteBWE) congestionDetectionStateMachine() (bool, bwe.CongestionState, bwe.CongestionState, int64) {
+	fromState := r.congestionState
+	toState := r.congestionState
+	update := false
+	trend, reason := r.channelObserver.GetTrend()
+
+	switch fromState {
+	case bwe.CongestionStateNone:
+		if trend == channelTrendCongesting {
+			if r.probeController.IsInProbe() || r.estimateAvailableChannelCapacity(reason) {
+				// when in probe, if congested, stays there till probe is done,
+				// the estimate stays at pre-probe level
+				toState = bwe.CongestionStateCongested
+			}
+		}
+
+	case bwe.CongestionStateCongested:
+		if trend == channelTrendCongesting {
+			if r.estimateAvailableChannelCapacity(reason) {
+				// update state as this needs to reset switch time to wait for congestion min duration again
+				update = true
+			}
+		} else {
+			toState = bwe.CongestionStateNone
+		}
+	}
+
+	shouldNotify := false
+	if toState != fromState || update {
+		fromState, toState = r.updateCongestionState(toState, reason)
+		shouldNotify = true
+	}
+
+	return shouldNotify, fromState, toState, r.committedChannelCapacity
+}
+
+func (r *RemoteBWE) estimateAvailableChannelCapacity(reason channelCongestionReason) bool {
+	var estimateToCommit int64
+	switch reason {
+	case channelCongestionReasonLoss:
+		estimateToCommit = int64(float64(r.lastExpectedBandwidthUsage) * (1.0 - r.params.Config.NackRatioAttenuator*r.channelObserver.GetNackRatio()))
+	default:
+		estimateToCommit = r.lastReceivedEstimate
+	}
+	if estimateToCommit > r.lastReceivedEstimate {
+		estimateToCommit = r.lastReceivedEstimate
+	}
+
+	commitThreshold := int64(r.params.Config.ExpectedUsageThreshold * float64(r.lastExpectedBandwidthUsage))
+	if estimateToCommit > commitThreshold || r.committedChannelCapacity == estimateToCommit {
+		return false
+	}
+
+	r.params.Logger.Infow(
+		"remote bwe: channel congestion detected, applying channel capacity update",
+		"reason", reason,
+		"old(bps)", r.committedChannelCapacity,
+		"new(bps)", estimateToCommit,
+		"lastReceived(bps)", r.lastReceivedEstimate,
+		"expectedUsage(bps)", r.lastExpectedBandwidthUsage,
+		"commitThreshold(bps)", commitThreshold,
+		"channel", r.channelObserver,
+	)
+	r.committedChannelCapacity = estimateToCommit
+	return true
+}
+
+func (r *RemoteBWE) updateCongestionState(state bwe.CongestionState, reason channelCongestionReason) (bwe.CongestionState, bwe.CongestionState) {
+	r.params.Logger.Debugw(
+		"remote bwe: congestion state change",
+		"from", r.congestionState,
+		"to", state,
+		"reason", reason,
+		"committedChannelCapacity", r.committedChannelCapacity,
+	)
+
+	fromState := r.congestionState
+	r.congestionState = state
+	r.congestionStateSwitchedAt = mono.Now()
+	return fromState, r.congestionState
+}
+
+func (r *RemoteBWE) CongestionState() bwe.CongestionState {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	return r.congestionState
+}
+
+func (r *RemoteBWE) CanProbe() bool {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	return r.congestionState == bwe.CongestionStateNone && r.probeController.CanProbe()
+}
+
+func (r *RemoteBWE) ProbeDuration() time.Duration {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	return r.probeController.ProbeDuration()
+}
+
+func (r *RemoteBWE) ProbeClusterStarting(pci ccutils.ProbeClusterInfo) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	r.lastExpectedBandwidthUsage = int64(pci.Goal.ExpectedUsageBps)
+
+	r.params.Logger.Debugw(
+		"remote bwe: starting probe",
+		"lastReceived", r.lastReceivedEstimate,
+		"expectedBandwidthUsage", r.lastExpectedBandwidthUsage,
+		"channel", r.channelObserver,
+	)
+
+	r.probeController.ProbeClusterStarting(pci)
+	r.newChannelObserver()
+}
+
+func (r *RemoteBWE) ProbeClusterDone(pci ccutils.ProbeClusterInfo) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	r.probeController.ProbeClusterDone(pci)
+}
+
+func (r *RemoteBWE) ProbeClusterIsGoalReached() bool {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	if !r.probeController.IsInProbe() ||
+		r.congestionState != bwe.CongestionStateNone ||
+		!r.channelObserver.HasEnoughEstimateSamples() {
+		return false
+	}
+
+	return r.probeController.ProbeClusterIsGoalReached(r.channelObserver.GetHighestEstimate())
+}
+
+func (r *RemoteBWE) ProbeClusterFinalize() (ccutils.ProbeSignal, int64, bool) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	pci, isFinalized := r.probeController.MaybeFinalizeProbe()
+	if !isFinalized {
+		return ccutils.ProbeSignalInconclusive, 0, isFinalized
+	}
+
+	// switch to a non-probe channel observer on probe end,
+	// reset congestion state to get a fresh trend
+	pco := r.channelObserver
+	probeCongestionState := r.congestionState
+
+	r.congestionState = bwe.CongestionStateNone
+	r.newChannelObserver()
+
+	r.params.Logger.Infow(
+		"remote bwe: probe finalized",
+		"lastReceived", r.lastReceivedEstimate,
+		"expectedBandwidthUsage", r.lastExpectedBandwidthUsage,
+		"channel", pco,
+		"isSignalValid", pco.HasEnoughEstimateSamples(),
+		"probeClusterInfo", pci,
+		"rtt", r.probeController.GetRTT(),
+	)
+
+	probeSignal := ccutils.ProbeSignalNotCongesting
+	if probeCongestionState != bwe.CongestionStateNone {
+		probeSignal = ccutils.ProbeSignalCongesting
+	} else if !pco.HasEnoughEstimateSamples() {
+		probeSignal = ccutils.ProbeSignalInconclusive
+	} else {
+		highestEstimate := pco.GetHighestEstimate()
+		if highestEstimate > r.committedChannelCapacity {
+			r.committedChannelCapacity = highestEstimate
+		}
+	}
+
+	r.probeController.ProbeSignal(probeSignal, pci.CreatedAt)
+	return probeSignal, r.committedChannelCapacity, true
+}
+
+func (r *RemoteBWE) newChannelObserver() {
+	var params channelObserverParams
+	if r.probeController.IsInProbe() {
+		params = channelObserverParams{
+			Name:   "probe",
+			Config: r.params.Config.ChannelObserverProbe,
+		}
+	} else {
+		params = channelObserverParams{
+			Name:   "non-probe",
+			Config: r.params.Config.ChannelObserverNonProbe,
+		}
+	}
+
+	r.channelObserver = newChannelObserver(params, r.params.Logger)
+}

pkg/sfu/bwe/sendsidebwe/packet_group.go

@@ -0,0 +1,310 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sendsidebwe
+
+import (
+	"errors"
+	"time"
+
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils"
+	"go.uber.org/zap/zapcore"
+)
+
+// -------------------------------------------------------------
+
+var (
+	errGroupFinalized = errors.New("packet group is finalized")
+	errOldPacket      = errors.New("packet is older than packet group start")
+)
+
+// -------------------------------------------------------------
+
+type PacketGroupConfig struct {
+	MinPackets        int           `yaml:"min_packets,omitempty"`
+	MaxWindowDuration time.Duration `yaml:"max_window_duration,omitempty"`
+}
+
+var (
+	defaultPacketGroupConfig = PacketGroupConfig{
+		MinPackets:        30,
+		MaxWindowDuration: 500 * time.Millisecond,
+	}
+)
+
+// -------------------------------------------------------------
+
+type stat struct {
+	numPackets int
+	numBytes   int
+}
+
+func (s *stat) add(size int) {
+	s.numPackets++
+	s.numBytes += size
+}
+
+func (s *stat) remove(size int) {
+	s.numPackets--
+	s.numBytes -= size
+}
+
+func (s *stat) getNumPackets() int {
+	return s.numPackets
+}
+
+func (s *stat) getNumBytes() int {
+	return s.numBytes
+}
+
+func (s stat) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	e.AddInt("numPackets", s.numPackets)
+	e.AddInt("numBytes", s.numBytes)
+	return nil
+}
+
+// -------------------------------------------------------------
+
+type classStat struct {
+	primary stat
+	rtx     stat
+	probe   stat
+}
+
+func (c *classStat) add(size int, isRTX bool, isProbe bool) {
+	if isRTX {
+		c.rtx.add(size)
+	} else if isProbe {
+		c.probe.add(size)
+	} else {
+		c.primary.add(size)
+	}
+}
+
+func (c *classStat) remove(size int, isRTX bool, isProbe bool) {
+	if isRTX {
+		c.rtx.remove(size)
+	} else if isProbe {
+		c.probe.remove(size)
+	} else {
+		c.primary.remove(size)
+	}
+}
+
+func (c *classStat) numPackets() int {
+	return c.primary.getNumPackets() + c.rtx.getNumPackets() + c.probe.getNumPackets()
+}
+
+func (c *classStat) numBytes() int {
+	return c.primary.getNumBytes() + c.rtx.getNumBytes() + c.probe.getNumBytes()
+}
+
+func (c classStat) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	e.AddObject("primary", c.primary)
+	e.AddObject("rtx", c.rtx)
+	e.AddObject("probe", c.probe)
+	return nil
+}
+
+// -------------------------------------------------------------
+
+type packetGroupParams struct {
+	Config       PacketGroupConfig
+	WeightedLoss WeightedLossConfig
+	Logger       logger.Logger
+}
+
+type packetGroup struct {
+	params packetGroupParams
+
+	minSequenceNumber uint64
+	maxSequenceNumber uint64
+
+	minSendTime int64
+	maxSendTime int64
+
+	minRecvTime int64 // for information only
+	maxRecvTime int64 // for information only
+
+	acked    classStat
+	lost     classStat
+	snBitmap *utils.Bitmap[uint64]
+
+	aggregateSendDelta    int64
+	aggregateRecvDelta    int64
+	inheritedQueuingDelay int64
+
+	isFinalized bool
+}
+
+func newPacketGroup(params packetGroupParams, inheritedQueuingDelay int64) *packetGroup {
+	return &packetGroup{
+		params:                params,
+		inheritedQueuingDelay: inheritedQueuingDelay,
+		snBitmap:              utils.NewBitmap[uint64](params.Config.MinPackets),
+	}
+}
+
+func (p *packetGroup) Add(pi *packetInfo, sendDelta, recvDelta int64, isLost bool) error {
+	if isLost {
+		return p.lostPacket(pi)
+	}
+
+	if err := p.inGroup(pi.sequenceNumber); err != nil {
+		return err
+	}
+
+	if p.minSequenceNumber == 0 || pi.sequenceNumber < p.minSequenceNumber {
+		p.minSequenceNumber = pi.sequenceNumber
+	}
+	p.maxSequenceNumber = max(p.maxSequenceNumber, pi.sequenceNumber)
+
+	if p.minSendTime == 0 || (pi.sendTime-sendDelta) < p.minSendTime {
+		p.minSendTime = pi.sendTime - sendDelta
+	}
+	p.maxSendTime = max(p.maxSendTime, pi.sendTime)
+
+	if p.minRecvTime == 0 || (pi.recvTime-recvDelta) < p.minRecvTime {
+		p.minRecvTime = pi.recvTime - recvDelta
+	}
+	p.maxRecvTime = max(p.maxRecvTime, pi.recvTime)
+
+	p.acked.add(int(pi.size), pi.isRTX, pi.isProbe)
+	if int(pi.sequenceNumber-p.minSequenceNumber) < p.snBitmap.Len() && p.snBitmap.IsSet(pi.sequenceNumber-p.minSequenceNumber) {
+		// an earlier packet reported as lost has been received
+		p.snBitmap.Clear(pi.sequenceNumber - p.minSequenceNumber)
+		p.lost.remove(int(pi.size), pi.isRTX, pi.isProbe)
+	}
+
+	// note that out-of-order deliveries will amplify the queueing delay.
+	// for e.g. a, b, c getting delivered as a, c, b.
+	// let us say packets are delivered with interval of `x`
+	//   send delta aggregate will go up by x((a, c) = 2x + (c, b) -1x)
+	//   recv delta aggregate will go up by 3x((a, c) = 2x + (c, b) 1x)
+	p.aggregateSendDelta += sendDelta
+	p.aggregateRecvDelta += recvDelta
+
+	if p.acked.numPackets() == p.params.Config.MinPackets || (pi.sendTime-p.minSendTime) > p.params.Config.MaxWindowDuration.Microseconds() {
+		p.isFinalized = true
+	}
+	return nil
+}
+
+func (p *packetGroup) lostPacket(pi *packetInfo) error {
+	if pi.recvTime != 0 {
+		// previously received packet, so not lost
+		return nil
+	}
+
+	if err := p.inGroup(pi.sequenceNumber); err != nil {
+		return err
+	}
+
+	if p.minSequenceNumber == 0 || pi.sequenceNumber < p.minSequenceNumber {
+		p.minSequenceNumber = pi.sequenceNumber
+	}
+	p.maxSequenceNumber = max(p.maxSequenceNumber, pi.sequenceNumber)
+	p.snBitmap.Set(pi.sequenceNumber - p.minSequenceNumber)
+
+	p.lost.add(int(pi.size), pi.isRTX, pi.isProbe)
+	return nil
+}
+
+func (p *packetGroup) MinSendTime() int64 {
+	return p.minSendTime
+}
+
+func (p *packetGroup) SendWindow() (int64, int64) {
+	return p.minSendTime, p.maxSendTime
+}
+
+func (p *packetGroup) PropagatedQueuingDelay() int64 {
+	if p.inheritedQueuingDelay+p.aggregateRecvDelta-p.aggregateSendDelta > 0 {
+		return p.inheritedQueuingDelay + p.aggregateRecvDelta - p.aggregateSendDelta
+	}
+
+	return max(0, p.aggregateRecvDelta-p.aggregateSendDelta)
+}
+
+func (p *packetGroup) FinalizedPropagatedQueuingDelay() (int64, bool) {
+	if !p.isFinalized {
+		return 0, false
+	}
+
+	return p.PropagatedQueuingDelay(), true
+}
+
+func (p *packetGroup) IsFinalized() bool {
+	return p.isFinalized
+}
+
+func (p *packetGroup) Traffic() *trafficStats {
+	return &trafficStats{
+		minSendTime:  p.minSendTime,
+		maxSendTime:  p.maxSendTime,
+		sendDelta:    p.aggregateSendDelta,
+		recvDelta:    p.aggregateRecvDelta,
+		ackedPackets: p.acked.numPackets(),
+		ackedBytes:   p.acked.numBytes(),
+		lostPackets:  p.lost.numPackets(),
+		lostBytes:    p.lost.numBytes(),
+	}
+}
+
+func (p *packetGroup) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if p == nil {
+		return nil
+	}
+
+	e.AddUint64("minSequenceNumber", p.minSequenceNumber)
+	e.AddUint64("maxSequenceNumber", p.maxSequenceNumber)
+	e.AddObject("acked", p.acked)
+	e.AddObject("lost", p.lost)
+
+	e.AddInt64("minRecvTime", p.minRecvTime)
+	e.AddInt64("maxRecvTime", p.maxRecvTime)
+	recvDuration := time.Duration((p.maxRecvTime - p.minRecvTime) * 1000)
+	e.AddDuration("recvDuration", recvDuration)
+
+	recvBitrate := float64(0)
+	if recvDuration != 0 {
+		recvBitrate = float64(p.acked.numBytes()*8) / recvDuration.Seconds()
+		e.AddFloat64("recvBitrate", recvBitrate)
+	}
+
+	ts := newTrafficStats(trafficStatsParams{
+		Config: p.params.WeightedLoss,
+		Logger: p.params.Logger,
+	})
+	ts.Merge(p.Traffic())
+	e.AddObject("trafficStats", ts)
+	e.AddInt64("inheritedQueuingDelay", p.inheritedQueuingDelay)
+	e.AddInt64("propagatedQueuingDelay", p.PropagatedQueuingDelay())
+
+	e.AddBool("isFinalized", p.isFinalized)
+	return nil
+}
+
+func (p *packetGroup) inGroup(sequenceNumber uint64) error {
+	if p.isFinalized && sequenceNumber > p.maxSequenceNumber {
+		return errGroupFinalized
+	}
+
+	if sequenceNumber < p.minSequenceNumber {
+		return errOldPacket
+	}
+
+	return nil
+}

pkg/sfu/bwe/sendsidebwe/packet_info.go

@@ -0,0 +1,45 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sendsidebwe
+
+import (
+	"github.com/livekit/livekit-server/pkg/sfu/ccutils"
+	"go.uber.org/zap/zapcore"
+)
+
+type packetInfo struct {
+	sequenceNumber uint64
+	sendTime       int64
+	recvTime       int64
+	probeClusterId ccutils.ProbeClusterId
+	size           uint16
+	isRTX          bool
+	isProbe        bool
+}
+
+func (pi *packetInfo) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if pi == nil {
+		return nil
+	}
+
+	e.AddUint64("sequenceNumber", pi.sequenceNumber)
+	e.AddInt64("sendTime", pi.sendTime)
+	e.AddInt64("recvTime", pi.recvTime)
+	e.AddUint32("probeClusterId", uint32(pi.probeClusterId))
+	e.AddUint16("size", pi.size)
+	e.AddBool("isRTX", pi.isRTX)
+	e.AddBool("isProbe", pi.isProbe)
+	return nil
+}

pkg/sfu/bwe/sendsidebwe/packet_tracker.go

@@ -0,0 +1,168 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sendsidebwe
+
+import (
+	"math/rand"
+	"sync"
+
+	"github.com/livekit/livekit-server/pkg/sfu/ccutils"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils/mono"
+)
+
+// -------------------------------------------------------------------------------
+
+type packetTrackerParams struct {
+	Logger logger.Logger
+}
+
+type packetTracker struct {
+	params packetTrackerParams
+
+	lock sync.Mutex
+
+	sequenceNumber uint64
+
+	baseSendTime int64
+	packetInfos  [2048]packetInfo
+
+	baseRecvTime int64
+	piLastRecv   *packetInfo
+
+	probeClusterId         ccutils.ProbeClusterId
+	probeMaxSequenceNumber uint64
+}
+
+func newPacketTracker(params packetTrackerParams) *packetTracker {
+	return &packetTracker{
+		params:         params,
+		sequenceNumber: uint64(rand.Intn(1<<14)) + uint64(1<<15), // a random number in third quartile of sequence number space
+	}
+}
+
+func (p *packetTracker) RecordPacketSendAndGetSequenceNumber(
+	atMicro int64,
+	size int,
+	isRTX bool,
+	probeClusterId ccutils.ProbeClusterId,
+	isProbe bool,
+) uint16 {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	if p.baseSendTime == 0 {
+		p.baseSendTime = atMicro
+	}
+
+	pi := p.getPacketInfo(uint16(p.sequenceNumber))
+	*pi = packetInfo{
+		sequenceNumber: p.sequenceNumber,
+		sendTime:       atMicro - p.baseSendTime,
+		size:           uint16(size),
+		isRTX:          isRTX,
+		probeClusterId: probeClusterId,
+		isProbe:        isProbe,
+	}
+
+	p.sequenceNumber++
+
+	// extreme case of wrap around before receiving any feedback
+	if pi == p.piLastRecv {
+		p.piLastRecv = nil
+	}
+
+	if p.probeClusterId != ccutils.ProbeClusterIdInvalid && p.probeClusterId == pi.probeClusterId && pi.sequenceNumber > p.probeMaxSequenceNumber {
+		p.probeMaxSequenceNumber = pi.sequenceNumber
+	}
+
+	return uint16(pi.sequenceNumber)
+}
+
+func (p *packetTracker) BaseSendTimeThreshold(threshold int64) (int64, bool) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	if p.baseSendTime == 0 {
+		return 0, false
+	}
+
+	return mono.UnixMicro() - p.baseSendTime - threshold, true
+}
+
+func (p *packetTracker) RecordPacketIndicationFromRemote(sn uint16, recvTime int64) (piRecv packetInfo, sendDelta, recvDelta int64) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	pi := p.getPacketInfoExisting(sn)
+	if pi == nil {
+		return
+	}
+
+	if recvTime == 0 {
+		// maybe lost OR already received but reported lost in a later report
+		piRecv = *pi
+		return
+	}
+
+	if p.baseRecvTime == 0 {
+		p.baseRecvTime = recvTime
+		p.piLastRecv = pi
+	}
+
+	pi.recvTime = recvTime - p.baseRecvTime
+	piRecv = *pi
+	if p.piLastRecv != nil {
+		sendDelta, recvDelta = pi.sendTime-p.piLastRecv.sendTime, pi.recvTime-p.piLastRecv.recvTime
+	}
+	p.piLastRecv = pi
+	return
+}
+
+func (p *packetTracker) getPacketInfo(sn uint16) *packetInfo {
+	return &p.packetInfos[int(sn)%len(p.packetInfos)]
+}
+
+func (p *packetTracker) getPacketInfoExisting(sn uint16) *packetInfo {
+	pi := &p.packetInfos[int(sn)%len(p.packetInfos)]
+	if uint16(pi.sequenceNumber) == sn {
+		return pi
+	}
+
+	return nil
+}
+
+func (p *packetTracker) ProbeClusterStarting(probeClusterId ccutils.ProbeClusterId) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	p.probeClusterId = probeClusterId
+}
+
+func (p *packetTracker) ProbeClusterDone(probeClusterId ccutils.ProbeClusterId) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	if p.probeClusterId == probeClusterId {
+		p.probeClusterId = ccutils.ProbeClusterIdInvalid
+	}
+}
+
+func (p *packetTracker) ProbeMaxSequenceNumber() uint64 {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	return p.probeMaxSequenceNumber
+}

pkg/sfu/bwe/sendsidebwe/probe_packet_group.go

@@ -0,0 +1,137 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sendsidebwe
+
+import (
+	"time"
+
+	"github.com/livekit/livekit-server/pkg/sfu/ccutils"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils/mono"
+	"go.uber.org/zap/zapcore"
+)
+
+// -------------------------------------------------------------
+
+type ProbePacketGroupConfig struct {
+	PacketGroup PacketGroupConfig `yaml:"packet_group,omitempty"`
+
+	SettleWaitNumRTT uint32        `yaml:"settle_wait_num_rtt,omitempty"`
+	SettleWaitMin    time.Duration `yaml:"settle_wait_min,omitempty"`
+	SettleWaitMax    time.Duration `yaml:"settle_wait_max,omitempty"`
+}
+
+var (
+	// large numbers to treat a probe packet group as one
+	defaultProbePacketGroupConfig = ProbePacketGroupConfig{
+		PacketGroup: PacketGroupConfig{
+			MinPackets:        16384,
+			MaxWindowDuration: time.Minute,
+		},
+
+		SettleWaitNumRTT: 5,
+		SettleWaitMin:    250 * time.Millisecond,
+		SettleWaitMax:    5 * time.Second,
+	}
+)
+
+// -------------------------------------------------------------
+
+type probePacketGroupParams struct {
+	Config       ProbePacketGroupConfig
+	WeightedLoss WeightedLossConfig
+	Logger       logger.Logger
+}
+
+type probePacketGroup struct {
+	params probePacketGroupParams
+	pci    ccutils.ProbeClusterInfo
+	*packetGroup
+	maxSequenceNumber uint64
+	doneAt            time.Time
+}
+
+func newProbePacketGroup(params probePacketGroupParams, pci ccutils.ProbeClusterInfo) *probePacketGroup {
+	return &probePacketGroup{
+		params: params,
+		pci:    pci,
+		packetGroup: newPacketGroup(
+			packetGroupParams{
+				Config:       params.Config.PacketGroup,
+				WeightedLoss: params.WeightedLoss,
+				Logger:       params.Logger,
+			},
+			0,
+		),
+	}
+}
+
+func (p *probePacketGroup) ProbeClusterDone(pci ccutils.ProbeClusterInfo) {
+	if p.pci.Id != pci.Id {
+		return
+	}
+
+	p.pci.Result = pci.Result
+	p.doneAt = mono.Now()
+}
+
+func (p *probePacketGroup) ProbeClusterInfo() ccutils.ProbeClusterInfo {
+	return p.pci
+}
+
+func (p *probePacketGroup) MaybeFinalizeProbe(maxSequenceNumber uint64, rtt float64) (ccutils.ProbeClusterInfo, bool) {
+	if p.doneAt.IsZero() {
+		return ccutils.ProbeClusterInfoInvalid, false
+	}
+
+	if maxSequenceNumber != 0 && p.maxSequenceNumber >= maxSequenceNumber {
+		return p.pci, true
+	}
+
+	settleWait := time.Duration(float64(p.params.Config.SettleWaitNumRTT) * rtt * float64(time.Second))
+	if settleWait < p.params.Config.SettleWaitMin {
+		settleWait = p.params.Config.SettleWaitMin
+	}
+	if settleWait > p.params.Config.SettleWaitMax {
+		settleWait = p.params.Config.SettleWaitMax
+	}
+	if time.Since(p.doneAt) < settleWait {
+		return ccutils.ProbeClusterInfoInvalid, false
+	}
+
+	return p.pci, true
+}
+
+func (p *probePacketGroup) Add(pi *packetInfo, sendDelta, recvDelta int64, isLost bool) error {
+	if pi.probeClusterId != p.pci.Id {
+		return nil
+	}
+
+	p.maxSequenceNumber = max(p.maxSequenceNumber, pi.sequenceNumber)
+
+	return p.packetGroup.Add(pi, sendDelta, recvDelta, isLost)
+}
+
+func (p *probePacketGroup) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if p == nil {
+		return nil
+	}
+
+	e.AddObject("pci", p.pci)
+	e.AddObject("packetGroup", p.packetGroup)
+	e.AddUint64("maxSequenceNumber", p.maxSequenceNumber)
+	e.AddTime("doneAt", p.doneAt)
+	return nil
+}

pkg/sfu/bwe/sendsidebwe/send_side_bwe.go

@@ -0,0 +1,151 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sendsidebwe
+
+import (
+	"time"
+
+	"github.com/livekit/livekit-server/pkg/sfu/bwe"
+	"github.com/livekit/livekit-server/pkg/sfu/ccutils"
+	"github.com/livekit/protocol/logger"
+	"github.com/pion/rtcp"
+)
+
+var _ bwe.BWE = (*SendSideBWE)(nil)
+
+//
+// Based on a simplified/modified version of JitterPath paper
+// (https://homepage.iis.sinica.edu.tw/papers/lcs/2114-F.pdf)
+//
+// TWCC feedback is used to calculate delta one-way-delay.
+// It is accumulated/propagated to determine in which region
+// groups of packets are operating in.
+//
+// In simplified terms,
+//   o JQR (Join Queuing Region) is when channel is congested.
+//   o DQR (Disjoint Queuing Region) is when channel is not.
+//
+// Packets are grouped and thresholds applied to smooth over
+// small variations. For example, in the paper,
+//    if propagated_queuing_delay + delta_one_way_delay > 0 {
+//       possibly_operating_in_jqr
+//    }
+// But, in this implementation it is checked at packet group level,
+// i. e. using queuing delay and aggregated delta one-way-delay of
+// the group and a minimum value threshold is applied before declaring
+// that a group is in JQR.
+//
+// There is also hysteresis to make transisitons smoother, i.e. if the
+// metric is above a certain threshold, it is JQR and it is DQR only if it
+// is below a certain value and the gap in between those two thresholds
+// are treated as interdeterminate groups.
+//
+
+// ---------------------------------------------------------------------------
+
+type SendSideBWEConfig struct {
+	CongestionDetector CongestionDetectorConfig `yaml:"congestion_detector,omitempty"`
+}
+
+var (
+	DefaultSendSideBWEConfig = SendSideBWEConfig{
+		CongestionDetector: defaultCongestionDetectorConfig,
+	}
+)
+
+// ---------------------------------------------------------------------------
+
+type SendSideBWEParams struct {
+	Config SendSideBWEConfig
+	Logger logger.Logger
+}
+
+type SendSideBWE struct {
+	bwe.NullBWE
+
+	params SendSideBWEParams
+
+	*congestionDetector
+}
+
+func NewSendSideBWE(params SendSideBWEParams) *SendSideBWE {
+	return &SendSideBWE{
+		params: params,
+		congestionDetector: newCongestionDetector(congestionDetectorParams{
+			Config: params.Config.CongestionDetector,
+			Logger: params.Logger,
+		}),
+	}
+}
+
+func (s *SendSideBWE) Type() bwe.BWEType {
+	return bwe.BWETypeSendSide
+}
+
+func (s *SendSideBWE) SetBWEListener(bweListener bwe.BWEListener) {
+	s.congestionDetector.SetBWEListener(bweListener)
+}
+
+func (s *SendSideBWE) Reset() {
+	s.congestionDetector.Reset()
+}
+
+func (s *SendSideBWE) RecordPacketSendAndGetSequenceNumber(
+	atMicro int64,
+	size int,
+	isRTX bool,
+	probeClusterId ccutils.ProbeClusterId,
+	isProbe bool,
+) uint16 {
+	return s.congestionDetector.RecordPacketSendAndGetSequenceNumber(atMicro, size, isRTX, probeClusterId, isProbe)
+}
+
+func (s *SendSideBWE) HandleTWCCFeedback(report *rtcp.TransportLayerCC) {
+	s.congestionDetector.HandleTWCCFeedback(report)
+}
+
+func (s *SendSideBWE) UpdateRTT(rtt float64) {
+	s.congestionDetector.UpdateRTT(rtt)
+}
+
+func (s *SendSideBWE) CongestionState() bwe.CongestionState {
+	return s.congestionDetector.CongestionState()
+}
+
+func (s *SendSideBWE) CanProbe() bool {
+	return s.congestionDetector.CanProbe()
+}
+
+func (s *SendSideBWE) ProbeDuration() time.Duration {
+	return s.congestionDetector.ProbeDuration()
+}
+
+func (s *SendSideBWE) ProbeClusterStarting(pci ccutils.ProbeClusterInfo) {
+	s.congestionDetector.ProbeClusterStarting(pci)
+}
+
+func (s *SendSideBWE) ProbeClusterDone(pci ccutils.ProbeClusterInfo) {
+	s.congestionDetector.ProbeClusterDone(pci)
+}
+
+func (s *SendSideBWE) ProbeClusterIsGoalReached() bool {
+	return s.congestionDetector.ProbeClusterIsGoalReached()
+}
+
+func (s *SendSideBWE) ProbeClusterFinalize() (ccutils.ProbeSignal, int64, bool) {
+	return s.congestionDetector.ProbeClusterFinalize()
+}
+
+// ------------------------------------------------

pkg/sfu/bwe/sendsidebwe/traffic_stats.go

@@ -0,0 +1,191 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sendsidebwe
+
+import (
+	"math"
+	"time"
+
+	"github.com/livekit/protocol/logger"
+	"go.uber.org/zap/zapcore"
+)
+
+// -----------------------------------------------------------
+
+type WeightedLossConfig struct {
+	MinDurationForLossValidity time.Duration `yaml:"min_duration_for_loss_validity,omitempty"`
+	BaseDuration               time.Duration `yaml:"base_duration,omitempty"`
+	BasePPS                    int           `yaml:"base_pps,omitempty"`
+	LossPenaltyFactor          float64       `yaml:"loss_penalty_factor,omitempty"`
+}
+
+var (
+	defaultWeightedLossConfig = WeightedLossConfig{
+		MinDurationForLossValidity: 100 * time.Millisecond,
+		BaseDuration:               500 * time.Millisecond,
+		BasePPS:                    30,
+		LossPenaltyFactor:          0.25,
+	}
+)
+
+// -----------------------------------------------------------
+
+type trafficStatsParams struct {
+	Config WeightedLossConfig
+	Logger logger.Logger
+}
+
+type trafficStats struct {
+	params trafficStatsParams
+
+	minSendTime  int64
+	maxSendTime  int64
+	sendDelta    int64
+	recvDelta    int64
+	ackedPackets int
+	ackedBytes   int
+	lostPackets  int
+	lostBytes    int
+}
+
+func newTrafficStats(params trafficStatsParams) *trafficStats {
+	return &trafficStats{
+		params: params,
+	}
+}
+
+func (ts *trafficStats) Merge(rhs *trafficStats) {
+	if ts.minSendTime == 0 || rhs.minSendTime < ts.minSendTime {
+		ts.minSendTime = rhs.minSendTime
+	}
+	if rhs.maxSendTime > ts.maxSendTime {
+		ts.maxSendTime = rhs.maxSendTime
+	}
+	ts.sendDelta += rhs.sendDelta
+	ts.recvDelta += rhs.recvDelta
+	ts.ackedPackets += rhs.ackedPackets
+	ts.ackedBytes += rhs.ackedBytes
+	ts.lostPackets += rhs.lostPackets
+	ts.lostBytes += rhs.lostBytes
+}
+
+func (ts *trafficStats) NumBytes() int {
+	return ts.ackedBytes + ts.lostBytes
+}
+
+func (ts *trafficStats) Duration() int64 {
+	return ts.maxSendTime - ts.minSendTime
+}
+
+func (ts *trafficStats) AcknowledgedBitrate() int64 {
+	duration := ts.Duration()
+	if duration == 0 {
+		return 0
+	}
+
+	ackedBitrate := float64(ts.ackedBytes) * 8 * 1e6 / float64(ts.Duration())
+	return int64(ackedBitrate * ts.CapturedTrafficRatio())
+}
+
+func (ts *trafficStats) CapturedTrafficRatio() float64 {
+	if ts.recvDelta == 0 {
+		return 0.0
+	}
+
+	// apply a penalty for lost packets,
+	// the rationale being packet dropping is a strategy to relieve congestion
+	// and if they were not dropped, they would have increased queuing delay,
+	// as it is not possible to know the reason for the losses,
+	// apply a small penalty to receive delta aggregate to simulate those packets
+	// building up queuing delay.
+	return min(1.0, float64(ts.sendDelta)/float64(ts.recvDelta+ts.lossPenalty()))
+}
+
+func (ts *trafficStats) WeightedLoss() float64 {
+	durationMicro := ts.Duration()
+	if time.Duration(durationMicro*1000) < ts.params.Config.MinDurationForLossValidity {
+		return 0.0
+	}
+
+	totalPackets := float64(ts.lostPackets + ts.ackedPackets)
+	pps := totalPackets * 1e6 / float64(durationMicro)
+
+	// longer duration, i. e. more time resolution, lower pps is acceptable as the measurement is more stable
+	deltaDuration := time.Duration(durationMicro*1000) - ts.params.Config.BaseDuration
+	if deltaDuration < 0 {
+		deltaDuration = 0
+	}
+	threshold := math.Exp(-deltaDuration.Seconds()) * float64(ts.params.Config.BasePPS)
+	if pps < threshold {
+		return 0.0
+	}
+
+	lossRatio := float64(0.0)
+	if totalPackets != 0 {
+		lossRatio = float64(ts.lostPackets) / totalPackets
+	}
+
+	// Log10 is used to give higher weight for the same loss ratio at higher packet rates,
+	// for e.g.
+	//    - 10% loss at 20 pps = 0.1 * log10(20) = 0.130
+	//    - 10% loss at 100 pps = 0.1 * log10(100) = 0.2
+	//    - 10% loss at 1000 pps = 0.1 * log10(1000) = 0.3
+	return lossRatio * math.Log10(pps)
+}
+
+func (ts *trafficStats) lossPenalty() int64 {
+	return int64(float64(ts.recvDelta) * ts.WeightedLoss() * ts.params.Config.LossPenaltyFactor)
+}
+
+func (ts *trafficStats) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if ts == nil {
+		return nil
+	}
+
+	e.AddInt64("minSendTime", ts.minSendTime)
+	e.AddInt64("maxSendTime", ts.maxSendTime)
+	duration := time.Duration(ts.Duration() * 1000)
+	e.AddDuration("duration", duration)
+
+	e.AddInt("ackedPackets", ts.ackedPackets)
+	e.AddInt("ackedBytes", ts.ackedBytes)
+	e.AddInt("lostPackets", ts.lostPackets)
+	e.AddInt("lostBytes", ts.lostBytes)
+
+	bitrate := float64(0)
+	if duration != 0 {
+		bitrate = float64(ts.ackedBytes*8) / duration.Seconds()
+		e.AddFloat64("bitrate", bitrate)
+	}
+
+	e.AddInt64("sendDelta", ts.sendDelta)
+	e.AddInt64("recvDelta", ts.recvDelta)
+	e.AddInt64("groupDelay", ts.recvDelta-ts.sendDelta)
+
+	totalPackets := ts.lostPackets + ts.ackedPackets
+	if duration != 0 {
+		e.AddFloat64("pps", float64(totalPackets)/duration.Seconds())
+	}
+	if (totalPackets) != 0 {
+		e.AddFloat64("rawLoss", float64(ts.lostPackets)/float64(totalPackets))
+	}
+	e.AddFloat64("weightedLoss", ts.WeightedLoss())
+	e.AddInt64("lossPenalty", ts.lossPenalty())
+
+	capturedTrafficRatio := ts.CapturedTrafficRatio()
+	e.AddFloat64("capturedTrafficRatio", capturedTrafficRatio)
+	e.AddFloat64("estimatedAvailableChannelCapacity", bitrate*capturedTrafficRatio)
+	return nil
+}

pkg/sfu/bwe/sendsidebwe/twcc_feedback.go

@@ -0,0 +1,127 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sendsidebwe
+
+import (
+	"time"
+
+	"github.com/livekit/protocol/logger"
+	"github.com/pion/rtcp"
+	"go.uber.org/zap/zapcore"
+)
+
+// ------------------------------------------------------
+
+const (
+	cOutlierReportFactor            = 3
+	cEstimatedFeedbackIntervalAlpha = float64(0.9)
+
+	cReferenceTimeMask       = (1 << 24) - 1
+	cReferenceTimeResolution = 64 // 64 ms
+)
+
+// ------------------------------------------------------
+
+type twccFeedbackParams struct {
+	Logger logger.Logger
+}
+
+type twccFeedback struct {
+	params twccFeedbackParams
+
+	lastFeedbackTime          time.Time
+	estimatedFeedbackInterval time.Duration
+	numReports                int
+	numReportsOutOfOrder      int
+
+	highestFeedbackCount uint8
+
+	cycles               int64
+	highestReferenceTime uint32
+}
+
+func newTWCCFeedback(params twccFeedbackParams) *twccFeedback {
+	return &twccFeedback{
+		params: params,
+	}
+}
+
+func (t *twccFeedback) ProcessReport(report *rtcp.TransportLayerCC, at time.Time) (int64, bool) {
+	t.numReports++
+	if t.lastFeedbackTime.IsZero() {
+		t.lastFeedbackTime = at
+		t.highestReferenceTime = report.ReferenceTime
+		t.highestFeedbackCount = report.FbPktCount
+		return (t.cycles + int64(report.ReferenceTime)) * cReferenceTimeResolution * 1000, false
+	}
+
+	isOutOfOrder := false
+	if (report.FbPktCount - t.highestFeedbackCount) > (1 << 7) {
+		t.numReportsOutOfOrder++
+		isOutOfOrder = true
+		t.params.Logger.Infow(
+			"send side bwe: received out-of-order feedback report",
+			"highestFeedbackCount", t.highestFeedbackCount,
+			"fbPktCount", report.FbPktCount,
+			"numReportsOutOfOrder", t.numReportsOutOfOrder,
+		)
+	}
+
+	// reference time wrap around handling
+	var referenceTime int64
+	if (report.ReferenceTime-t.highestReferenceTime)&cReferenceTimeMask < (1 << 23) {
+		if report.ReferenceTime < t.highestReferenceTime {
+			t.cycles += (1 << 24)
+		}
+		t.highestReferenceTime = report.ReferenceTime
+		referenceTime = t.cycles + int64(report.ReferenceTime)
+	} else {
+		cycles := t.cycles
+		if report.ReferenceTime > t.highestReferenceTime && cycles >= (1<<24) {
+			cycles -= (1 << 24)
+		}
+		referenceTime = cycles + int64(report.ReferenceTime)
+	}
+
+	if !isOutOfOrder {
+		sinceLast := at.Sub(t.lastFeedbackTime)
+		if t.estimatedFeedbackInterval == 0 {
+			t.estimatedFeedbackInterval = sinceLast
+		} else {
+			// filter out outliers from estimate
+			if sinceLast > t.estimatedFeedbackInterval/cOutlierReportFactor && sinceLast < cOutlierReportFactor*t.estimatedFeedbackInterval {
+				// smoothed version of inter feedback interval
+				t.estimatedFeedbackInterval = time.Duration(cEstimatedFeedbackIntervalAlpha*float64(t.estimatedFeedbackInterval) + (1.0-cEstimatedFeedbackIntervalAlpha)*float64(sinceLast))
+			}
+		}
+		t.lastFeedbackTime = at
+		t.highestFeedbackCount = report.FbPktCount
+	}
+
+	return referenceTime * cReferenceTimeResolution * 1000, isOutOfOrder
+}
+
+func (t *twccFeedback) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if t == nil {
+		return nil
+	}
+
+	e.AddTime("lastFeedbackTime", t.lastFeedbackTime)
+	e.AddDuration("estimatedFeedbackInterval", t.estimatedFeedbackInterval)
+	e.AddInt("numReports", t.numReports)
+	e.AddInt("numReportsOutOfOrder", t.numReportsOutOfOrder)
+	e.AddInt64("cycles", t.cycles/(1<<24))
+	return nil
+}

pkg/sfu/ccutils/probe_regulator.go

@@ -0,0 +1,106 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ccutils
+
+import (
+	"time"
+
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils/mono"
+)
+
+// ------------------------------------------------
+
+type ProbeRegulatorConfig struct {
+	BaseInterval  time.Duration `yaml:"base_interval,omitempty"`
+	BackoffFactor float64       `yaml:"backoff_factor,omitempty"`
+	MaxInterval   time.Duration `yaml:"max_interval,omitempty"`
+
+	MinDuration            time.Duration `yaml:"min_duration,omitempty"`
+	MaxDuration            time.Duration `yaml:"max_duration,omitempty"`
+	DurationIncreaseFactor float64       `yaml:"duration_increase_factor,omitempty"`
+}
+
+var (
+	DefaultProbeRegulatorConfig = ProbeRegulatorConfig{
+		BaseInterval:  3 * time.Second,
+		BackoffFactor: 1.5,
+		MaxInterval:   2 * time.Minute,
+
+		MinDuration:            200 * time.Millisecond,
+		MaxDuration:            20 * time.Second,
+		DurationIncreaseFactor: 1.5,
+	}
+)
+
+// ---------------------------------------------------------------------------
+
+type ProbeRegulatorParams struct {
+	Config ProbeRegulatorConfig
+	Logger logger.Logger
+}
+
+type ProbeRegulator struct {
+	params ProbeRegulatorParams
+
+	probeInterval       time.Duration
+	probeDuration       time.Duration
+	nextProbeEarliestAt time.Time
+}
+
+func NewProbeRegulator(params ProbeRegulatorParams) *ProbeRegulator {
+	return &ProbeRegulator{
+		params:              params,
+		probeInterval:       params.Config.BaseInterval,
+		probeDuration:       params.Config.MinDuration,
+		nextProbeEarliestAt: mono.Now(),
+	}
+}
+
+func (p *ProbeRegulator) CanProbe() bool {
+	return mono.Now().After(p.nextProbeEarliestAt)
+}
+
+func (p *ProbeRegulator) ProbeDuration() time.Duration {
+	return p.probeDuration
+}
+
+func (p *ProbeRegulator) ProbeSignal(probeSignal ProbeSignal, baseTime time.Time) {
+	if probeSignal == ProbeSignalCongesting {
+		// wait longer till next probe
+		p.probeInterval = time.Duration(p.probeInterval.Seconds()*p.params.Config.BackoffFactor) * time.Second
+		if p.probeInterval > p.params.Config.MaxInterval {
+			p.probeInterval = p.params.Config.MaxInterval
+		}
+
+		// revert back to starting with shortest probe
+		p.probeDuration = p.params.Config.MinDuration
+	} else {
+		// probe can be started again after minimal interval as previous congestion signal indicated congestion clearing
+		p.probeInterval = p.params.Config.BaseInterval
+
+		// can do longer probe after a good probe
+		p.probeDuration = time.Duration(float64(p.probeDuration.Milliseconds())*p.params.Config.DurationIncreaseFactor) * time.Millisecond
+		if p.probeDuration > p.params.Config.MaxDuration {
+			p.probeDuration = p.params.Config.MaxDuration
+		}
+	}
+
+	if baseTime.IsZero() {
+		p.nextProbeEarliestAt = mono.Now().Add(p.probeInterval)
+	} else {
+		p.nextProbeEarliestAt = baseTime.Add(p.probeInterval)
+	}
+}

pkg/sfu/ccutils/probe_signal.go

@@ -0,0 +1,40 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ccutils
+
+import "fmt"
+
+// ------------------------------------------------
+
+type ProbeSignal int
+
+const (
+	ProbeSignalInconclusive ProbeSignal = iota
+	ProbeSignalCongesting
+	ProbeSignalNotCongesting
+)
+
+func (p ProbeSignal) String() string {
+	switch p {
+	case ProbeSignalInconclusive:
+		return "INCONCLUSIVE"
+	case ProbeSignalCongesting:
+		return "CONGESTING"
+	case ProbeSignalNotCongesting:
+		return "NOT_CONGESTING"
+	default:
+		return fmt.Sprintf("%d", int(p))
+	}
+}

pkg/sfu/ccutils/prober.go

@@ -0,0 +1,596 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Design of Prober
+//
+// Probing is used to check for existence of excess channel capacity.
+// This is especially useful in the downstream direction of SFU.
+// SFU forwards audio/video streams from one or more publishers to
+// all the subscribers. But, the downstream channel of a subscriber
+// may not be big enough to carry all the streams. It is also a time
+// varying quantity.
+//
+// When there is not enough capacity, some streams will be paused.
+// To resume a stream, SFU would need to know that the channel has
+// enough capacity. That's where probing comes in. When conditions
+// are favorable, SFU can send probe packets so that the bandwidth
+// estimator has more data to estimate available channel capacity
+// better.
+// NOTE: What defines `favorable conditions` is implementation dependent.
+//
+// There are two options for probing
+//   - Use padding only RTP packets: This one is preferable as
+//     probe rate can be controlled more tightly.
+//   - Resume a paused stream or forward a higher spatial layer:
+//     Have to find a stream at probing rate. Also, a stream could
+//     get a key frame unexpectedly boosting rate in the probing
+//     window.
+//
+// The strategy used depends on stream allocator implementation.
+// This module can be used if the stream allocator decides to use
+// padding only RTP packets for probing purposes.
+//
+// Implementation:
+// There are a couple of options
+//   - Check prober in the forwarding path (pull from prober).
+//     This is preferred for scalability reasons. But, this
+//     suffers from not being able to probe when all streams
+//     are paused (could be due to downstream bandwidth
+//     constraints or the corresponding upstream tracks may
+//     have paused due to upstream bandwidth constraints).
+//     Another issue is not being able to have tight control on
+//     probing window boundary as the packet forwarding path
+//     may not have a packet to forward. But, it should not
+//     be a major concern as long as some stream(s) is/are
+//     forwarded as there should be a packet at least every
+//     60 ms or so (forwarding only one stream at 15 fps).
+//     Usually, it will be serviced much more frequently when
+//     there are multiple streams getting forwarded.
+//   - Run it a go routine. But, that would have to wake up
+//     very often to prevent bunching up of probe
+//     packets. So, a scalability concern as there is one prober
+//     per subscriber peer connection. But, probe windows
+//     should be very short (of the order of 100s of ms).
+//     So, this approach might be fine.
+//
+// The implementation here follows the second approach of using a
+// go routine.
+//
+// Pacing:
+// ------
+// Ideally, the subscriber peer connection should have a pacer which
+// trickles data out at the estimated channel capacity rate (and
+// estimated channel capacity + probing rate when actively probing).
+//
+// But, there a few significant challenges
+//  1. Pacer will require buffering of forwarded packets. That means
+//     more memory, more CPU (have to make copy of packets) and
+//     more latency in the media stream.
+//  2. Scalability concern as SFU may be handling hundreds of
+//     subscriber peer connections and each one processing the pacing
+//     loop at 5ms interval will add up.
+//
+// So, this module assumes that pacing is inherently provided by the
+// publishers for media streams. That is a reasonable assumption given
+// that publishing clients will run their own pacer and pacing data out
+// at a steady rate.
+//
+// A further assumption is that if there are multiple publishers for
+// a subscriber peer connection, all the publishers are not pacing
+// in sync, i.e. each publisher's pacer is completely independent
+// and SFU will be receiving the media packets with a good spread and
+// not clumped together.
+//
+// Given those assumptions, this module monitors media send rate and
+// adjusts probing packet sends accordingly. Although the probing may
+// have a high enough wake up frequency, it is for short windows.
+// For example, probing at 5 Mbps for 1/2 second and sending 1000 byte
+// probe per iteration will wake up every 1.6 ms. That is very high,
+// but should last for 1/2 second or so.
+//
+//	5 Mbps over 1/2 second = 2.5 Mbps
+//	2.5 Mbps = 312500 bytes = 313 probes at 1000 byte probes
+//	313 probes over 1/2 second = 1.6 ms between probes
+//
+// A few things to note
+//  1. When a probe cluster is added, the expected media rate is provided.
+//     So, the wake-up interval takes that into account. For example,
+//     if probing at 5 Mbps for 1/2 second and if 4 Mbps of it is expected
+//     to be provided by media traffic, the wake-up interval becomes 8 ms.
+//  2. The amount of probing should actually be capped at some value to
+//     avoid too much self-induced congestion. It maybe something like 500 kbps.
+//     That will increase the wake-up interval to 16 ms in the above example.
+//  3. In practice, the probing interval may also be shorter. Typically,
+//     it can be run for 2 - 3 RTTs to get a good measurement. For
+//     the longest hauls, RTT could be 250 ms or so leading to the probing
+//     window being long(ish). But, RTT should be much shorter especially if
+//     the subscriber peer connection of the client is able to connect to
+//     the nearest data center.
+package ccutils
+
+import (
+	"fmt"
+	"math"
+	"sync"
+	"time"
+
+	"github.com/gammazero/deque"
+	"go.uber.org/atomic"
+	"go.uber.org/zap/zapcore"
+
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils/mono"
+)
+
+type ProberListener interface {
+	OnProbeClusterSwitch(info ProbeClusterInfo)
+	OnSendProbe(bytesToSend int)
+}
+
+type ProberParams struct {
+	Listener ProberListener
+	Logger   logger.Logger
+}
+
+type Prober struct {
+	params ProberParams
+
+	clusterId atomic.Uint32
+
+	clustersMu    sync.RWMutex
+	clusters      deque.Deque[*Cluster]
+	activeCluster *Cluster
+}
+
+func NewProber(params ProberParams) *Prober {
+	p := &Prober{
+		params: params,
+	}
+	p.clusters.SetBaseCap(2)
+	return p
+}
+
+func (p *Prober) IsRunning() bool {
+	p.clustersMu.RLock()
+	defer p.clustersMu.RUnlock()
+
+	return p.clusters.Len() > 0
+}
+
+func (p *Prober) Reset(info ProbeClusterInfo) {
+	p.clustersMu.Lock()
+	defer p.clustersMu.Unlock()
+
+	if p.activeCluster != nil && p.activeCluster.Id() == info.Id {
+		p.activeCluster.MarkCompleted(info.Result)
+		p.params.Logger.Debugw("prober: resetting active cluster", "cluster", p.activeCluster)
+	}
+
+	p.clusters.Clear()
+	p.activeCluster = nil
+}
+
+func (p *Prober) AddCluster(mode ProbeClusterMode, pcg ProbeClusterGoal) ProbeClusterInfo {
+	if pcg.DesiredBps <= 0 {
+		return ProbeClusterInfoInvalid
+	}
+
+	clusterId := ProbeClusterId(p.clusterId.Inc())
+	cluster := newCluster(clusterId, mode, pcg, p.params.Listener)
+	p.params.Logger.Debugw("cluster added", "cluster", cluster)
+
+	p.pushBackClusterAndMaybeStart(cluster)
+
+	return cluster.Info()
+}
+
+func (p *Prober) ProbesSent(bytesSent int) {
+	cluster := p.getFrontCluster()
+	if cluster == nil {
+		return
+	}
+
+	cluster.ProbesSent(bytesSent)
+}
+
+func (p *Prober) ClusterDone(info ProbeClusterInfo) {
+	cluster := p.getFrontCluster()
+	if cluster == nil {
+		return
+	}
+
+	if cluster.Id() == info.Id {
+		cluster.MarkCompleted(info.Result)
+		p.params.Logger.Debugw("cluster done", "cluster", cluster)
+		p.popFrontCluster(cluster)
+	}
+}
+
+func (p *Prober) GetActiveClusterId() ProbeClusterId {
+	p.clustersMu.RLock()
+	defer p.clustersMu.RUnlock()
+
+	if p.activeCluster != nil {
+		return p.activeCluster.Id()
+	}
+
+	return ProbeClusterIdInvalid
+}
+
+func (p *Prober) getFrontCluster() *Cluster {
+	p.clustersMu.Lock()
+	defer p.clustersMu.Unlock()
+
+	if p.activeCluster != nil {
+		return p.activeCluster
+	}
+
+	if p.clusters.Len() == 0 {
+		p.activeCluster = nil
+	} else {
+		p.activeCluster = p.clusters.Front()
+		p.activeCluster.Start()
+	}
+	return p.activeCluster
+}
+
+func (p *Prober) popFrontCluster(cluster *Cluster) {
+	p.clustersMu.Lock()
+
+	if p.clusters.Len() == 0 {
+		p.activeCluster = nil
+		p.clustersMu.Unlock()
+		return
+	}
+
+	if p.clusters.Front() == cluster {
+		p.clusters.PopFront()
+	}
+
+	if cluster == p.activeCluster {
+		p.activeCluster = nil
+	}
+
+	p.clustersMu.Unlock()
+}
+
+func (p *Prober) pushBackClusterAndMaybeStart(cluster *Cluster) {
+	p.clustersMu.Lock()
+	p.clusters.PushBack(cluster)
+
+	if p.clusters.Len() == 1 {
+		go p.run()
+	}
+	p.clustersMu.Unlock()
+}
+
+func (p *Prober) run() {
+	ticker := time.NewTicker(100 * time.Millisecond)
+	defer ticker.Stop()
+	for {
+		cluster := p.getFrontCluster()
+		if cluster == nil {
+			return
+		}
+
+		sleepDuration := cluster.Process()
+		if sleepDuration == 0 {
+			p.popFrontCluster(cluster)
+			continue
+		}
+
+		ticker.Reset(sleepDuration)
+		<-ticker.C
+	}
+}
+
+// ---------------------------------
+
+type ProbeClusterId uint32
+
+const (
+	ProbeClusterIdInvalid ProbeClusterId = 0
+
+	// padding only packets are 255 bytes max + 20 byte header = 4 packets per probe,
+	// when not using padding only packets, this is a min and actual sent could be higher
+	cBytesPerProbe    = 1100
+	cSleepDuration    = 20 * time.Millisecond
+	cSleepDurationMin = 10 * time.Millisecond
+)
+
+// -----------------------------------
+
+type ProbeClusterMode int
+
+const (
+	ProbeClusterModeUniform ProbeClusterMode = iota
+	ProbeClusterModeLinearChirp
+)
+
+func (p ProbeClusterMode) String() string {
+	switch p {
+	case ProbeClusterModeUniform:
+		return "UNIFORM"
+	case ProbeClusterModeLinearChirp:
+		return "LINEAR_CHIRP"
+	default:
+		return fmt.Sprintf("%d", int(p))
+	}
+}
+
+// ---------------------------------------------------------------------------
+
+type ProbeClusterGoal struct {
+	AvailableBandwidthBps int
+	ExpectedUsageBps      int
+	DesiredBps            int
+	Duration              time.Duration
+	DesiredBytes          int
+}
+
+func (p ProbeClusterGoal) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	e.AddInt("AvailableBandwidthBps", p.AvailableBandwidthBps)
+	e.AddInt("ExpectedUsageBps", p.ExpectedUsageBps)
+	e.AddInt("DesiredBps", p.DesiredBps)
+	e.AddDuration("Duration", p.Duration)
+	e.AddInt("DesiredBytes", p.DesiredBytes)
+	return nil
+}
+
+type ProbeClusterResult struct {
+	StartTime              int64
+	EndTime                int64
+	PacketsProbe           int
+	BytesProbe             int
+	PacketsNonProbePrimary int
+	BytesNonProbePrimary   int
+	PacketsNonProbeRTX     int
+	BytesNonProbeRTX       int
+	IsCompleted            bool
+}
+
+func (p ProbeClusterResult) Bytes() int {
+	return p.BytesProbe + p.BytesNonProbePrimary + p.BytesNonProbeRTX
+}
+
+func (p ProbeClusterResult) Duration() time.Duration {
+	return time.Duration(p.EndTime - p.StartTime)
+}
+
+func (p ProbeClusterResult) Bitrate() float64 {
+	duration := p.Duration().Seconds()
+	if duration != 0 {
+		return float64(p.Bytes()*8) / duration
+	}
+
+	return 0
+}
+
+func (p ProbeClusterResult) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	e.AddTime("StartTime", time.Unix(0, p.StartTime))
+	e.AddTime("EndTime", time.Unix(0, p.EndTime))
+	e.AddDuration("Duration", p.Duration())
+	e.AddInt("PacketsProbe", p.PacketsProbe)
+	e.AddInt("BytesProbe", p.BytesProbe)
+	e.AddInt("PacketsNonProbePrimary", p.PacketsNonProbePrimary)
+	e.AddInt("BytesNonProbePrimary", p.BytesNonProbePrimary)
+	e.AddInt("PacketsNonProbeRTX", p.PacketsNonProbeRTX)
+	e.AddInt("BytesNonProbeRTX", p.BytesNonProbeRTX)
+	e.AddInt("Bytes", p.Bytes())
+	e.AddFloat64("Bitrate", p.Bitrate())
+	e.AddBool("IsCompleted", p.IsCompleted)
+	return nil
+}
+
+type ProbeClusterInfo struct {
+	Id        ProbeClusterId
+	CreatedAt time.Time
+	Goal      ProbeClusterGoal
+	Result    ProbeClusterResult
+}
+
+var (
+	ProbeClusterInfoInvalid = ProbeClusterInfo{Id: ProbeClusterIdInvalid}
+)
+
+func (p ProbeClusterInfo) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	e.AddUint32("Id", uint32(p.Id))
+	e.AddTime("CreatedAt", p.CreatedAt)
+	e.AddObject("Goal", p.Goal)
+	e.AddObject("Result", p.Result)
+	return nil
+}
+
+// ---------------------------------------------------------------------------
+
+type bucket struct {
+	expectedElapsedDuration time.Duration
+	expectedProbeBytesSent  int
+}
+
+func (b bucket) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	e.AddDuration("expectedElapsedDuration", b.expectedElapsedDuration)
+	e.AddInt("expectedProbesBytesSent", b.expectedProbeBytesSent)
+	return nil
+}
+
+// ---------------------------------------------------------------------------
+
+type Cluster struct {
+	lock sync.RWMutex
+
+	info     ProbeClusterInfo
+	mode     ProbeClusterMode
+	listener ProberListener
+
+	baseSleepDuration time.Duration
+	buckets           []bucket
+	bucketIdx         int
+
+	probeBytesSent int
+
+	startTime  time.Time
+	isComplete bool
+}
+
+func newCluster(id ProbeClusterId, mode ProbeClusterMode, pcg ProbeClusterGoal, listener ProberListener) *Cluster {
+	c := &Cluster{
+		mode: mode,
+		info: ProbeClusterInfo{
+			Id:        id,
+			CreatedAt: mono.Now(),
+			Goal:      pcg,
+		},
+		listener: listener,
+	}
+	c.initProbes()
+	return c
+}
+
+func (c *Cluster) initProbes() {
+	c.info.Goal.DesiredBytes = int(math.Round(float64(c.info.Goal.DesiredBps)*c.info.Goal.Duration.Seconds()/8 + 0.5))
+
+	numBuckets := int(math.Round(c.info.Goal.Duration.Seconds()/cSleepDuration.Seconds() + 0.5))
+	if numBuckets < 1 {
+		numBuckets = 1
+	}
+	numIntervals := numBuckets
+
+	// for linear chirp, group intervals with decreasing duration, i.e. incraasing bitrate,
+	// by aiming to send same number of bytes in each interval, as intervals get shorter, the bitrate is higher
+	if c.mode == ProbeClusterModeLinearChirp {
+		sum := 0
+		i := 1
+		for {
+			sum += i
+			if sum >= numBuckets {
+				break
+			}
+			i++
+		}
+		numBuckets = i
+		numIntervals = sum
+	}
+
+	c.baseSleepDuration = c.info.Goal.Duration / time.Duration(numIntervals)
+	if c.baseSleepDuration < cSleepDurationMin {
+		c.baseSleepDuration = cSleepDurationMin
+	}
+
+	numIntervals = int(math.Round(c.info.Goal.Duration.Seconds()/c.baseSleepDuration.Seconds() + 0.5))
+	desiredProbeBytesPerInterval := int(math.Round(((c.info.Goal.Duration.Seconds()*float64(c.info.Goal.DesiredBps-c.info.Goal.ExpectedUsageBps)/8)+float64(numIntervals)-1)/float64(numIntervals) + 0.5))
+
+	c.buckets = make([]bucket, numBuckets)
+	for i := 0; i < numBuckets; i++ {
+		switch c.mode {
+		case ProbeClusterModeUniform:
+			c.buckets[i] = bucket{
+				expectedElapsedDuration: c.baseSleepDuration,
+			}
+
+		case ProbeClusterModeLinearChirp:
+			c.buckets[i] = bucket{
+				expectedElapsedDuration: time.Duration(numBuckets-i) * c.baseSleepDuration,
+			}
+		}
+		if i > 0 {
+			c.buckets[i].expectedElapsedDuration += c.buckets[i-1].expectedElapsedDuration
+		}
+		c.buckets[i].expectedProbeBytesSent = (i + 1) * desiredProbeBytesPerInterval
+	}
+}
+
+func (c *Cluster) Start() {
+	if c.listener != nil {
+		c.listener.OnProbeClusterSwitch(c.info)
+	}
+}
+
+func (c *Cluster) Id() ProbeClusterId {
+	return c.info.Id
+}
+
+func (c *Cluster) Info() ProbeClusterInfo {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+
+	return c.info
+}
+
+func (c *Cluster) ProbesSent(bytesSent int) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	c.probeBytesSent += bytesSent
+}
+
+func (c *Cluster) MarkCompleted(result ProbeClusterResult) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	c.isComplete = true
+	c.info.Result = result
+}
+
+func (c *Cluster) Process() time.Duration {
+	c.lock.Lock()
+	if c.isComplete {
+		c.lock.Unlock()
+		return 0
+	}
+
+	bytesToSend := 0
+	if c.startTime.IsZero() {
+		c.startTime = mono.Now()
+		bytesToSend = cBytesPerProbe
+	} else {
+		sinceStart := time.Since(c.startTime)
+		if sinceStart > c.buckets[c.bucketIdx].expectedElapsedDuration {
+			c.bucketIdx++
+			overflow := false
+			if c.bucketIdx >= len(c.buckets) {
+				// when overflowing, repeat the last bucket
+				c.bucketIdx = len(c.buckets) - 1
+				overflow = true
+			}
+			if c.buckets[c.bucketIdx].expectedProbeBytesSent > c.probeBytesSent || overflow {
+				bytesToSend = max(cBytesPerProbe, c.buckets[c.bucketIdx].expectedProbeBytesSent-c.probeBytesSent)
+			}
+		}
+	}
+	c.lock.Unlock()
+
+	if bytesToSend != 0 && c.listener != nil {
+		c.listener.OnSendProbe(bytesToSend)
+	}
+
+	return cSleepDurationMin
+}
+
+func (c *Cluster) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if c != nil {
+		e.AddString("mode", c.mode.String())
+		e.AddObject("info", c.info)
+		e.AddDuration("baseSleepDuration", c.baseSleepDuration)
+		e.AddInt("numBuckets", len(c.buckets))
+		e.AddInt("bucketIdx", c.bucketIdx)
+		e.AddInt("probeBytesSent", c.probeBytesSent)
+		e.AddTime("startTime", c.startTime)
+		e.AddDuration("elapsed", time.Since(c.startTime))
+		e.AddBool("isComplete", c.isComplete)
+	}
+	return nil
+}
+
+// ----------------------------------------------------------------------

pkg/sfu/ccutils/trenddetector.go

@@ -0,0 +1,274 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ccutils
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/livekit/protocol/logger"
+	"go.uber.org/zap/zapcore"
+)
+
+// ------------------------------------------------
+
+type TrendDirection int
+
+const (
+	TrendDirectionInconclusive TrendDirection = iota
+	TrendDirectionUpward
+	TrendDirectionDownward
+)
+
+func (t TrendDirection) String() string {
+	switch t {
+	case TrendDirectionInconclusive:
+		return "INCONCLUSIVE"
+	case TrendDirectionUpward:
+		return "UPWARD"
+	case TrendDirectionDownward:
+		return "DOWNWARD"
+	default:
+		return fmt.Sprintf("%d", int(t))
+	}
+}
+
+// ------------------------------------------------
+
+type trendDetectorNumber interface {
+	int64 | float64
+}
+
+// ------------------------------------------------
+
+type trendDetectorSample[T trendDetectorNumber] struct {
+	value T
+	at    time.Time
+}
+
+type trendDetectorSampleElapsed[T trendDetectorNumber] struct {
+	value      T
+	sinceFirst time.Duration
+}
+
+func (t trendDetectorSampleElapsed[T]) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	e.AddFloat64("value", float64(t.value))
+	e.AddDuration("sinceFirst", t.sinceFirst)
+	return nil
+}
+
+// ------------------------------------------------
+
+type TrendDetectorConfig struct {
+	RequiredSamples        int           `yaml:"required_samples,omitempty"`
+	RequiredSamplesMin     int           `yaml:"required_samples_min,omitempty"`
+	DownwardTrendThreshold float64       `yaml:"downward_trend_threshold,omitempty"`
+	DownwardTrendMaxWait   time.Duration `yaml:"downward_trend_max_wait,omitempty"`
+	CollapseThreshold      time.Duration `yaml:"collapse_threshold,omitempty"`
+	ValidityWindow         time.Duration `yaml:"validity_window,omitempty"`
+}
+
+// ------------------------------------------------
+
+type TrendDetectorParams struct {
+	Name   string
+	Logger logger.Logger
+	Config TrendDetectorConfig
+}
+
+type TrendDetector[T trendDetectorNumber] struct {
+	params TrendDetectorParams
+
+	startTime    time.Time
+	numSamples   int
+	samples      []trendDetectorSample[T]
+	lowestValue  T
+	highestValue T
+
+	direction TrendDirection
+}
+
+func NewTrendDetector[T trendDetectorNumber](params TrendDetectorParams) *TrendDetector[T] {
+	return &TrendDetector[T]{
+		params:    params,
+		startTime: time.Now(),
+		direction: TrendDirectionInconclusive,
+	}
+}
+
+func (t *TrendDetector[T]) Seed(value T) {
+	if len(t.samples) != 0 {
+		return
+	}
+
+	t.samples = append(t.samples, trendDetectorSample[T]{value: value, at: time.Now()})
+}
+
+func (t *TrendDetector[T]) AddValue(value T) {
+	t.numSamples++
+	if t.lowestValue == 0 || value < t.lowestValue {
+		t.lowestValue = value
+	}
+	if value > t.highestValue {
+		t.highestValue = value
+	}
+
+	// Ignore duplicate values in collapse window.
+	//
+	// Bandwidth estimate is received periodically. If the estimate does not change, it will be repeated.
+	// When there is congestion, there are several estimates received with decreasing values.
+	//
+	// Using a sliding window, collapsing repeated values and waiting for falling trend to ensure that
+	// the reaction is not too fast, i. e. reacting to falling values too quick could mean a lot of re-allocation
+	// resulting in layer switches, key frames and more congestion.
+	//
+	// But, on the flip side, estimate could fall once or twice within a sliding window and stay there.
+	// In those cases, using a collapse window to record a value even if it is duplicate. By doing that,
+	// a trend could be detected eventually. It will be delayed, but that is fine with slow changing estimates.
+	var lastSample *trendDetectorSample[T]
+	if len(t.samples) != 0 {
+		lastSample = &t.samples[len(t.samples)-1]
+	}
+	if lastSample != nil && lastSample.value == value && t.params.Config.CollapseThreshold > 0 && time.Since(lastSample.at) < t.params.Config.CollapseThreshold {
+		return
+	}
+
+	t.samples = append(t.samples, trendDetectorSample[T]{value: value, at: time.Now()})
+	t.prune()
+	t.updateDirection()
+}
+
+func (t *TrendDetector[T]) GetLowest() T {
+	return t.lowestValue
+}
+
+func (t *TrendDetector[T]) GetHighest() T {
+	return t.highestValue
+}
+
+func (t *TrendDetector[T]) GetDirection() TrendDirection {
+	return t.direction
+}
+
+func (t *TrendDetector[T]) HasEnoughSamples() bool {
+	return t.numSamples >= t.params.Config.RequiredSamples
+}
+
+func (t *TrendDetector[T]) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if t == nil {
+		return nil
+	}
+
+	var samples []trendDetectorSampleElapsed[T]
+	if len(t.samples) > 0 {
+		firstTime := t.samples[0].at
+		for _, sample := range t.samples {
+			samples = append(samples, trendDetectorSampleElapsed[T]{sample.value, sample.at.Sub(firstTime)})
+		}
+	}
+
+	e.AddString("name", t.params.Name)
+	e.AddTime("startTime", t.startTime)
+	e.AddDuration("elapsed", time.Since(t.startTime))
+	e.AddInt("numSamples", t.numSamples)
+	e.AddArray("samples", logger.ObjectSlice(samples))
+	e.AddFloat64("lowestValue", float64(t.lowestValue))
+	e.AddFloat64("highestValue", float64(t.highestValue))
+	e.AddFloat64("kendallsTau", t.kendallsTau())
+	e.AddString("direction", t.direction.String())
+	return nil
+}
+
+func (t *TrendDetector[T]) prune() {
+	// prune based on a few rules
+
+	//  1. If there are more than required samples
+	if len(t.samples) > t.params.Config.RequiredSamples {
+		t.samples = t.samples[len(t.samples)-t.params.Config.RequiredSamples:]
+	}
+
+	// 2. drop samples that are too old
+	if len(t.samples) != 0 && t.params.Config.ValidityWindow > 0 {
+		cutoffTime := time.Now().Add(-t.params.Config.ValidityWindow)
+		cutoffIndex := -1
+		for i := 0; i < len(t.samples); i++ {
+			if t.samples[i].at.After(cutoffTime) {
+				cutoffIndex = i
+				break
+			}
+		}
+		if cutoffIndex >= 0 {
+			t.samples = t.samples[cutoffIndex:]
+		}
+	}
+
+	//  3. collapse same values at the front to just the last of those samples
+	if len(t.samples) != 0 {
+		cutoffIndex := -1
+		firstValue := t.samples[0].value
+		for i := 1; i < len(t.samples); i++ {
+			if t.samples[i].value != firstValue {
+				cutoffIndex = i - 1
+				break
+			}
+		}
+
+		if cutoffIndex >= 0 {
+			t.samples = t.samples[cutoffIndex:]
+		} else {
+			// all values are the same, just keep the last one
+			t.samples = t.samples[len(t.samples)-1:]
+		}
+	}
+}
+
+func (t *TrendDetector[T]) updateDirection() {
+	if len(t.samples) < t.params.Config.RequiredSamplesMin {
+		t.direction = TrendDirectionInconclusive
+		return
+	}
+
+	// using Kendall's Tau to find trend
+	kt := t.kendallsTau()
+
+	t.direction = TrendDirectionInconclusive
+	switch {
+	case kt > 0 && len(t.samples) >= t.params.Config.RequiredSamples:
+		t.direction = TrendDirectionUpward
+	case kt < t.params.Config.DownwardTrendThreshold && (len(t.samples) >= t.params.Config.RequiredSamples || t.samples[len(t.samples)-1].at.Sub(t.samples[0].at) > t.params.Config.DownwardTrendMaxWait):
+		t.direction = TrendDirectionDownward
+	}
+}
+
+func (t *TrendDetector[T]) kendallsTau() float64 {
+	concordantPairs := 0
+	discordantPairs := 0
+
+	for i := 0; i < len(t.samples)-1; i++ {
+		for j := i + 1; j < len(t.samples); j++ {
+			if t.samples[i].value < t.samples[j].value {
+				concordantPairs++
+			} else if t.samples[i].value > t.samples[j].value {
+				discordantPairs++
+			}
+		}
+	}
+
+	if (concordantPairs + discordantPairs) == 0 {
+		return 0.0
+	}
+
+	return (float64(concordantPairs) - float64(discordantPairs)) / (float64(concordantPairs) + float64(discordantPairs))
+}

pkg/sfu/codecmunger/codecmunger.go

@@ -0,0 +1,39 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package codecmunger
+
+import (
+	"errors"
+
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+)
+
+var (
+	ErrNotVP8                          = errors.New("not VP8")
+	ErrOutOfOrderVP8PictureIdCacheMiss = errors.New("out-of-order VP8 picture id not found in cache")
+	ErrFilteredVP8TemporalLayer        = errors.New("filtered VP8 temporal layer")
+)
+
+type CodecMunger interface {
+	GetState() any
+	SeedState(state any)
+
+	SetLast(extPkt *buffer.ExtPacket)
+	UpdateOffsets(extPkt *buffer.ExtPacket)
+
+	UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32) (int, []byte, error)
+
+	UpdateAndGetPadding(newPicture bool) ([]byte, error)
+}

pkg/sfu/codecmunger/null.go

@@ -0,0 +1,54 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package codecmunger
+
+import (
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/protocol/logger"
+)
+
+type Null struct {
+	seededState any
+}
+
+func NewNull(_logger logger.Logger) *Null {
+	return &Null{}
+}
+
+func (n *Null) GetState() any {
+	return nil
+}
+
+func (n *Null) SeedState(state any) {
+	n.seededState = state
+}
+
+func (n *Null) GetSeededState() any {
+	return n.seededState
+}
+
+func (n *Null) SetLast(_extPkt *buffer.ExtPacket) {
+}
+
+func (n *Null) UpdateOffsets(_extPkt *buffer.ExtPacket) {
+}
+
+func (n *Null) UpdateAndGet(_extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32) (int, []byte, error) {
+	return 0, nil, nil
+}
+
+func (n *Null) UpdateAndGetPadding(newPicture bool) ([]byte, error) {
+	return nil, nil
+}

pkg/sfu/codecmunger/vp8.go

@@ -0,0 +1,491 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package codecmunger
+
+import (
+	"github.com/elliotchance/orderedmap/v3"
+
+	"github.com/livekit/mediatransportutil/pkg/codec"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+)
+
+const (
+	missingPictureIdsThreshold  = 50
+	droppedPictureIdsThreshold  = 20
+	exemptedPictureIdsThreshold = 20
+)
+
+// -----------------------------------------------------------
+
+type VP8 struct {
+	logger logger.Logger
+
+	pictureIdWrapHandler VP8PictureIdWrapHandler
+	extLastPictureId     int32
+	pictureIdOffset      int32
+	pictureIdUsed        bool
+	lastTl0PicIdx        uint8
+	tl0PicIdxOffset      uint8
+	tl0PicIdxUsed        bool
+	tidUsed              bool
+	lastKeyIdx           uint8
+	keyIdxOffset         uint8
+	keyIdxUsed           bool
+
+	missingPictureIds  *orderedmap.OrderedMap[int32, int32]
+	droppedPictureIds  *orderedmap.OrderedMap[int32, bool]
+	exemptedPictureIds *orderedmap.OrderedMap[int32, bool]
+}
+
+func NewVP8(logger logger.Logger) *VP8 {
+	return &VP8{
+		logger:             logger,
+		missingPictureIds:  orderedmap.NewOrderedMap[int32, int32](),
+		droppedPictureIds:  orderedmap.NewOrderedMap[int32, bool](),
+		exemptedPictureIds: orderedmap.NewOrderedMap[int32, bool](),
+	}
+}
+
+func NewVP8FromOther(cm CodecMunger, logger logger.Logger) *VP8 {
+	v := NewVP8(logger)
+	switch cm := cm.(type) {
+	case *Null:
+		v.SeedState(cm.GetSeededState())
+	case *VP8:
+		v.SeedState(cm.GetState())
+	}
+	return v
+}
+
+func (v *VP8) GetState() any {
+	return &livekit.VP8MungerState{
+		ExtLastPictureId: v.extLastPictureId,
+		PictureIdUsed:    v.pictureIdUsed,
+		LastTl0PicIdx:    uint32(v.lastTl0PicIdx),
+		Tl0PicIdxUsed:    v.tl0PicIdxUsed,
+		TidUsed:          v.tidUsed,
+		LastKeyIdx:       uint32(v.lastKeyIdx),
+		KeyIdxUsed:       v.keyIdxUsed,
+	}
+}
+
+func (v *VP8) SeedState(seed any) {
+	var state *livekit.VP8MungerState
+	switch cm := seed.(type) {
+	case *livekit.RTPForwarderState_Vp8Munger:
+		state = cm.Vp8Munger
+	case *livekit.VP8MungerState:
+		state = cm
+	}
+	if state != nil {
+		v.extLastPictureId = state.ExtLastPictureId
+		v.pictureIdUsed = state.PictureIdUsed
+		v.lastTl0PicIdx = uint8(state.LastTl0PicIdx)
+		v.tl0PicIdxUsed = state.Tl0PicIdxUsed
+		v.tidUsed = state.TidUsed
+		v.lastKeyIdx = uint8(state.LastKeyIdx)
+		v.keyIdxUsed = state.KeyIdxUsed
+	}
+}
+
+func (v *VP8) SetLast(extPkt *buffer.ExtPacket) {
+	vp8, ok := extPkt.Payload.(codec.VP8)
+	if !ok {
+		return
+	}
+
+	v.pictureIdUsed = vp8.I
+	if v.pictureIdUsed {
+		v.pictureIdWrapHandler.Init(int32(vp8.PictureID)-1, vp8.M)
+		v.extLastPictureId = int32(vp8.PictureID)
+	}
+
+	v.tl0PicIdxUsed = vp8.L
+	if v.tl0PicIdxUsed {
+		v.lastTl0PicIdx = vp8.TL0PICIDX
+	}
+
+	v.tidUsed = vp8.T
+
+	v.keyIdxUsed = vp8.K
+	if v.keyIdxUsed {
+		v.lastKeyIdx = vp8.KEYIDX
+	}
+}
+
+func (v *VP8) UpdateOffsets(extPkt *buffer.ExtPacket) {
+	vp8, ok := extPkt.Payload.(codec.VP8)
+	if !ok {
+		return
+	}
+
+	if v.pictureIdUsed {
+		v.pictureIdWrapHandler.Init(int32(vp8.PictureID)-1, vp8.M)
+		v.pictureIdOffset = int32(vp8.PictureID) - v.extLastPictureId - 1
+	}
+
+	if v.tl0PicIdxUsed {
+		v.tl0PicIdxOffset = vp8.TL0PICIDX - v.lastTl0PicIdx - 1
+	}
+
+	if v.keyIdxUsed {
+		v.keyIdxOffset = (vp8.KEYIDX - v.lastKeyIdx - 1) & 0x1f
+	}
+
+	// clear picture id caches on layer switch
+	v.missingPictureIds = orderedmap.NewOrderedMap[int32, int32]()
+	v.droppedPictureIds = orderedmap.NewOrderedMap[int32, bool]()
+	v.exemptedPictureIds = orderedmap.NewOrderedMap[int32, bool]()
+}
+
+func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporalLayer int32) (int, []byte, error) {
+	vp8, ok := extPkt.Payload.(codec.VP8)
+	if !ok {
+		return 0, nil, ErrNotVP8
+	}
+
+	extPictureId := v.pictureIdWrapHandler.Unwrap(vp8.PictureID, vp8.M)
+
+	// if out-of-order, look up missing picture id cache
+	if snOutOfOrder {
+		pictureIdOffset, ok := v.missingPictureIds.Get(extPictureId)
+		if !ok {
+			return 0, nil, ErrOutOfOrderVP8PictureIdCacheMiss
+		}
+
+		// the out-of-order picture id cannot be deleted from the cache
+		// as there could more than one packet in a picture and more
+		// than one packet of a picture could come out-of-order.
+		// To prevent picture id cache from growing, it is truncated
+		// when it reaches a certain size.
+
+		mungedPictureId := uint16((extPictureId - pictureIdOffset) & 0x7fff)
+		vp8Packet := codec.VP8{
+			FirstByte:  vp8.FirstByte,
+			I:          vp8.I,
+			M:          mungedPictureId > 127,
+			PictureID:  mungedPictureId,
+			L:          vp8.L,
+			TL0PICIDX:  vp8.TL0PICIDX - v.tl0PicIdxOffset,
+			T:          vp8.T,
+			TID:        vp8.TID,
+			Y:          vp8.Y,
+			K:          vp8.K,
+			KEYIDX:     vp8.KEYIDX - v.keyIdxOffset,
+			IsKeyFrame: vp8.IsKeyFrame,
+			HeaderSize: vp8.HeaderSize + codec.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M),
+		}
+		vp8HeaderBytes, err := vp8Packet.Marshal()
+		if err != nil {
+			return 0, nil, err
+		}
+		return vp8.HeaderSize, vp8HeaderBytes, nil
+	}
+
+	prevMaxPictureId := v.pictureIdWrapHandler.MaxPictureId()
+	v.pictureIdWrapHandler.UpdateMaxPictureId(extPictureId, vp8.M)
+
+	// if there is a gap in sequence number, record possible pictures that
+	// the missing packets can belong to in missing picture id cache.
+	// The missing picture cache should contain the previous picture id
+	// and the current picture id and all the intervening pictures.
+	// This is to handle a scenario as follows
+	//   o Packet 10 -> Picture ID 10
+	//   o Packet 11 -> missing
+	//   o Packet 12 -> Picture ID 11
+	// In this case, Packet 11 could belong to either Picture ID 10 (last packet of that picture)
+	// or Picture ID 11 (first packet of the current picture). Although in this simple case,
+	// it is possible to deduce that (for example by looking at previous packet's RTP marker
+	// and check if that was the last packet of Picture 10), it could get complicated when
+	// the gap is larger.
+	if snHasGap {
+		for lostPictureId := prevMaxPictureId; lostPictureId <= extPictureId; lostPictureId++ {
+			// Record missing only if picture id was not dropped. This is to avoid a subsequent packet of dropped frame going through.
+			// A sequence like this
+			//   o Packet 10 - Picture 11 - TID that should be dropped
+			//   o Packet 11 - missing - belongs to Picture 11 still
+			//   o Packet 12 - Picture 12 - will be reported as GAP, so missing picture id mapping will be set up for Picture 11 also.
+			//   o Next packet - Packet 11 - this will use the wrong offset from missing pictures cache
+			_, ok := v.droppedPictureIds.Get(lostPictureId)
+			if !ok {
+				v.missingPictureIds.Set(lostPictureId, v.pictureIdOffset)
+			}
+		}
+
+		// trim cache if necessary
+		for v.missingPictureIds.Len() > missingPictureIdsThreshold {
+			el := v.missingPictureIds.Front()
+			v.missingPictureIds.Delete(el.Key)
+		}
+
+		// if there is a gap, packet is forwarded irrespective of temporal layer as it cannot be determined
+		// which layer the missing packets belong to. A layer could have multiple packets. So, keep track
+		// of pictures that are forwarded even though they will be filtered out based on temporal layer
+		// requirements. That allows forwarding of the complete picture.
+		if extPkt.Temporal > maxTemporalLayer {
+			v.exemptedPictureIds.Set(extPictureId, true)
+			// trim cache if necessary
+			for v.exemptedPictureIds.Len() > exemptedPictureIdsThreshold {
+				el := v.exemptedPictureIds.Front()
+				v.exemptedPictureIds.Delete(el.Key)
+			}
+		}
+	} else {
+		if extPkt.Temporal > maxTemporalLayer {
+			// drop only if not exempted
+			_, ok := v.exemptedPictureIds.Get(extPictureId)
+			if !ok {
+				// adjust only once per picture as a picture could have multiple packets
+				if vp8.I && prevMaxPictureId != extPictureId {
+					// keep track of dropped picture ids so that they do not get into the missing picture cache
+					v.droppedPictureIds.Set(extPictureId, true)
+					// trim cache if necessary
+					for v.droppedPictureIds.Len() > droppedPictureIdsThreshold {
+						el := v.droppedPictureIds.Front()
+						v.droppedPictureIds.Delete(el.Key)
+					}
+
+					v.pictureIdOffset += 1
+				}
+				return 0, nil, ErrFilteredVP8TemporalLayer
+			}
+		}
+	}
+
+	// in-order incoming sequence number, may or may not be contiguous.
+	// In the case of loss (i.e. incoming sequence number is not contiguous),
+	// forward even if it is a filtered layer. With temporal scalability,
+	// it is unclear if the current packet should be dropped if it is not
+	// contiguous. Hence, forward anything that is not contiguous.
+	// Reference: http://www.rtcbits.com/2017/04/howto-implement-temporal-scalability.html
+	extMungedPictureId := extPictureId - v.pictureIdOffset
+	mungedPictureId := uint16(extMungedPictureId & 0x7fff)
+	mungedTl0PicIdx := vp8.TL0PICIDX - v.tl0PicIdxOffset
+	mungedKeyIdx := (vp8.KEYIDX - v.keyIdxOffset) & 0x1f
+
+	v.extLastPictureId = extMungedPictureId
+	v.lastTl0PicIdx = mungedTl0PicIdx
+	v.lastKeyIdx = mungedKeyIdx
+
+	vp8Packet := codec.VP8{
+		FirstByte:  vp8.FirstByte,
+		I:          vp8.I,
+		M:          mungedPictureId > 127,
+		PictureID:  mungedPictureId,
+		L:          vp8.L,
+		TL0PICIDX:  mungedTl0PicIdx,
+		T:          vp8.T,
+		TID:        vp8.TID,
+		Y:          vp8.Y,
+		K:          vp8.K,
+		KEYIDX:     mungedKeyIdx,
+		IsKeyFrame: vp8.IsKeyFrame,
+		HeaderSize: vp8.HeaderSize + codec.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M),
+	}
+	vp8HeaderBytes, err := vp8Packet.Marshal()
+	if err != nil {
+		return 0, nil, err
+	}
+	return vp8.HeaderSize, vp8HeaderBytes, nil
+}
+
+func (v *VP8) UpdateAndGetPadding(newPicture bool) ([]byte, error) {
+	offset := 0
+	if newPicture {
+		offset = 1
+	}
+
+	headerSize := 1
+	if v.pictureIdUsed || v.tl0PicIdxUsed || v.tidUsed || v.keyIdxUsed {
+		headerSize += 1
+	}
+
+	extPictureId := v.extLastPictureId
+	if v.pictureIdUsed {
+		extPictureId = v.extLastPictureId + int32(offset)
+		v.extLastPictureId = extPictureId
+		v.pictureIdOffset -= int32(offset)
+		if (extPictureId & 0x7fff) > 127 {
+			headerSize += 2
+		} else {
+			headerSize += 1
+		}
+	}
+	pictureId := uint16(extPictureId & 0x7fff)
+
+	tl0PicIdx := uint8(0)
+	if v.tl0PicIdxUsed {
+		tl0PicIdx = v.lastTl0PicIdx + uint8(offset)
+		v.lastTl0PicIdx = tl0PicIdx
+		v.tl0PicIdxOffset -= uint8(offset)
+		headerSize += 1
+	}
+
+	if v.tidUsed || v.keyIdxUsed {
+		headerSize += 1
+	}
+
+	keyIdx := uint8(0)
+	if v.keyIdxUsed {
+		keyIdx = (v.lastKeyIdx + uint8(offset)) & 0x1f
+		v.lastKeyIdx = keyIdx
+		v.keyIdxOffset -= uint8(offset)
+	}
+
+	vp8Packet := &codec.VP8{
+		FirstByte:  0x10, // partition 0, start of VP8 Partition, reference frame
+		I:          v.pictureIdUsed,
+		M:          pictureId > 127,
+		PictureID:  pictureId,
+		L:          v.tl0PicIdxUsed,
+		TL0PICIDX:  tl0PicIdx,
+		T:          v.tidUsed,
+		TID:        0,
+		Y:          true,
+		K:          v.keyIdxUsed,
+		KEYIDX:     keyIdx,
+		IsKeyFrame: true,
+		HeaderSize: headerSize,
+	}
+	return vp8Packet.Marshal()
+}
+
+// for testing only
+func (v *VP8) PictureIdOffset(extPictureId int32) (int32, bool) {
+	return v.missingPictureIds.Get(extPictureId)
+}
+
+// -----------------------------
+
+// VP8PictureIdWrapHandler
+func isWrapping7Bit(val1 int32, val2 int32) bool {
+	return val2 < val1 && (val1-val2) > (1<<6)
+}
+
+func isWrapping15Bit(val1 int32, val2 int32) bool {
+	return val2 < val1 && (val1-val2) > (1<<14)
+}
+
+type VP8PictureIdWrapHandler struct {
+	maxPictureId int32
+	maxMBit      bool
+	totalWrap    int32
+	lastWrap     int32
+}
+
+func (v *VP8PictureIdWrapHandler) Init(extPictureId int32, mBit bool) {
+	v.maxPictureId = extPictureId
+	v.maxMBit = mBit
+	v.totalWrap = 0
+	v.lastWrap = 0
+}
+
+func (v *VP8PictureIdWrapHandler) MaxPictureId() int32 {
+	return v.maxPictureId
+}
+
+// unwrap picture id and update the maxPictureId. return unwrapped value
+func (v *VP8PictureIdWrapHandler) Unwrap(pictureId uint16, mBit bool) int32 {
+	//
+	// VP8 Picture ID is specified very flexibly.
+	//
+	// Reference: https://datatracker.ietf.org/doc/html/draft-ietf-payload-vp8
+	//
+	// Quoting from the RFC
+	// ----------------------------
+	// PictureID:  7 or 15 bits (shown left and right, respectively, in
+	//    Figure 2) not including the M bit.  This is a running index of
+	//    the frames, which MAY start at a random value, MUST increase by
+	//    1 for each subsequent frame, and MUST wrap to 0 after reaching
+	//    the maximum ID (all bits set).  The 7 or 15 bits of the
+	//    PictureID go from most significant to least significant,
+	//    beginning with the first bit after the M bit.  The sender
+	//    chooses a 7 or 15 bit index and sets the M bit accordingly.
+	//    The receiver MUST NOT assume that the number of bits in
+	//    PictureID stay the same through the session.  Having sent a
+	//    7-bit PictureID with all bits set to 1, the sender may either
+	//    wrap the PictureID to 0, or extend to 15 bits and continue
+	//    incrementing
+	// ----------------------------
+	//
+	// While in practice, senders may not switch between modes indiscriminately,
+	// it is possible that small picture ids are sent in 7 bits and then switch
+	// to 15 bits. But, to ensure correctness, this code keeps track of how much
+	// quantity has wrapped and uses that to figure out if the incoming picture id
+	// is newer OR out-of-order.
+	//
+	maxPictureId := v.maxPictureId
+	// maxPictureId can be -1 at the start
+	if maxPictureId > 0 {
+		if v.maxMBit {
+			maxPictureId = v.maxPictureId & 0x7fff
+		} else {
+			maxPictureId = v.maxPictureId & 0x7f
+		}
+	}
+
+	var newPictureId int32
+	if mBit {
+		newPictureId = int32(pictureId & 0x7fff)
+	} else {
+		newPictureId = int32(pictureId & 0x7f)
+	}
+
+	//
+	// if the new picture id is too far ahead of max, i.e. more than half of last wrap,
+	// it is out-of-order, unwrap backwards
+	//
+	if v.totalWrap > 0 {
+		if (v.maxPictureId + (v.lastWrap >> 1)) < (newPictureId + v.totalWrap) {
+			return newPictureId + v.totalWrap - v.lastWrap
+		}
+	}
+
+	//
+	// check for wrap around based on mode of previous picture id.
+	// There are three cases here
+	//   1. Wrapping from 15-bit -> 8-bit (32767 -> 0)
+	//   2. Wrapping from 15-bit -> 15-bit (32767 -> 0)
+	//   3. Wrapping from 8-bit -> 8-bit (127 -> 0)
+	// In all cases, looking at the mode of previous picture id will
+	// ensure that we are calculating the wrap properly.
+	//
+	wrap := int32(0)
+	if v.maxMBit {
+		if isWrapping15Bit(maxPictureId, newPictureId) {
+			wrap = 1 << 15
+		}
+	} else {
+		if isWrapping7Bit(maxPictureId, newPictureId) {
+			wrap = 1 << 7
+		}
+	}
+
+	v.totalWrap += wrap
+	if wrap != 0 {
+		v.lastWrap = wrap
+	}
+	newPictureId += v.totalWrap
+
+	return newPictureId
+}
+
+func (v *VP8PictureIdWrapHandler) UpdateMaxPictureId(extPictureId int32, mBit bool) {
+	v.maxPictureId = extPictureId
+	v.maxMBit = mBit
+}

pkg/sfu/codecmunger/vp8_test.go

@@ -0,0 +1,532 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package codecmunger
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/mediatransportutil/pkg/codec"
+	"github.com/livekit/protocol/logger"
+
+	"github.com/livekit/livekit-server/pkg/sfu/testutils"
+)
+
+func compare(expected *VP8, actual *VP8) bool {
+	return reflect.DeepEqual(expected.pictureIdWrapHandler, actual.pictureIdWrapHandler) &&
+		expected.extLastPictureId == actual.extLastPictureId &&
+		expected.pictureIdOffset == actual.pictureIdOffset &&
+		expected.pictureIdUsed == actual.pictureIdUsed &&
+		expected.lastTl0PicIdx == actual.lastTl0PicIdx &&
+		expected.tl0PicIdxOffset == actual.tl0PicIdxOffset &&
+		expected.tl0PicIdxUsed == actual.tl0PicIdxUsed &&
+		expected.tidUsed == actual.tidUsed &&
+		expected.lastKeyIdx == actual.lastKeyIdx &&
+		expected.keyIdxOffset == actual.keyIdxOffset &&
+		expected.keyIdxUsed == actual.keyIdxUsed
+}
+
+func newVP8() *VP8 {
+	return NewVP8(logger.GetLogger())
+}
+
+func TestSetLast(t *testing.T) {
+	v := newVP8()
+
+	params := &testutils.TestExtPacketParams{
+		SequenceNumber: 23333,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+	}
+	vp8 := &codec.VP8{
+		FirstByte:  25,
+		I:          true,
+		M:          true,
+		PictureID:  13467,
+		L:          true,
+		TL0PICIDX:  233,
+		T:          true,
+		TID:        13,
+		Y:          true,
+		K:          true,
+		KEYIDX:     23,
+		HeaderSize: 6,
+		IsKeyFrame: true,
+	}
+	extPkt, err := testutils.GetTestExtPacketVP8(params, vp8)
+	require.NoError(t, err)
+	require.NotNil(t, extPkt)
+
+	expectedVP8 := VP8{
+		pictureIdWrapHandler: VP8PictureIdWrapHandler{
+			maxPictureId: 13466,
+			maxMBit:      true,
+			totalWrap:    0,
+			lastWrap:     0,
+		},
+		extLastPictureId: 13467,
+		pictureIdOffset:  0,
+		pictureIdUsed:    true,
+		lastTl0PicIdx:    233,
+		tl0PicIdxOffset:  0,
+		tl0PicIdxUsed:    true,
+		tidUsed:          true,
+		lastKeyIdx:       23,
+		keyIdxOffset:     0,
+		keyIdxUsed:       true,
+	}
+
+	v.SetLast(extPkt)
+	require.True(t, compare(&expectedVP8, v))
+}
+
+func TestUpdateOffsets(t *testing.T) {
+	v := newVP8()
+
+	params := &testutils.TestExtPacketParams{
+		SequenceNumber: 23333,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+	}
+	vp8 := &codec.VP8{
+		FirstByte:  25,
+		I:          true,
+		M:          true,
+		PictureID:  13467,
+		L:          true,
+		TL0PICIDX:  233,
+		T:          true,
+		TID:        13,
+		Y:          true,
+		K:          true,
+		KEYIDX:     23,
+		HeaderSize: 6,
+		IsKeyFrame: true,
+	}
+	extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8)
+	v.SetLast(extPkt)
+
+	params = &testutils.TestExtPacketParams{
+		SequenceNumber: 56789,
+		Timestamp:      0xabcdef,
+		SSRC:           0x87654321,
+	}
+	vp8 = &codec.VP8{
+		FirstByte:  25,
+		I:          true,
+		M:          true,
+		PictureID:  345,
+		L:          true,
+		TL0PICIDX:  12,
+		T:          true,
+		TID:        13,
+		Y:          true,
+		K:          true,
+		KEYIDX:     4,
+		HeaderSize: 6,
+		IsKeyFrame: true,
+	}
+	extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
+	v.UpdateOffsets(extPkt)
+
+	expectedVP8 := VP8{
+		pictureIdWrapHandler: VP8PictureIdWrapHandler{
+			maxPictureId: 344,
+			maxMBit:      true,
+			totalWrap:    0,
+			lastWrap:     0,
+		},
+		extLastPictureId: 13467,
+		pictureIdOffset:  345 - 13467 - 1,
+		pictureIdUsed:    true,
+		lastTl0PicIdx:    233,
+		tl0PicIdxOffset:  (12 - 233 - 1) & 0xff,
+		tl0PicIdxUsed:    true,
+		tidUsed:          true,
+		lastKeyIdx:       23,
+		keyIdxOffset:     (4 - 23 - 1) & 0x1f,
+		keyIdxUsed:       true,
+	}
+	require.True(t, compare(&expectedVP8, v))
+}
+
+func TestOutOfOrderPictureId(t *testing.T) {
+	v := newVP8()
+
+	params := &testutils.TestExtPacketParams{
+		SequenceNumber: 23333,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+	}
+	vp8 := &codec.VP8{
+		FirstByte:  25,
+		I:          true,
+		M:          true,
+		PictureID:  13467,
+		L:          true,
+		TL0PICIDX:  233,
+		T:          true,
+		TID:        1,
+		Y:          true,
+		K:          true,
+		KEYIDX:     23,
+		HeaderSize: 6,
+		IsKeyFrame: true,
+	}
+	extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8)
+	v.SetLast(extPkt)
+	v.UpdateAndGet(extPkt, false, false, 2)
+
+	// out-of-order sequence number not in the missing picture id cache
+	vp8.PictureID = 13466
+	extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
+
+	nIn, buf, err := v.UpdateAndGet(extPkt, true, false, 2)
+	require.Error(t, err)
+	require.ErrorIs(t, err, ErrOutOfOrderVP8PictureIdCacheMiss)
+	require.Equal(t, 0, nIn)
+	require.Nil(t, buf)
+
+	// create a hole in picture id
+	vp8.PictureID = 13469
+	extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
+
+	expectedVP8 := &codec.VP8{
+		FirstByte:  25,
+		I:          true,
+		M:          true,
+		PictureID:  13469,
+		L:          true,
+		TL0PICIDX:  233,
+		T:          true,
+		TID:        1,
+		Y:          true,
+		K:          true,
+		KEYIDX:     23,
+		HeaderSize: 6,
+		IsKeyFrame: true,
+	}
+	marshalledVP8, err := expectedVP8.Marshal()
+	require.NoError(t, err)
+	nIn, buf, err = v.UpdateAndGet(extPkt, false, true, 2)
+	require.NoError(t, err)
+	require.Equal(t, 6, nIn)
+	require.Equal(t, marshalledVP8, buf)
+
+	// all three, the last, the current and the in-between should have been added to missing picture id cache
+	value, ok := v.PictureIdOffset(13467)
+	require.True(t, ok)
+	require.EqualValues(t, 0, value)
+
+	value, ok = v.PictureIdOffset(13468)
+	require.True(t, ok)
+	require.EqualValues(t, 0, value)
+
+	value, ok = v.PictureIdOffset(13469)
+	require.True(t, ok)
+	require.EqualValues(t, 0, value)
+
+	// out-of-order sequence number should be in the missing picture id cache
+	vp8.PictureID = 13468
+	extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
+
+	expectedVP8 = &codec.VP8{
+		FirstByte:  25,
+		I:          true,
+		M:          true,
+		PictureID:  13468,
+		L:          true,
+		TL0PICIDX:  233,
+		T:          true,
+		TID:        1,
+		Y:          true,
+		K:          true,
+		KEYIDX:     23,
+		HeaderSize: 6,
+		IsKeyFrame: true,
+	}
+	marshalledVP8, err = expectedVP8.Marshal()
+	require.NoError(t, err)
+	nIn, buf, err = v.UpdateAndGet(extPkt, true, false, 2)
+	require.NoError(t, err)
+	require.Equal(t, 6, nIn)
+	require.Equal(t, marshalledVP8, buf)
+}
+
+func TestTemporalLayerFiltering(t *testing.T) {
+	v := newVP8()
+
+	params := &testutils.TestExtPacketParams{
+		SequenceNumber: 23333,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+	}
+	vp8 := &codec.VP8{
+		FirstByte:  25,
+		I:          true,
+		M:          true,
+		PictureID:  13467,
+		L:          true,
+		TL0PICIDX:  233,
+		T:          true,
+		TID:        1,
+		Y:          true,
+		K:          true,
+		KEYIDX:     23,
+		HeaderSize: 6,
+		IsKeyFrame: true,
+	}
+	extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8)
+	v.SetLast(extPkt)
+
+	// translate
+	nIn, buf, err := v.UpdateAndGet(extPkt, false, false, 0)
+	require.Error(t, err)
+	require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer)
+	require.Equal(t, 0, nIn)
+	require.Nil(t, buf)
+	dropped, _ := v.droppedPictureIds.Get(13467)
+	require.True(t, dropped)
+	require.EqualValues(t, 1, v.pictureIdOffset)
+
+	// another packet with the same picture id.
+	// It should be dropped, but offset should not be updated.
+	params.SequenceNumber = 23334
+	extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
+
+	nIn, buf, err = v.UpdateAndGet(extPkt, false, false, 0)
+	require.Error(t, err)
+	require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer)
+	require.Equal(t, 0, nIn)
+	require.Nil(t, buf)
+	dropped, _ = v.droppedPictureIds.Get(13467)
+	require.True(t, dropped)
+	require.EqualValues(t, 1, v.pictureIdOffset)
+
+	// another packet with the same picture id, but a gap in sequence number.
+	// It should be dropped, but offset should not be updated.
+	params.SequenceNumber = 23337
+	extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
+
+	nIn, buf, err = v.UpdateAndGet(extPkt, false, false, 0)
+	require.Error(t, err)
+	require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer)
+	require.Equal(t, 0, nIn)
+	require.Nil(t, buf)
+	dropped, _ = v.droppedPictureIds.Get(13467)
+	require.True(t, dropped)
+	require.EqualValues(t, 1, v.pictureIdOffset)
+}
+
+func TestGapInSequenceNumberSamePicture(t *testing.T) {
+	v := newVP8()
+
+	params := &testutils.TestExtPacketParams{
+		SequenceNumber: 65533,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+		PayloadSize:    33,
+	}
+	vp8 := &codec.VP8{
+		FirstByte:  25,
+		I:          true,
+		M:          true,
+		PictureID:  13467,
+		L:          true,
+		TL0PICIDX:  233,
+		T:          true,
+		TID:        1,
+		Y:          true,
+		K:          true,
+		KEYIDX:     23,
+		HeaderSize: 6,
+		IsKeyFrame: true,
+	}
+	extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8)
+	v.SetLast(extPkt)
+
+	expectedVP8 := &codec.VP8{
+		FirstByte:  25,
+		I:          true,
+		M:          true,
+		PictureID:  13467,
+		L:          true,
+		TL0PICIDX:  233,
+		T:          true,
+		TID:        1,
+		Y:          true,
+		K:          true,
+		KEYIDX:     23,
+		HeaderSize: 6,
+		IsKeyFrame: true,
+	}
+	marshalledVP8, err := expectedVP8.Marshal()
+	require.NoError(t, err)
+	nIn, buf, err := v.UpdateAndGet(extPkt, false, false, 2)
+	require.NoError(t, err)
+	require.Equal(t, 6, nIn)
+	require.Equal(t, marshalledVP8, buf)
+
+	// telling there is a gap in sequence number will add pictures to missing picture cache
+	expectedVP8 = &codec.VP8{
+		FirstByte:  25,
+		I:          true,
+		M:          true,
+		PictureID:  13467,
+		L:          true,
+		TL0PICIDX:  233,
+		T:          true,
+		TID:        1,
+		Y:          true,
+		K:          true,
+		KEYIDX:     23,
+		HeaderSize: 6,
+		IsKeyFrame: true,
+	}
+	marshalledVP8, err = expectedVP8.Marshal()
+	require.NoError(t, err)
+	nIn, buf, err = v.UpdateAndGet(extPkt, false, true, 2)
+	require.NoError(t, err)
+	require.Equal(t, 6, nIn)
+	require.Equal(t, marshalledVP8, buf)
+
+	value, ok := v.PictureIdOffset(13467)
+	require.True(t, ok)
+	require.EqualValues(t, 0, value)
+}
+
+func TestUpdateAndGetPadding(t *testing.T) {
+	v := newVP8()
+
+	params := &testutils.TestExtPacketParams{
+		SequenceNumber: 23333,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+		PayloadSize:    20,
+	}
+	vp8 := &codec.VP8{
+		FirstByte:  25,
+		I:          true,
+		M:          true,
+		PictureID:  13467,
+		L:          true,
+		TL0PICIDX:  233,
+		T:          true,
+		TID:        13,
+		Y:          true,
+		K:          true,
+		KEYIDX:     23,
+		HeaderSize: 6,
+		IsKeyFrame: true,
+	}
+	extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8)
+
+	v.SetLast(extPkt)
+
+	// getting padding with repeat of last picture
+	buf, err := v.UpdateAndGetPadding(false)
+	require.NoError(t, err)
+	expectedVP8 := codec.VP8{
+		FirstByte:  16,
+		I:          true,
+		M:          true,
+		PictureID:  13467,
+		L:          true,
+		TL0PICIDX:  233,
+		T:          true,
+		TID:        0,
+		Y:          true,
+		K:          true,
+		KEYIDX:     23,
+		HeaderSize: 6,
+		IsKeyFrame: true,
+	}
+	marshalledVP8, err := expectedVP8.Marshal()
+	require.NoError(t, err)
+	require.Equal(t, marshalledVP8, buf)
+
+	// getting padding with new picture
+	buf, err = v.UpdateAndGetPadding(true)
+	require.NoError(t, err)
+	expectedVP8 = codec.VP8{
+		FirstByte:  16,
+		I:          true,
+		M:          true,
+		PictureID:  13468,
+		L:          true,
+		TL0PICIDX:  234,
+		T:          true,
+		TID:        0,
+		Y:          true,
+		K:          true,
+		KEYIDX:     24,
+		HeaderSize: 6,
+		IsKeyFrame: true,
+	}
+	marshalledVP8, err = expectedVP8.Marshal()
+	require.NoError(t, err)
+	require.Equal(t, marshalledVP8, buf)
+}
+
+func TestVP8PictureIdWrapHandler(t *testing.T) {
+	v := &VP8PictureIdWrapHandler{}
+
+	v.Init(109, false)
+	require.Equal(t, int32(109), v.MaxPictureId())
+	require.False(t, v.maxMBit)
+
+	v.UpdateMaxPictureId(109350, true)
+	require.Equal(t, int32(109350), v.MaxPictureId())
+	require.True(t, v.maxMBit)
+
+	// start with something close to the 15-bit wrap around point
+	v.Init(32766, true)
+
+	// out-of-order, do not wrap
+	extPictureId := v.Unwrap(32750, true)
+	require.Equal(t, int32(32750), extPictureId)
+	require.Equal(t, int32(0), v.totalWrap)
+	require.Equal(t, int32(0), v.lastWrap)
+
+	// wrap at 15-bits
+	extPictureId = v.Unwrap(5, false)
+	require.Equal(t, int32(32773), extPictureId) // 15-bit wrap at 32768 + 5 = 32773
+	require.Equal(t, int32(32768), v.totalWrap)
+	require.Equal(t, int32(32768), v.lastWrap)
+
+	// set things near 7-bit wrap point
+	v.UpdateMaxPictureId(32893, false) // 32768 + 125
+
+	// wrap at 7-bits
+	extPictureId = v.Unwrap(5, true)
+	require.Equal(t, int32(32901), extPictureId) // 15-bit wrap at 32768 + 7-bit wrap at 128 + 5 =  32901
+	require.Equal(t, int32(32896), v.totalWrap)  // one 15-bit wrap + one 7-bit wrap
+	require.Equal(t, int32(128), v.lastWrap)
+
+	// a new picture in 7-bit mode much with a gap in between.
+	// A big enough gap which would have been treated as out-of-order in 7-bit mode.
+	v.UpdateMaxPictureId(32901, false)
+	extPictureId = v.Unwrap(73, false)
+	require.Equal(t, int32(32841), extPictureId) // 15-bit wrap at 32768 + 73 =  32841
+
+	// a new picture in 15-bit mode much with a gap in between.
+	// A big enough gap which would have been treated as out-of-order in 7-bit mode.
+	v.UpdateMaxPictureId(32901, true)
+	v.lastWrap = int32(32768)
+	extPictureId = v.Unwrap(73, false)
+	require.Equal(t, int32(32969), extPictureId) // 15-bit wrap at 32768 + 7-bit wrap at 128 + 73 =  32969
+}

pkg/sfu/connectionquality/connectionstats.go

@@ -0,0 +1,519 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package connectionquality
+
+import (
+	"sync"
+	"time"
+
+	"github.com/frostbyte73/core"
+	"go.uber.org/atomic"
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	"github.com/livekit/protocol/codecs/mime"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/sfu/rtpstats"
+)
+
+const (
+	UpdateInterval                   = 5 * time.Second
+	noReceiverReportTooLongThreshold = 30 * time.Second
+)
+
+type ConnectionStatsReceiverProvider interface {
+	GetDeltaStats() map[uint32]*buffer.StreamStatsWithLayers
+	GetLastSenderReportTime() time.Time
+}
+
+type ConnectionStatsSenderProvider interface {
+	GetDeltaStatsSender() map[uint32]*buffer.StreamStatsWithLayers
+	GetPrimaryStreamLastReceiverReportTime() time.Time
+	GetPrimaryStreamPacketsSent() uint64
+}
+
+type ConnectionStatsParams struct {
+	UpdateInterval     time.Duration
+	IncludeRTT         bool
+	IncludeJitter      bool
+	EnableBitrateScore bool
+	ReceiverProvider   ConnectionStatsReceiverProvider
+	SenderProvider     ConnectionStatsSenderProvider
+	Logger             logger.Logger
+}
+
+type ConnectionStats struct {
+	params ConnectionStatsParams
+
+	codecMimeType atomic.Value // mime.MimeType
+
+	isStarted atomic.Bool
+	isVideo   atomic.Bool
+
+	onStatsUpdate func(cs *ConnectionStats, stat *livekit.AnalyticsStat)
+
+	lock               sync.RWMutex
+	packetsSent        uint64
+	streamingStartedAt time.Time
+
+	scorer *qualityScorer
+
+	done core.Fuse
+}
+
+func NewConnectionStats(params ConnectionStatsParams) *ConnectionStats {
+	return &ConnectionStats{
+		params: params,
+		scorer: newQualityScorer(qualityScorerParams{
+			IncludeRTT:         params.IncludeRTT,
+			IncludeJitter:      params.IncludeJitter,
+			EnableBitrateScore: params.EnableBitrateScore,
+			Logger:             params.Logger,
+		}),
+	}
+}
+
+func (cs *ConnectionStats) StartAt(codecMimeType mime.MimeType, isFECEnabled bool, at time.Time) {
+	if cs.isStarted.Swap(true) {
+		return
+	}
+
+	cs.isVideo.Store(mime.IsMimeTypeVideo(codecMimeType))
+	cs.codecMimeType.Store(codecMimeType)
+	cs.scorer.StartAt(getPacketLossWeight(codecMimeType, isFECEnabled), at)
+
+	go cs.updateStatsWorker()
+}
+
+func (cs *ConnectionStats) Start(codecMimeType mime.MimeType, isFECEnabled bool) {
+	cs.StartAt(codecMimeType, isFECEnabled, time.Now())
+}
+
+func (cs *ConnectionStats) Close() {
+	cs.done.Break()
+}
+
+func (cs *ConnectionStats) UpdateCodec(codecMimeType mime.MimeType, isFECEnabled bool) {
+	cs.isVideo.Store(mime.IsMimeTypeVideo(codecMimeType))
+	cs.codecMimeType.Store(codecMimeType)
+	cs.scorer.UpdatePacketLossWeight(getPacketLossWeight(codecMimeType, isFECEnabled))
+}
+
+func (cs *ConnectionStats) OnStatsUpdate(fn func(cs *ConnectionStats, stat *livekit.AnalyticsStat)) {
+	cs.onStatsUpdate = fn
+}
+
+func (cs *ConnectionStats) UpdateMuteAt(isMuted bool, at time.Time) {
+	if cs.done.IsBroken() {
+		return
+	}
+
+	cs.scorer.UpdateMuteAt(isMuted, at)
+}
+
+func (cs *ConnectionStats) UpdateMute(isMuted bool) {
+	if cs.done.IsBroken() {
+		return
+	}
+
+	cs.scorer.UpdateMute(isMuted)
+}
+
+func (cs *ConnectionStats) AddBitrateTransitionAt(bitrate int64, at time.Time) {
+	if cs.done.IsBroken() {
+		return
+	}
+
+	cs.scorer.AddBitrateTransitionAt(bitrate, at)
+}
+
+func (cs *ConnectionStats) AddBitrateTransition(bitrate int64) {
+	if cs.done.IsBroken() {
+		return
+	}
+
+	cs.scorer.AddBitrateTransition(bitrate)
+}
+
+func (cs *ConnectionStats) UpdateLayerMuteAt(isMuted bool, at time.Time) {
+	if cs.done.IsBroken() {
+		return
+	}
+
+	cs.scorer.UpdateLayerMuteAt(isMuted, at)
+}
+
+func (cs *ConnectionStats) UpdateLayerMute(isMuted bool) {
+	if cs.done.IsBroken() {
+		return
+	}
+
+	cs.scorer.UpdateLayerMute(isMuted)
+}
+
+func (cs *ConnectionStats) UpdatePauseAt(isPaused bool, at time.Time) {
+	if cs.done.IsBroken() {
+		return
+	}
+
+	cs.scorer.UpdatePauseAt(isPaused, at)
+}
+
+func (cs *ConnectionStats) UpdatePause(isPaused bool) {
+	if cs.done.IsBroken() {
+		return
+	}
+
+	cs.scorer.UpdatePause(isPaused)
+}
+
+func (cs *ConnectionStats) AddLayerTransitionAt(distance float64, at time.Time) {
+	if cs.done.IsBroken() {
+		return
+	}
+
+	cs.scorer.AddLayerTransitionAt(distance, at)
+}
+
+func (cs *ConnectionStats) AddLayerTransition(distance float64) {
+	if cs.done.IsBroken() {
+		return
+	}
+
+	cs.scorer.AddLayerTransition(distance)
+}
+
+func (cs *ConnectionStats) GetScoreAndQuality() (float32, livekit.ConnectionQuality) {
+	return cs.scorer.GetMOSAndQuality()
+}
+
+func (cs *ConnectionStats) updateScoreWithAggregate(agg *rtpstats.RTPDeltaInfo, lastRTCPAt time.Time, at time.Time) float32 {
+	var stat windowStat
+	if agg != nil {
+		stat.startedAt = agg.StartTime
+		stat.duration = agg.EndTime.Sub(agg.StartTime)
+		stat.packets = agg.Packets
+		stat.packetsPadding = agg.PacketsPadding
+		stat.packetsLost = agg.PacketsLost
+		stat.packetsMissing = agg.PacketsMissing
+		stat.packetsOutOfOrder = agg.PacketsOutOfOrder
+		stat.bytes = agg.Bytes - agg.HeaderBytes // only use media payload size
+		stat.rttMax = agg.RttMax
+		stat.jitterMax = agg.JitterMax
+
+		stat.lastRTCPAt = lastRTCPAt
+	}
+	if at.IsZero() {
+		cs.scorer.Update(&stat)
+	} else {
+		cs.scorer.UpdateAt(&stat, at)
+	}
+
+	mos, _ := cs.scorer.GetMOSAndQuality()
+	return mos
+}
+
+func (cs *ConnectionStats) updateScoreFromReceiverReport(at time.Time) (float32, map[uint32]*buffer.StreamStatsWithLayers) {
+	if cs.params.SenderProvider == nil {
+		return MinMOS, nil
+	}
+
+	streamingStartedAt := cs.updateStreamingStart(at)
+	if streamingStartedAt.IsZero() {
+		// not streaming, just return current score
+		mos, _ := cs.scorer.GetMOSAndQuality()
+		return mos, nil
+	}
+
+	streams := cs.params.SenderProvider.GetDeltaStatsSender()
+	if len(streams) == 0 {
+		//  check for receiver report not received for a while
+		marker := cs.params.SenderProvider.GetPrimaryStreamLastReceiverReportTime()
+		if marker.IsZero() || streamingStartedAt.After(marker) {
+			marker = streamingStartedAt
+		}
+		if time.Since(marker) > noReceiverReportTooLongThreshold {
+			// have not received receiver report for a long time when streaming, run with nil stat
+			return cs.updateScoreWithAggregate(nil, time.Time{}, at), nil
+		}
+
+		// wait for receiver report, return current score
+		mos, _ := cs.scorer.GetMOSAndQuality()
+		return mos, nil
+	}
+
+	// delta stat duration could be large due to not receiving receiver report for a long time (for example, due to mute),
+	// adjust to streaming start if necessary
+	if streamingStartedAt.After(cs.params.SenderProvider.GetPrimaryStreamLastReceiverReportTime()) {
+		// last receiver report was before streaming started, wait for next one
+		mos, _ := cs.scorer.GetMOSAndQuality()
+		return mos, streams
+	}
+
+	agg := toAggregateDeltaInfo(streams, true)
+	if agg == nil {
+		// no receiver report in the window
+		mos, _ := cs.scorer.GetMOSAndQuality()
+		return mos, streams
+	}
+	if streamingStartedAt.After(agg.StartTime) {
+		agg.StartTime = streamingStartedAt
+	}
+	return cs.updateScoreWithAggregate(agg, time.Time{}, at), streams
+}
+
+func (cs *ConnectionStats) updateScoreAt(at time.Time) (float32, map[uint32]*buffer.StreamStatsWithLayers, bool) {
+	if cs.params.SenderProvider != nil {
+		// receiver report based quality scoring, use stats from receiver report for scoring
+		score, streams := cs.updateScoreFromReceiverReport(at)
+		return score, streams, true
+	}
+
+	if cs.params.ReceiverProvider == nil {
+		return MinMOS, nil, false
+	}
+
+	streams := cs.params.ReceiverProvider.GetDeltaStats()
+	if len(streams) == 0 {
+		mos, _ := cs.scorer.GetMOSAndQuality()
+		return mos, nil, false
+	}
+
+	agg := toAggregateDeltaInfo(streams, false)
+	if agg == nil {
+		// no receiver report in the window
+		mos, _ := cs.scorer.GetMOSAndQuality()
+		return mos, streams, false
+	}
+	return cs.updateScoreWithAggregate(agg, cs.params.ReceiverProvider.GetLastSenderReportTime(), at), streams, false
+}
+
+func (cs *ConnectionStats) updateStreamingStart(at time.Time) time.Time {
+	cs.lock.Lock()
+	defer cs.lock.Unlock()
+
+	packetsSent := cs.params.SenderProvider.GetPrimaryStreamPacketsSent()
+	if packetsSent > cs.packetsSent {
+		if cs.streamingStartedAt.IsZero() {
+			// the start could be anywhere after last update, but using `at` as this is not required to be accurate
+			if at.IsZero() {
+				cs.streamingStartedAt = time.Now()
+			} else {
+				cs.streamingStartedAt = at
+			}
+		}
+	} else {
+		cs.streamingStartedAt = time.Time{}
+	}
+	cs.packetsSent = packetsSent
+
+	return cs.streamingStartedAt
+}
+
+func (cs *ConnectionStats) getStat() {
+	score, streams, isSender := cs.updateScoreAt(time.Time{})
+
+	if cs.onStatsUpdate != nil && len(streams) != 0 {
+		analyticsStreams := make([]*livekit.AnalyticsStream, 0, len(streams))
+		for ssrc, stream := range streams {
+			as := toAnalyticsStream(ssrc, stream.RTPStats, stream.RTPStatsRemoteView, isSender)
+			if as == nil {
+				continue
+			}
+
+			//
+			// add video layer if either
+			//   1. Simulcast - even if there is only one layer per stream as it provides layer id
+			//   2. A stream has multiple layers
+			//
+			if (len(streams) > 1 || len(stream.Layers) > 1) && cs.isVideo.Load() {
+				for layer, layerStats := range stream.Layers {
+					avl := toAnalyticsVideoLayer(layer, layerStats)
+					if avl != nil {
+						as.VideoLayers = append(as.VideoLayers, avl)
+					}
+				}
+			}
+
+			analyticsStreams = append(analyticsStreams, as)
+		}
+
+		if len(analyticsStreams) != 0 {
+			cs.onStatsUpdate(cs, &livekit.AnalyticsStat{
+				Score:   score,
+				Streams: analyticsStreams,
+				Mime:    cs.codecMimeType.Load().(mime.MimeType).String(),
+			})
+		}
+	}
+}
+
+func (cs *ConnectionStats) updateStatsWorker() {
+	interval := cs.params.UpdateInterval
+	if interval == 0 {
+		interval = UpdateInterval
+	}
+
+	tk := time.NewTicker(interval)
+	defer tk.Stop()
+
+	for {
+		select {
+		case <-cs.done.Watch():
+			cs.getStat()
+			return
+
+		case <-tk.C:
+			if cs.done.IsBroken() {
+				return
+			}
+
+			cs.getStat()
+		}
+	}
+}
+
+// -----------------------------------------------------------------------
+
+// how much weight to give to packet loss rate when calculating score.
+// It is codec dependent.
+// For audio:
+//
+//	o Opus without FEC or RED suffers the most through packet loss, hence has the highest weight
+//	o RED with two packet redundancy can absorb one out of every two packets lost, so packet loss is not as detrimental and therefore lower weight
+//
+// For video:
+//
+//	o No in-built codec repair available, hence same for all codecs
+func getPacketLossWeight(mimeType mime.MimeType, isFecEnabled bool) float64 {
+	var plw float64
+	switch {
+	case mimeType == mime.MimeTypeOpus:
+		// 2.5%: fall to GOOD, 7.5%: fall to POOR
+		plw = 8.0
+		if isFecEnabled {
+			// 3.75%: fall to GOOD, 11.25%: fall to POOR
+			plw /= 1.5
+		}
+
+	case mimeType == mime.MimeTypeRED:
+		// 5%: fall to GOOD, 15.0%: fall to POOR
+		plw = 4.0
+		if isFecEnabled {
+			// 7.5%: fall to GOOD, 22.5%: fall to POOR
+			plw /= 1.5
+		}
+
+	case mime.IsMimeTypeVideo(mimeType):
+		// 2%: fall to GOOD, 6%: fall to POOR
+		plw = 10.0
+	}
+
+	return plw
+}
+
+func toAggregateDeltaInfo(streams map[uint32]*buffer.StreamStatsWithLayers, useRemoteView bool) *rtpstats.RTPDeltaInfo {
+	deltaInfoList := make([]*rtpstats.RTPDeltaInfo, 0, len(streams))
+	for _, s := range streams {
+		if useRemoteView {
+			if s.RTPStatsRemoteView != nil {
+				// discount jitter from publisher side + internal processing while reporting downstream jitter
+				if s.RTPStats != nil {
+					s.RTPStatsRemoteView.JitterMax -= s.RTPStats.JitterMax
+					if s.RTPStatsRemoteView.JitterMax < 0.0 {
+						s.RTPStatsRemoteView.JitterMax = 0.0
+					}
+				}
+				deltaInfoList = append(deltaInfoList, s.RTPStatsRemoteView)
+			}
+		} else {
+			if s.RTPStats != nil {
+				deltaInfoList = append(deltaInfoList, s.RTPStats)
+			}
+		}
+	}
+	return rtpstats.AggregateRTPDeltaInfo(deltaInfoList)
+}
+
+func toAnalyticsStream(
+	ssrc uint32,
+	deltaStats *rtpstats.RTPDeltaInfo,
+	deltaStatsRemoteView *rtpstats.RTPDeltaInfo,
+	isSender bool,
+) *livekit.AnalyticsStream {
+	if deltaStats == nil {
+		return nil
+	}
+
+	// discount the feed side loss when reporting forwarded track stats,
+	// discount jitter from publisher side + internal processing while reporting downstream jitter
+	packetsLost := deltaStats.PacketsLost
+	rtt := deltaStats.RttMax
+	maxJitter := deltaStats.JitterMax
+	if deltaStatsRemoteView != nil {
+		packetsLost = deltaStatsRemoteView.PacketsLost
+		if deltaStatsRemoteView.PacketsMissing > packetsLost {
+			packetsLost = 0
+		} else {
+			packetsLost -= deltaStatsRemoteView.PacketsMissing
+		}
+
+		rtt = deltaStatsRemoteView.RttMax
+		maxJitter = deltaStatsRemoteView.JitterMax
+	} else if isSender {
+		rtt = 0
+		maxJitter = 0
+	}
+
+	return &livekit.AnalyticsStream{
+		StartTime:         timestamppb.New(deltaStats.StartTime),
+		EndTime:           timestamppb.New(deltaStats.EndTime),
+		Ssrc:              ssrc,
+		PrimaryPackets:    deltaStats.Packets,
+		PrimaryBytes:      deltaStats.Bytes,
+		RetransmitPackets: deltaStats.PacketsDuplicate,
+		RetransmitBytes:   deltaStats.BytesDuplicate,
+		PaddingPackets:    deltaStats.PacketsPadding,
+		PaddingBytes:      deltaStats.BytesPadding,
+		PacketsLost:       packetsLost,
+		PacketsOutOfOrder: deltaStats.PacketsOutOfOrder,
+		Frames:            deltaStats.Frames,
+		Rtt:               rtt,
+		Jitter:            uint32(maxJitter),
+		Nacks:             deltaStats.Nacks,
+		Plis:              deltaStats.Plis,
+		Firs:              deltaStats.Firs,
+	}
+}
+
+func toAnalyticsVideoLayer(layer int32, layerStats *rtpstats.RTPDeltaInfo) *livekit.AnalyticsVideoLayer {
+	if layerStats == nil {
+		return nil
+	}
+
+	avl := &livekit.AnalyticsVideoLayer{
+		Layer:   layer,
+		Packets: layerStats.Packets + layerStats.PacketsDuplicate + layerStats.PacketsPadding,
+		Bytes:   layerStats.Bytes + layerStats.BytesDuplicate + layerStats.BytesPadding,
+		Frames:  layerStats.Frames,
+	}
+	if avl.Packets == 0 || avl.Bytes == 0 || avl.Frames == 0 {
+		return nil
+	}
+
+	return avl
+}

pkg/sfu/connectionquality/connectionstats_test.go

@@ -0,0 +1,903 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package connectionquality
+
+import (
+	"math"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/sfu/rtpstats"
+	"github.com/livekit/protocol/codecs/mime"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+)
+
+// -----------------------------------------------
+
+type testReceiverProvider struct {
+	streams              map[uint32]*buffer.StreamStatsWithLayers
+	lastSenderReportTime time.Time
+}
+
+func newTestReceiverProvider() *testReceiverProvider {
+	return &testReceiverProvider{}
+}
+
+func (trp *testReceiverProvider) setStreams(streams map[uint32]*buffer.StreamStatsWithLayers) {
+	trp.streams = streams
+}
+
+func (trp *testReceiverProvider) GetDeltaStats() map[uint32]*buffer.StreamStatsWithLayers {
+	return trp.streams
+}
+
+func (trp *testReceiverProvider) setLastSenderReportTime(at time.Time) {
+	trp.lastSenderReportTime = at
+}
+
+func (trp *testReceiverProvider) GetLastSenderReportTime() time.Time {
+	return trp.lastSenderReportTime
+}
+
+// -----------------------------------------------
+
+func TestConnectionQuality(t *testing.T) {
+	trp := newTestReceiverProvider()
+	t.Run("quality scorer operation", func(t *testing.T) {
+		cs := NewConnectionStats(ConnectionStatsParams{
+			IncludeRTT:         true,
+			IncludeJitter:      true,
+			EnableBitrateScore: true,
+			ReceiverProvider:   trp,
+			Logger:             logger.GetLogger(),
+		})
+
+		duration := 5 * time.Second
+		now := time.Now()
+		cs.StartAt(mime.MimeTypeOpus, false, now.Add(-duration))
+		cs.UpdateMuteAt(false, now.Add(-1*time.Second))
+
+		// no data and not enough unmute time should return default state which is EXCELLENT quality
+		cs.updateScoreAt(now)
+		mos, quality := cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.6), mos)
+		require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
+
+		// best conditions (no loss, jitter/rtt = 0) - quality should stay EXCELLENT
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime: now,
+					EndTime:   now.Add(duration),
+					Packets:   250,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.6), mos)
+		require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
+
+		// introduce loss and the score should drop - 12% loss for Opus -> POOR
+		now = now.Add(duration)
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime:   now,
+					EndTime:     now.Add(duration),
+					Packets:     120,
+					PacketsLost: 30,
+				},
+			},
+			2: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime:   now,
+					EndTime:     now.Add(duration),
+					Packets:     130,
+					PacketsLost: 0,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(2.1), mos)
+		require.Equal(t, livekit.ConnectionQuality_POOR, quality)
+
+		// should climb to GOOD quality in one iteration if the conditions improve.
+		// although significant loss (12%) in the previous window, lowest score is
+		// bound so that climbing back does not take too long even under excellent conditions.
+		now = now.Add(duration)
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime: now,
+					EndTime:   now.Add(duration),
+					Packets:   250,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.1), mos)
+		require.Equal(t, livekit.ConnectionQuality_GOOD, quality)
+
+		// should stay at GOOD if conditions continue to be good
+		now = now.Add(duration)
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime: now,
+					EndTime:   now.Add(duration),
+					Packets:   250,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.1), mos)
+		require.Equal(t, livekit.ConnectionQuality_GOOD, quality)
+
+		// should climb up to EXCELLENT if conditions continue to be good
+		now = now.Add(duration)
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime: now,
+					EndTime:   now.Add(duration),
+					Packets:   250,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.6), mos)
+		require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
+
+		// introduce loss and the score should drop - 5% loss for Opus -> GOOD
+		now = now.Add(duration)
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime:   now,
+					EndTime:     now.Add(duration),
+					Packets:     250,
+					PacketsLost: 13,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.1), mos)
+		require.Equal(t, livekit.ConnectionQuality_GOOD, quality)
+
+		// should stay at GOOD quality for another iteration even if the conditions improve
+		now = now.Add(duration)
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime: now,
+					EndTime:   now.Add(duration),
+					Packets:   250,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.1), mos)
+		require.Equal(t, livekit.ConnectionQuality_GOOD, quality)
+
+		// should climb up to EXCELLENT if conditions continue to be good
+		now = now.Add(duration)
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime: now,
+					EndTime:   now.Add(duration),
+					Packets:   250,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.6), mos)
+		require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
+
+		// mute when quality is POOR should return quality to EXCELLENT
+		now = now.Add(duration)
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime:   now,
+					EndTime:     now.Add(duration),
+					Packets:     250,
+					PacketsLost: 30,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(2.1), mos)
+		require.Equal(t, livekit.ConnectionQuality_POOR, quality)
+
+		now = now.Add(duration)
+		cs.UpdateMuteAt(true, now.Add(1*time.Second))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.6), mos)
+		require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
+
+		// unmute at specific time to ensure next window does not satisfy the unmute time threshold.
+		// that means even if the next update has 0 packets, it should hold state and stay at EXCELLENT quality
+		cs.UpdateMuteAt(false, now.Add(3*time.Second))
+
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime: now,
+					EndTime:   now.Add(duration),
+					Packets:   0,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.6), mos)
+		require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
+
+		// next update with no packets,
+		// but last RTCP is not set, should knock quality down to POOR
+		now = now.Add(duration)
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime: now,
+					EndTime:   now.Add(duration),
+					Packets:   0,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(2.1), mos)
+		require.Equal(t, livekit.ConnectionQuality_POOR, quality)
+
+		// another dry spell, but last RTCP is not stale, should keep quality at POOR
+		now = now.Add(duration)
+		trp.setLastSenderReportTime(now.Add(time.Second))
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime: now,
+					EndTime:   now.Add(duration),
+					Packets:   0,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(2.1), mos)
+		require.Equal(t, livekit.ConnectionQuality_POOR, quality)
+
+		// yet another dry spell, but last RTCP is stale, should knock down quality at LOST
+		now = now.Add(duration)
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime: now,
+					EndTime:   now.Add(duration),
+					Packets:   0,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(1.3), mos)
+		require.Equal(t, livekit.ConnectionQuality_LOST, quality)
+
+		// mute when LOST should not bump up score/quality
+		now = now.Add(duration)
+		cs.UpdateMuteAt(true, now.Add(1*time.Second))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(1.3), mos)
+		require.Equal(t, livekit.ConnectionQuality_LOST, quality)
+
+		// unmute and send packets to bring quality back up
+		now = now.Add(duration)
+		cs.UpdateMuteAt(false, now.Add(2*time.Second))
+		for range 3 {
+			trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+				1: {
+					RTPStats: &rtpstats.RTPDeltaInfo{
+						StartTime:   now,
+						EndTime:     now.Add(duration),
+						Packets:     250,
+						PacketsLost: 0,
+					},
+				},
+			})
+			cs.updateScoreAt(now.Add(duration))
+			now = now.Add(duration)
+		}
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.6), mos)
+		require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
+
+		// with lesser number of packet (simulating DTX).
+		// even higher loss (like 10%) should not knock down quality due to quadratic weighting of packet loss ratio
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime:   now,
+					EndTime:     now.Add(duration),
+					Packets:     50,
+					PacketsLost: 5,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.6), mos)
+		require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
+
+		// mute/unmute to bring quality back up
+		now = now.Add(duration)
+		cs.UpdateMuteAt(true, now.Add(1*time.Second))
+		cs.UpdateMuteAt(false, now.Add(2*time.Second))
+
+		// RTT and jitter can knock quality down.
+		// at 2% loss, quality should stay at EXCELLENT purely based on loss, but with added RTT/jitter, should drop to GOOD
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime:   now,
+					EndTime:     now.Add(duration),
+					Packets:     250,
+					PacketsLost: 5,
+					RttMax:      400,
+					JitterMax:   30000,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.1), mos)
+		require.Equal(t, livekit.ConnectionQuality_GOOD, quality)
+
+		// mute/unmute to bring quality back up
+		now = now.Add(duration)
+		cs.UpdateMuteAt(true, now.Add(1*time.Second))
+		cs.UpdateMuteAt(false, now.Add(2*time.Second))
+
+		// bitrate based calculation can drop quality even if there is no loss
+		cs.AddBitrateTransitionAt(1_000_000, now)
+		cs.AddBitrateTransitionAt(2_000_000, now.Add(2*time.Second))
+
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime: now,
+					EndTime:   now.Add(duration),
+					Packets:   250,
+					Bytes:     8_000_000 / 8 / 5,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.1), mos)
+		require.Equal(t, livekit.ConnectionQuality_GOOD, quality)
+
+		// test layer mute via UpdateLayerMute API
+		cs.AddBitrateTransitionAt(1_000_000, now)
+		cs.AddBitrateTransitionAt(2_000_000, now.Add(2*time.Second))
+
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime: now,
+					EndTime:   now.Add(duration),
+					Packets:   250,
+					Bytes:     8_000_000 / 8 / 5,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.1), mos)
+		require.Equal(t, livekit.ConnectionQuality_GOOD, quality)
+
+		now = now.Add(duration)
+		cs.UpdateLayerMuteAt(true, now)
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.6), mos)
+		require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
+
+		// unmute layer
+		cs.UpdateLayerMuteAt(false, now.Add(2*time.Second))
+
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime: now,
+					EndTime:   now.Add(duration),
+					Packets:   250,
+					Bytes:     8_000_000 / 8 / 5,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.6), mos)
+		require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
+
+		// pause
+		now = now.Add(duration)
+		cs.UpdatePauseAt(true, now)
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(2.1), mos)
+		require.Equal(t, livekit.ConnectionQuality_POOR, quality)
+
+		// resume
+		cs.UpdatePauseAt(false, now.Add(2*time.Second))
+
+		// although conditions are perfect, climbing back from POOR (because of pause above)
+		// will only climb to GOOD.
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime: now,
+					EndTime:   now.Add(duration),
+					Packets:   250,
+					Bytes:     8_000_000 / 8 / 5,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality = cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.1), mos)
+		require.Equal(t, livekit.ConnectionQuality_GOOD, quality)
+	})
+
+	t.Run("quality scorer dependent rtt", func(t *testing.T) {
+		cs := NewConnectionStats(ConnectionStatsParams{
+			IncludeRTT:       false,
+			IncludeJitter:    true,
+			ReceiverProvider: trp,
+			Logger:           logger.GetLogger(),
+		})
+
+		duration := 5 * time.Second
+		now := time.Now()
+		cs.StartAt(mime.MimeTypeOpus, false, now.Add(-duration))
+		cs.UpdateMuteAt(false, now.Add(-1*time.Second))
+
+		// RTT does not knock quality down because it is dependent and hence not taken into account
+		// at 2% loss, quality should stay at EXCELLENT purely based on loss. With high RTT (700 ms)
+		// quality should drop to GOOD if RTT were taken into consideration
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime:   now,
+					EndTime:     now.Add(duration),
+					Packets:     250,
+					PacketsLost: 5,
+					RttMax:      700,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality := cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.6), mos)
+		require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
+	})
+
+	t.Run("quality scorer dependent jitter", func(t *testing.T) {
+		cs := NewConnectionStats(ConnectionStatsParams{
+			IncludeRTT:       true,
+			IncludeJitter:    false,
+			ReceiverProvider: trp,
+			Logger:           logger.GetLogger(),
+		})
+
+		duration := 5 * time.Second
+		now := time.Now()
+		cs.StartAt(mime.MimeTypeOpus, false, now.Add(-duration))
+		cs.UpdateMuteAt(false, now.Add(-1*time.Second))
+
+		// Jitter does not knock quality down because it is dependent and hence not taken into account
+		// at 2% loss, quality should stay at EXCELLENT purely based on loss. With high jitter (200 ms)
+		// quality should drop to GOOD if jitter were taken into consideration
+		trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+			1: {
+				RTPStats: &rtpstats.RTPDeltaInfo{
+					StartTime:   now,
+					EndTime:     now.Add(duration),
+					Packets:     250,
+					PacketsLost: 5,
+					JitterMax:   200,
+				},
+			},
+		})
+		cs.updateScoreAt(now.Add(duration))
+		mos, quality := cs.GetScoreAndQuality()
+		require.Greater(t, float32(4.6), mos)
+		require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
+	})
+
+	t.Run("codecs - packet", func(t *testing.T) {
+		type expectedQuality struct {
+			packetLossPercentage float64
+			expectedMOS          float32
+			expectedQuality      livekit.ConnectionQuality
+		}
+		testCases := []struct {
+			name              string
+			mimeType          mime.MimeType
+			isFECEnabled      bool
+			packetsExpected   uint32
+			expectedQualities []expectedQuality
+		}{
+			// NOTE: Because of EWMA (Exponentially Weighted Moving Average), these cut off points are not exact
+			// "audio/opus" - no fec - 0 <= loss < 2.5%: EXCELLENT, 2.5% <= loss < 7.5%: GOOD, >= 7.5%: POOR
+			{
+				name:            "audio/opus - no fec",
+				mimeType:        mime.MimeTypeOpus,
+				isFECEnabled:    false,
+				packetsExpected: 200,
+				expectedQualities: []expectedQuality{
+					{
+						packetLossPercentage: 1.0,
+						expectedMOS:          4.6,
+						expectedQuality:      livekit.ConnectionQuality_EXCELLENT,
+					},
+					{
+						packetLossPercentage: 4.0,
+						expectedMOS:          4.1,
+						expectedQuality:      livekit.ConnectionQuality_GOOD,
+					},
+					{
+						packetLossPercentage: 9.2,
+						expectedMOS:          2.1,
+						expectedQuality:      livekit.ConnectionQuality_POOR,
+					},
+				},
+			},
+			// "audio/opus" - fec - 0 <= loss < 3.75%: EXCELLENT, 3.75% <= loss < 11.25%: GOOD, >= 11.25%: POOR
+			{
+				name:            "audio/opus - fec",
+				mimeType:        mime.MimeTypeOpus,
+				isFECEnabled:    true,
+				packetsExpected: 200,
+				expectedQualities: []expectedQuality{
+					{
+						packetLossPercentage: 3.0,
+						expectedMOS:          4.6,
+						expectedQuality:      livekit.ConnectionQuality_EXCELLENT,
+					},
+					{
+						packetLossPercentage: 4.4,
+						expectedMOS:          4.1,
+						expectedQuality:      livekit.ConnectionQuality_GOOD,
+					},
+					{
+						packetLossPercentage: 15.0,
+						expectedMOS:          2.1,
+						expectedQuality:      livekit.ConnectionQuality_POOR,
+					},
+				},
+			},
+			// "audio/red" - no fec - 0 <= loss < 5%: EXCELLENT, 5% <= loss < 15%: GOOD, >= 15%: POOR
+			{
+				name:            "audio/red - no fec",
+				mimeType:        mime.MimeTypeRED,
+				isFECEnabled:    false,
+				packetsExpected: 200,
+				expectedQualities: []expectedQuality{
+					{
+						packetLossPercentage: 4.0,
+						expectedMOS:          4.6,
+						expectedQuality:      livekit.ConnectionQuality_EXCELLENT,
+					},
+					{
+						packetLossPercentage: 6.0,
+						expectedMOS:          4.1,
+						expectedQuality:      livekit.ConnectionQuality_GOOD,
+					},
+					{
+						packetLossPercentage: 19.5,
+						expectedMOS:          2.1,
+						expectedQuality:      livekit.ConnectionQuality_POOR,
+					},
+				},
+			},
+			// "audio/red" - fec - 0 <= loss < 7.5%: EXCELLENT, 7.5% <= loss < 22.5%: GOOD, >= 22.5%: POOR
+			{
+				name:            "audio/red - fec",
+				mimeType:        mime.MimeTypeRED,
+				isFECEnabled:    true,
+				packetsExpected: 200,
+				expectedQualities: []expectedQuality{
+					{
+						packetLossPercentage: 6.0,
+						expectedMOS:          4.6,
+						expectedQuality:      livekit.ConnectionQuality_EXCELLENT,
+					},
+					{
+						packetLossPercentage: 10.0,
+						expectedMOS:          4.1,
+						expectedQuality:      livekit.ConnectionQuality_GOOD,
+					},
+					{
+						packetLossPercentage: 30.0,
+						expectedMOS:          2.1,
+						expectedQuality:      livekit.ConnectionQuality_POOR,
+					},
+				},
+			},
+			// "video/*" - 0 <= loss < 2%: EXCELLENT, 2% <= loss < 6%: GOOD, >= 6%: POOR
+			{
+				name:            "video/*",
+				mimeType:        mime.MimeTypeVP8,
+				isFECEnabled:    false,
+				packetsExpected: 200,
+				expectedQualities: []expectedQuality{
+					{
+						packetLossPercentage: 1.0,
+						expectedMOS:          4.6,
+						expectedQuality:      livekit.ConnectionQuality_EXCELLENT,
+					},
+					{
+						packetLossPercentage: 3.5,
+						expectedMOS:          4.1,
+						expectedQuality:      livekit.ConnectionQuality_GOOD,
+					},
+					{
+						packetLossPercentage: 8.0,
+						expectedMOS:          2.1,
+						expectedQuality:      livekit.ConnectionQuality_POOR,
+					},
+				},
+			},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				cs := NewConnectionStats(ConnectionStatsParams{
+					IncludeRTT:       true,
+					IncludeJitter:    true,
+					ReceiverProvider: trp,
+					Logger:           logger.GetLogger(),
+				})
+
+				duration := 5 * time.Second
+				now := time.Now()
+				cs.StartAt(tc.mimeType, tc.isFECEnabled, now.Add(-duration))
+
+				for _, eq := range tc.expectedQualities {
+					trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+						123: {
+							RTPStats: &rtpstats.RTPDeltaInfo{
+								StartTime:   now,
+								EndTime:     now.Add(duration),
+								Packets:     tc.packetsExpected,
+								PacketsLost: uint32(math.Ceil(eq.packetLossPercentage * float64(tc.packetsExpected) / 100.0)),
+							},
+						},
+					})
+					cs.updateScoreAt(now.Add(duration))
+					mos, quality := cs.GetScoreAndQuality()
+					require.Greater(t, eq.expectedMOS, mos)
+					require.Equal(t, eq.expectedQuality, quality)
+
+					now = now.Add(duration)
+				}
+			})
+		}
+	})
+
+	t.Run("bitrate", func(t *testing.T) {
+		type transition struct {
+			bitrate int64
+			offset  time.Duration
+		}
+		testCases := []struct {
+			name            string
+			transitions     []transition
+			bytes           uint64
+			expectedMOS     float32
+			expectedQuality livekit.ConnectionQuality
+		}{
+			// NOTE: Because of EWMA (Exponentially Weighted Moving Average), these cut off points are not exact
+			// 1.0 <= expectedBits / actualBits < ~2.7 = EXCELLENT
+			// ~2.7 <= expectedBits / actualBits < ~20.1 = GOOD
+			// expectedBits / actualBits >= ~20.1 = POOR
+			{
+				name: "excellent",
+				transitions: []transition{
+					{
+						bitrate: 1_000_000,
+					},
+					{
+						bitrate: 2_000_000,
+						offset:  3 * time.Second,
+					},
+				},
+				bytes:           6_000_000 / 8,
+				expectedMOS:     4.6,
+				expectedQuality: livekit.ConnectionQuality_EXCELLENT,
+			},
+			{
+				name: "good",
+				transitions: []transition{
+					{
+						bitrate: 1_000_000,
+					},
+					{
+						bitrate: 2_000_000,
+						offset:  3 * time.Second,
+					},
+				},
+				bytes:           uint64(math.Ceil(7_000_000.0 / 8.0 / 4.2)),
+				expectedMOS:     4.1,
+				expectedQuality: livekit.ConnectionQuality_GOOD,
+			},
+			{
+				name: "poor",
+				transitions: []transition{
+					{
+						bitrate: 2_000_000,
+					},
+					{
+						bitrate: 1_000_000,
+						offset:  3 * time.Second,
+					},
+				},
+				bytes:           uint64(math.Ceil(8_000_000.0 / 8.0 / 75.0)),
+				expectedMOS:     2.1,
+				expectedQuality: livekit.ConnectionQuality_POOR,
+			},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				cs := NewConnectionStats(ConnectionStatsParams{
+					IncludeRTT:         true,
+					IncludeJitter:      true,
+					EnableBitrateScore: true,
+					ReceiverProvider:   trp,
+					Logger:             logger.GetLogger(),
+				})
+
+				duration := 5 * time.Second
+				now := time.Now()
+				cs.StartAt(mime.MimeTypeVP8, false, now)
+
+				for _, tr := range tc.transitions {
+					cs.AddBitrateTransitionAt(tr.bitrate, now.Add(tr.offset))
+				}
+
+				trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+					123: {
+						RTPStats: &rtpstats.RTPDeltaInfo{
+							StartTime: now,
+							EndTime:   now.Add(duration),
+							Packets:   100,
+							Bytes:     tc.bytes,
+						},
+					},
+				})
+				cs.updateScoreAt(now.Add(duration))
+				mos, quality := cs.GetScoreAndQuality()
+				require.Greater(t, tc.expectedMOS, mos)
+				require.Equal(t, tc.expectedQuality, quality)
+			})
+		}
+	})
+
+	t.Run("layer", func(t *testing.T) {
+		type transition struct {
+			distance float64
+			offset   time.Duration
+		}
+		testCases := []struct {
+			name            string
+			transitions     []transition
+			expectedMOS     float32
+			expectedQuality livekit.ConnectionQuality
+		}{
+			// NOTE: Because of EWMA (Exponentially Weighted Moving Average), these cut off points are not exact
+			// each spatial layer missed drops o quality level
+			{
+				name: "excellent",
+				transitions: []transition{
+					{
+						distance: 0.5,
+					},
+					{
+						distance: 0.0,
+						offset:   3 * time.Second,
+					},
+				},
+				expectedMOS:     4.6,
+				expectedQuality: livekit.ConnectionQuality_EXCELLENT,
+			},
+			{
+				name: "good",
+				transitions: []transition{
+					{
+						distance: 1.0,
+					},
+					{
+						distance: 1.5,
+						offset:   2 * time.Second,
+					},
+				},
+				expectedMOS:     4.1,
+				expectedQuality: livekit.ConnectionQuality_GOOD,
+			},
+			{
+				name: "poor",
+				transitions: []transition{
+					{
+						distance: 2.0,
+					},
+					{
+						distance: 2.6,
+						offset:   1 * time.Second,
+					},
+				},
+				expectedMOS:     2.1,
+				expectedQuality: livekit.ConnectionQuality_POOR,
+			},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				cs := NewConnectionStats(ConnectionStatsParams{
+					IncludeRTT:       true,
+					IncludeJitter:    true,
+					ReceiverProvider: trp,
+					Logger:           logger.GetLogger(),
+				})
+
+				duration := 5 * time.Second
+				now := time.Now()
+				cs.StartAt(mime.MimeTypeVP8, false, now)
+
+				for _, tr := range tc.transitions {
+					cs.AddLayerTransitionAt(tr.distance, now.Add(tr.offset))
+				}
+
+				trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
+					123: {
+						RTPStats: &rtpstats.RTPDeltaInfo{
+							StartTime: now,
+							EndTime:   now.Add(duration),
+							Packets:   200,
+						},
+					},
+				})
+				cs.updateScoreAt(now.Add(duration))
+				mos, quality := cs.GetScoreAndQuality()
+				require.Greater(t, tc.expectedMOS, mos)
+				require.Equal(t, tc.expectedQuality, quality)
+			})
+		}
+	})
+}

pkg/sfu/connectionquality/scorer.go

@@ -0,0 +1,645 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package connectionquality
+
+import (
+	"fmt"
+	"math"
+	"sync"
+	"time"
+
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils"
+	"go.uber.org/zap/zapcore"
+)
+
+const (
+	MaxMOS = float32(4.5)
+	MinMOS = float32(1.0)
+
+	cMaxScore = float64(100.0)
+	cMinScore = float64(30.0)
+
+	cIncreaseFactor = float64(0.4) // slower increase, i. e. when score is recovering move up slower -> conservative
+	cDecreaseFactor = float64(0.8) // faster decrease, i. e. when score is dropping move down faster -> aggressive to be responsive to quality drops
+
+	cDistanceWeight = float64(35.0) // each spatial layer missed drops a quality level
+
+	cUnmuteTimeThreshold = float64(0.5)
+
+	cPPSQuantization         = float64(2)
+	cPPSMinReadings          = 10
+	cModeCalculationInterval = 2 * time.Minute
+)
+
+var (
+	qualityTransitionScore = map[livekit.ConnectionQuality]float64{
+		livekit.ConnectionQuality_GOOD: 80,
+		livekit.ConnectionQuality_POOR: 40,
+		livekit.ConnectionQuality_LOST: 20,
+	}
+)
+
+// ------------------------------------------
+
+type windowStat struct {
+	startedAt         time.Time
+	duration          time.Duration
+	packets           uint32
+	packetsPadding    uint32
+	packetsLost       uint32
+	packetsMissing    uint32
+	packetsOutOfOrder uint32
+	bytes             uint64
+	rttMax            uint32
+	jitterMax         float64
+	lastRTCPAt        time.Time
+}
+
+func (w *windowStat) calculatePacketScore(aplw float64, includeRTT bool, includeJitter bool) float64 {
+	// this is based on simplified E-model based on packet loss, rtt, jitter as
+	// outlined at https://www.pingman.com/kb/article/how-is-mos-calculated-in-pingplotter-pro-50.html.
+	effectiveDelay := 0.0
+	// discount the dependent factors if dependency indicated.
+	// for example,
+	// 1. in the up stream, RTT cannot be measured without RTCP-XR, it is using down stream RTT.
+	// 2. in the down stream, up stream jitter affects it. although jitter can be adjusted to account for up stream
+	//    jitter, this lever can be used to discount jitter in scoring.
+	if includeRTT {
+		effectiveDelay += float64(w.rttMax) / 2.0
+	}
+	if includeJitter {
+		effectiveDelay += (w.jitterMax * 2.0) / 1000.0
+	}
+	delayEffect := effectiveDelay / 40.0
+	if effectiveDelay > 160.0 {
+		delayEffect = (effectiveDelay - 120.0) / 10.0
+	}
+
+	// discount out-of-order packets from loss to deal with a scenario like
+	// 1. up stream has loss
+	// 2. down stream forwards with loss/hole in sequence number
+	// 3. down stream client reports a certain number of loss via RTCP RR
+	// 4. while processing that RTCP RR, up stream could have retransmitted missing packets
+	// 5. those retransmitted packets are forwarded,
+	//    - server's view: it has forwarded those packets
+	//    - client's view: it had not seen those packets when sending RTCP RR
+	//    so those retransmitted packets appear like down stream loss to server.
+	//
+	// retransmitted packets would have arrived out-of-order. So, discounting them
+	// will account for it.
+	//
+	// Note that packets can arrive out-of-order in the upstream during regular
+	// streaming as well, i. e. without loss + NACK + retransmit. Those will be
+	// discounted too. And that will skew the real loss. For example, let
+	// us say that 40 out of 100 packets were reported lost by down stream.
+	// These could be real losses. In the same window, 40 packets could have been
+	// delivered out-of-order by the up stream, thus cancelling out the real loss.
+	// But, those situations should be rare and is a compromise for not letting
+	// up stream loss penalise down stream.
+	actualLost := w.packetsLost - w.packetsMissing - w.packetsOutOfOrder
+	if int32(actualLost) < 0 {
+		actualLost = 0
+	}
+
+	var lossEffect float64
+	if w.packets+w.packetsPadding > 0 {
+		lossEffect = float64(actualLost) * 100.0 / float64(w.packets+w.packetsPadding)
+	}
+	lossEffect *= aplw
+
+	score := cMaxScore - delayEffect - lossEffect
+	if score < 0.0 {
+		score = 0.0
+	}
+
+	return score
+}
+
+func (w *windowStat) calculateBitrateScore(expectedBits int64, isEnabled bool) float64 {
+	if expectedBits == 0 || !isEnabled {
+		// unsupported mode OR all layers stopped
+		return cMaxScore
+	}
+
+	var score float64
+	if w.bytes != 0 {
+		// using the ratio of expectedBits / actualBits
+		// the quality inflection points are approximately
+		// GOOD at ~2.7x, POOR at ~20.1x
+		score = cMaxScore - 20*math.Log(float64(expectedBits)/float64(w.bytes*8))
+		if score > cMaxScore {
+			score = cMaxScore
+		}
+		if score < 0.0 {
+			score = 0.0
+		}
+	}
+
+	return score
+}
+
+func (w *windowStat) String() string {
+	return fmt.Sprintf("start: %+v, dur: %+v, p: %d, pp: %d, pl: %d, pm: %d, pooo: %d, b: %d, rtt: %d, jitter: %0.2f, lastRTCP: %+v",
+		w.startedAt,
+		w.duration,
+		w.packets,
+		w.packetsPadding,
+		w.packetsLost,
+		w.packetsMissing,
+		w.packetsOutOfOrder,
+		w.bytes,
+		w.rttMax,
+		w.jitterMax,
+		w.lastRTCPAt,
+	)
+}
+
+func (w *windowStat) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if w == nil {
+		return nil
+	}
+
+	e.AddTime("startedAt", w.startedAt)
+	e.AddString("duration", w.duration.String())
+	e.AddUint32("packets", w.packets)
+	e.AddUint32("packetsPadding", w.packetsPadding)
+	e.AddUint32("packetsLost", w.packetsLost)
+	e.AddUint32("packetsMissing", w.packetsMissing)
+	e.AddUint32("packetsOutOfOrder", w.packetsOutOfOrder)
+	e.AddUint64("bytes", w.bytes)
+	e.AddUint32("rttMax", w.rttMax)
+	e.AddFloat64("jitterMax", w.jitterMax)
+	e.AddTime("lastRTCPAt", w.lastRTCPAt)
+	return nil
+}
+
+// ------------------------------------------
+
+type qualityScorerParams struct {
+	IncludeRTT         bool
+	IncludeJitter      bool
+	EnableBitrateScore bool
+	Logger             logger.Logger
+}
+
+type qualityScorer struct {
+	params qualityScorerParams
+
+	lock         sync.RWMutex
+	lastUpdateAt time.Time
+
+	packetLossWeight float64
+
+	score float64
+	stat  windowStat
+
+	mutedAt   time.Time
+	unmutedAt time.Time
+
+	layerMutedAt   time.Time
+	layerUnmutedAt time.Time
+
+	pausedAt  time.Time
+	resumedAt time.Time
+
+	ppsHistogram     [250]int
+	numPPSReadings   int
+	ppsMode          int
+	modeCalculatedAt time.Time
+
+	aggregateBitrate *utils.TimedAggregator[int64]
+	layerDistance    *utils.TimedAggregator[float64]
+}
+
+func newQualityScorer(params qualityScorerParams) *qualityScorer {
+	return &qualityScorer{
+		params: params,
+		score:  cMaxScore,
+		aggregateBitrate: utils.NewTimedAggregator[int64](utils.TimedAggregatorParams{
+			CapNegativeValues: true,
+		}),
+		layerDistance: utils.NewTimedAggregator[float64](utils.TimedAggregatorParams{
+			CapNegativeValues: true,
+		}),
+		modeCalculatedAt: time.Now().Add(-cModeCalculationInterval),
+	}
+}
+
+func (q *qualityScorer) startAtLocked(packetLossWeight float64, at time.Time) {
+	q.packetLossWeight = packetLossWeight
+	q.lastUpdateAt = at
+}
+
+func (q *qualityScorer) StartAt(packetLossWeight float64, at time.Time) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.startAtLocked(packetLossWeight, at)
+}
+
+func (q *qualityScorer) Start(packetLossWeight float64) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.startAtLocked(packetLossWeight, time.Now())
+}
+
+func (q *qualityScorer) UpdatePacketLossWeight(packetLossWeight float64) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.packetLossWeight = packetLossWeight
+}
+
+func (q *qualityScorer) updateMuteAtLocked(isMuted bool, at time.Time) {
+	if isMuted {
+		q.mutedAt = at
+		// muting when LOST should not push quality to EXCELLENT
+		if q.score != qualityTransitionScore[livekit.ConnectionQuality_LOST] {
+			q.score = cMaxScore
+		}
+	} else {
+		q.unmutedAt = at
+	}
+}
+
+func (q *qualityScorer) UpdateMuteAt(isMuted bool, at time.Time) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.updateMuteAtLocked(isMuted, at)
+}
+
+func (q *qualityScorer) UpdateMute(isMuted bool) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.updateMuteAtLocked(isMuted, time.Now())
+}
+
+func (q *qualityScorer) addBitrateTransitionAtLocked(bitrate int64, at time.Time) {
+	q.aggregateBitrate.AddSampleAt(bitrate, at)
+}
+
+func (q *qualityScorer) AddBitrateTransitionAt(bitrate int64, at time.Time) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.addBitrateTransitionAtLocked(bitrate, at)
+}
+
+func (q *qualityScorer) AddBitrateTransition(bitrate int64) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.addBitrateTransitionAtLocked(bitrate, time.Now())
+}
+
+func (q *qualityScorer) updateLayerMuteAtLocked(isMuted bool, at time.Time) {
+	if isMuted {
+		if !q.isLayerMuted() {
+			q.aggregateBitrate.Reset()
+			q.layerDistance.Reset()
+			q.layerMutedAt = at
+			q.score = cMaxScore
+		}
+	} else {
+		if q.isLayerMuted() {
+			q.layerUnmutedAt = at
+		}
+	}
+}
+
+func (q *qualityScorer) UpdateLayerMuteAt(isMuted bool, at time.Time) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.updateLayerMuteAtLocked(isMuted, at)
+}
+
+func (q *qualityScorer) UpdateLayerMute(isMuted bool) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.updateLayerMuteAtLocked(isMuted, time.Now())
+}
+
+func (q *qualityScorer) updatePauseAtLocked(isPaused bool, at time.Time) {
+	if isPaused {
+		if !q.isPaused() {
+			q.aggregateBitrate.Reset()
+			q.layerDistance.Reset()
+			q.pausedAt = at
+			q.score = cMinScore
+		}
+	} else {
+		if q.isPaused() {
+			q.resumedAt = at
+		}
+	}
+}
+
+func (q *qualityScorer) UpdatePauseAt(isPaused bool, at time.Time) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.updatePauseAtLocked(isPaused, at)
+}
+
+func (q *qualityScorer) UpdatePause(isPaused bool) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.updatePauseAtLocked(isPaused, time.Now())
+}
+
+func (q *qualityScorer) addLayerTransitionAtLocked(distance float64, at time.Time) {
+	q.layerDistance.AddSampleAt(distance, at)
+}
+
+func (q *qualityScorer) AddLayerTransitionAt(distance float64, at time.Time) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.addLayerTransitionAtLocked(distance, at)
+}
+
+func (q *qualityScorer) AddLayerTransition(distance float64) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.addLayerTransitionAtLocked(distance, time.Now())
+}
+
+func (q *qualityScorer) updateAtLocked(stat *windowStat, at time.Time) {
+	// always update transitions
+	expectedBits, _, err := q.aggregateBitrate.GetAggregateAndRestartAt(at)
+	if err != nil {
+		q.params.Logger.Warnw("error getting expected bitrate", err)
+	}
+	expectedDistance, err := q.layerDistance.GetAverageAndRestartAt(at)
+	if err != nil {
+		q.params.Logger.Warnw("error getting expected distance", err)
+	}
+
+	// nothing to do when muted or not unmuted for long enough
+	// NOTE: it is possible that unmute -> mute -> unmute transition happens in the
+	//       same analysis window. On a transition to mute, quality is immediately moved
+	//       EXCELLENT for responsiveness. On an unmute, the entire window data is
+	//       considered (as long as enough time has passed since unmute).
+	//
+	//       Similarly, when paused (possibly due to congestion), score is immediately
+	//       set to cMinScore for responsiveness. The layer transition is reset.
+	//       On a resume, quality climbs back up using normal operation.
+	if q.isMuted() || !q.isUnmutedEnough(at) || q.isLayerMuted() || q.isPaused() {
+		q.lastUpdateAt = at
+		return
+	}
+
+	aplw := q.getAdjustedPacketLossWeight(stat)
+	reason := "none"
+	var score, packetScore, bitrateScore, layerScore float64
+	if stat.packets+stat.packetsPadding == 0 {
+		if !stat.lastRTCPAt.IsZero() && at.Sub(stat.lastRTCPAt) > stat.duration {
+			reason = "rtcp"
+			score = qualityTransitionScore[livekit.ConnectionQuality_LOST]
+		} else {
+			reason = "dry"
+			score = qualityTransitionScore[livekit.ConnectionQuality_POOR]
+		}
+	} else {
+		packetScore = stat.calculatePacketScore(aplw, q.params.IncludeRTT, q.params.IncludeJitter)
+		bitrateScore = stat.calculateBitrateScore(expectedBits, q.params.EnableBitrateScore)
+		layerScore = math.Max(math.Min(cMaxScore, cMaxScore-(expectedDistance*cDistanceWeight)), 0.0)
+
+		minScore := math.Min(packetScore, bitrateScore)
+		minScore = math.Min(minScore, layerScore)
+
+		switch {
+		case packetScore == minScore:
+			reason = "packet"
+			score = packetScore
+
+		case bitrateScore == minScore:
+			reason = "bitrate"
+			score = bitrateScore
+
+		case layerScore == minScore:
+			reason = "layer"
+			score = layerScore
+		}
+
+		factor := cIncreaseFactor
+		if score < q.score {
+			factor = cDecreaseFactor
+		}
+		score = factor*score + (1.0-factor)*q.score
+		if score < cMinScore {
+			// lower bound to prevent score from becoming very small values due to extreme conditions.
+			// Without a lower bound, it can get so low that it takes a long time to climb back to
+			// better quality even under excellent conditions.
+			score = cMinScore
+		}
+	}
+
+	prevCQ := scoreToConnectionQuality(q.score)
+	currCQ := scoreToConnectionQuality(score)
+	ulgr := q.params.Logger.WithUnlikelyValues(
+		"reason", reason,
+		"prevScore", q.score,
+		"prevQuality", prevCQ,
+		"prevStat", &q.stat,
+		"score", score,
+		"packetScore", packetScore,
+		"layerScore", layerScore,
+		"bitrateScore", bitrateScore,
+		"quality", currCQ,
+		"stat", stat,
+		"packetLossWeight", q.packetLossWeight,
+		"adjustedPacketLossWeight", aplw,
+		"modePPS", q.ppsMode*int(cPPSQuantization),
+		"expectedBits", expectedBits,
+		"expectedDistance", expectedDistance,
+	)
+	switch {
+	case utils.IsConnectionQualityLower(prevCQ, currCQ):
+		ulgr.Debugw("quality drop")
+	case utils.IsConnectionQualityHigher(prevCQ, currCQ):
+		ulgr.Debugw("quality rise")
+	default:
+		packets := stat.packets + stat.packetsPadding
+		lost := stat.packetsLost - stat.packetsMissing - stat.packetsOutOfOrder
+		if int32(lost) < 0 {
+			lost = 0
+		}
+		if packets != 0 && lost*100/packets > 10 {
+			ulgr.Debugw("quality hold - high loss")
+		}
+	}
+
+	q.score = score
+	q.stat = *stat
+	q.lastUpdateAt = at
+}
+
+func (q *qualityScorer) UpdateAt(stat *windowStat, at time.Time) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.updateAtLocked(stat, at)
+}
+
+func (q *qualityScorer) Update(stat *windowStat) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.updateAtLocked(stat, time.Now())
+}
+
+func (q *qualityScorer) isMuted() bool {
+	return !q.mutedAt.IsZero() && (q.unmutedAt.IsZero() || q.mutedAt.After(q.unmutedAt))
+}
+
+func (q *qualityScorer) isUnmutedEnough(at time.Time) bool {
+	var sinceUnmute time.Duration
+	if q.unmutedAt.IsZero() {
+		sinceUnmute = at.Sub(q.lastUpdateAt)
+	} else {
+		sinceUnmute = at.Sub(q.unmutedAt)
+	}
+
+	var sinceLayerUnmute time.Duration
+	if q.layerUnmutedAt.IsZero() {
+		sinceLayerUnmute = at.Sub(q.lastUpdateAt)
+	} else {
+		sinceLayerUnmute = at.Sub(q.layerUnmutedAt)
+	}
+
+	validDuration := min(sinceLayerUnmute, sinceUnmute)
+
+	sinceLastUpdate := at.Sub(q.lastUpdateAt)
+
+	return validDuration.Seconds()/sinceLastUpdate.Seconds() > cUnmuteTimeThreshold
+}
+
+func (q *qualityScorer) isLayerMuted() bool {
+	return !q.layerMutedAt.IsZero() && (q.layerUnmutedAt.IsZero() || q.layerMutedAt.After(q.layerUnmutedAt))
+}
+
+func (q *qualityScorer) isPaused() bool {
+	return !q.pausedAt.IsZero() && (q.resumedAt.IsZero() || q.pausedAt.After(q.resumedAt))
+}
+
+func (q *qualityScorer) getAdjustedPacketLossWeight(stat *windowStat) float64 {
+	if stat == nil || stat.duration <= 0 {
+		return q.packetLossWeight
+	}
+
+	// packet loss is weighted by comparing against mode of packet rate seen.
+	// this is to handle situations like DTX in audio and variable bit rate tracks like screen share.
+	// and the effect of loss is not pronounced in those scenarios (audio silence, static screen share).
+	// for example, DTX typically uses only 5% of packets of full packet rate. at that rate,
+	// packet loss weight is reduced to ~22% of configured weight (i. e. sqrt(0.05) * configured weight)
+	pps := float64(stat.packets) / stat.duration.Seconds()
+	ppsQuantized := int(pps/cPPSQuantization + 0.5)
+	if ppsQuantized < len(q.ppsHistogram)-1 {
+		q.ppsHistogram[ppsQuantized]++
+	} else {
+		q.ppsHistogram[len(q.ppsHistogram)-1]++
+	}
+	q.numPPSReadings++
+
+	// calculate mode sparingly, do it under the following conditions
+	//  1. minimum number of readings available (AND)
+	//  2. enough time has elapsed since last calculation
+	if q.numPPSReadings > cPPSMinReadings && time.Since(q.modeCalculatedAt) > cModeCalculationInterval {
+		q.ppsMode = 0
+		for i := range len(q.ppsHistogram) {
+			if q.ppsHistogram[i] > q.ppsMode {
+				q.ppsMode = i
+			}
+		}
+		q.modeCalculatedAt = time.Now()
+		q.params.Logger.Debugw("updating pps mode", "expected", stat.packets, "duration", stat.duration.Seconds(), "pps", pps, "ppsMode", q.ppsMode)
+	}
+
+	if q.ppsMode == 0 || q.ppsMode == len(q.ppsHistogram)-1 {
+		return q.packetLossWeight
+	}
+
+	packetRatio := pps / (float64(q.ppsMode) * cPPSQuantization)
+	if packetRatio > 1.0 {
+		packetRatio = 1.0
+	}
+	return math.Sqrt(packetRatio) * q.packetLossWeight
+}
+
+func (q *qualityScorer) GetScoreAndQuality() (float32, livekit.ConnectionQuality) {
+	q.lock.RLock()
+	defer q.lock.RUnlock()
+
+	return float32(q.score), scoreToConnectionQuality(q.score)
+}
+
+func (q *qualityScorer) GetMOSAndQuality() (float32, livekit.ConnectionQuality) {
+	q.lock.RLock()
+	defer q.lock.RUnlock()
+
+	return scoreToMOS(q.score), scoreToConnectionQuality(q.score)
+}
+
+// ------------------------------------------
+
+func scoreToConnectionQuality(score float64) livekit.ConnectionQuality {
+	// R-factor -> livekit.ConnectionQuality scale mapping roughly based on
+	// https://www.itu.int/ITU-T/2005-2008/com12/emodelv1/tut.htm
+	//
+	// As there are only three levels in livekit.ConnectionQuality scale,
+	// using a larger range for middling quality. Empirical evidence suggests
+	// that a score of 60 does not correspond to `POOR` quality. Repair
+	// mechanisms and use of algorithms like de-jittering makes the experience
+	// better even under harsh conditions.
+	if score > qualityTransitionScore[livekit.ConnectionQuality_GOOD] {
+		return livekit.ConnectionQuality_EXCELLENT
+	}
+
+	if score > qualityTransitionScore[livekit.ConnectionQuality_POOR] {
+		return livekit.ConnectionQuality_GOOD
+	}
+
+	if score > qualityTransitionScore[livekit.ConnectionQuality_LOST] {
+		return livekit.ConnectionQuality_POOR
+	}
+
+	return livekit.ConnectionQuality_LOST
+}
+
+// ------------------------------------------
+
+func scoreToMOS(score float64) float32 {
+	if score <= 0.0 {
+		return 1.0
+	}
+
+	if score >= 100.0 {
+		return 4.5
+	}
+
+	return float32(1.0 + 0.035*score + (0.000007 * score * (score - 60.0) * (100.0 - score)))
+}
+
+// ------------------------------------------

pkg/sfu/datachannel/bitrate.go

@@ -0,0 +1,175 @@
+package datachannel
+
+import (
+	"sync"
+	"time"
+
+	"github.com/gammazero/deque"
+
+	"github.com/livekit/protocol/utils/mono"
+)
+
+const (
+	BitrateDuration = 2 * time.Second
+	BitrateWindow   = 100 * time.Millisecond
+)
+
+// BitrateMode selects how the sliding-window rate is computed.
+type BitrateMode int
+
+const (
+	// BitrateModeBusyOnly divides drained bytes by backlogged ("busy") time
+	// only; idle gaps are excluded from the denominator. All drained bytes count.
+	BitrateModeBusyOnly BitrateMode = iota
+
+	// BitrateModeExcludeIdleDrain is like BitrateModeBusyOnly but also excludes
+	// bytes drained across a non-backlogged interval from the numerator.
+	BitrateModeExcludeIdleDrain
+
+	// BitrateModeWallClock divides all bytes by elapsed wall-clock time. For
+	// consumers with no send buffer, e.g. a receiver measuring its incoming rate.
+	BitrateModeWallClock
+)
+
+func (m BitrateMode) String() string {
+	switch m {
+	case BitrateModeBusyOnly:
+		return "busyOnly"
+	case BitrateModeExcludeIdleDrain:
+		return "excludeIdleDrain"
+	case BitrateModeWallClock:
+		return "wallClock"
+	default:
+		return "unknown"
+	}
+}
+
+// BitrateCalculator calculates bitrate over sliding window
+type BitrateCalculator struct {
+	lock           sync.Mutex
+	windowDuration time.Duration
+	duration       time.Duration
+
+	windows deque.Deque[bitrateWindow]
+	active  bitrateWindow
+
+	bytes              int
+	busy               time.Duration
+	lastBufferedAmount int
+	last               time.Time
+	start              time.Time
+	mode               BitrateMode
+}
+
+// NewBitrateCalculator creates a calculator. See BitrateMode for how mode
+// governs the numerator and denominator.
+func NewBitrateCalculator(duration time.Duration, window time.Duration, mode BitrateMode) *BitrateCalculator {
+	windowCnt := int((duration + (window - 1)) / window)
+	if windowCnt == 0 {
+		windowCnt = 1
+	}
+	now := mono.Now()
+	c := &BitrateCalculator{
+		duration:       duration,
+		windowDuration: window,
+		start:          now,
+		active:         bitrateWindow{start: now},
+		mode:           mode,
+	}
+	c.windows.SetBaseCap(windowCnt + 1)
+
+	return c
+}
+
+func (c *BitrateCalculator) AddBytes(bytes int, bufferedAmout int, ts time.Time) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	// bufferedAmout is sampled right after the write, so it includes the bytes
+	// just enqueued; subtracting them gives the buffer occupancy before the
+	// write. Between writes the buffer only drains, so a non-zero pre-write
+	// level means it stayed backlogged for the whole interval.
+	backlogged := bufferedAmout-bytes > 0
+	var busy time.Duration
+	if !c.last.IsZero() && backlogged {
+		if busy = ts.Sub(c.last); busy < 0 {
+			busy = 0
+		}
+	}
+	c.last = ts
+
+	// bytes that actually drained since the previous sample
+	bytes -= bufferedAmout - c.lastBufferedAmount
+	if bytes < 0 {
+		// it is possible that internal buffering (non-data like DCEP packet from webrtc) caused bytes to be negative
+		bytes = 0
+	}
+	c.lastBufferedAmount = bufferedAmout
+
+	if c.mode == BitrateModeExcludeIdleDrain && !backlogged {
+		// drain across a non-backlogged interval finished at an unknown point,
+		// so it is not a reliable rate sample
+		bytes = 0
+	}
+
+	if ts.Sub(c.active.start) >= c.windowDuration {
+		c.windows.PushBack(c.active)
+		c.active.start = ts
+		c.active.bytes = 0
+		c.active.busy = 0
+
+		for c.windows.Len() > 0 {
+			// pop expired windows
+			if w := c.windows.Front(); ts.Sub(w.start) > (c.duration + c.windowDuration) {
+				c.bytes -= w.bytes
+				c.busy -= w.busy
+				c.windows.PopFront()
+			} else {
+				c.start = w.start
+				break
+			}
+		}
+		if c.windows.Len() == 0 {
+			c.start = ts
+			c.bytes = 0
+			c.busy = 0
+		}
+	}
+	c.bytes += bytes
+	c.active.bytes += bytes
+	c.busy += busy
+	c.active.busy += busy
+}
+
+func (c *BitrateCalculator) Bitrate(ts time.Time) (int, bool) {
+	return c.bitrate(ts, false)
+}
+
+func (c *BitrateCalculator) ForceBitrate(ts time.Time) (int, bool) {
+	return c.bitrate(ts, true)
+}
+
+func (c *BitrateCalculator) bitrate(ts time.Time, force bool) (int, bool) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	// busy modes divide by backlogged time; wall-clock mode by elapsed time
+	denom := c.busy
+	if c.mode == BitrateModeWallClock {
+		denom = ts.Sub(c.start)
+	}
+	if denom < c.windowDuration {
+		if force {
+			denom = c.windowDuration
+		} else {
+			return 0, false
+		}
+	}
+
+	return c.bytes * 8 * 1000 / int(denom.Milliseconds()), true
+}
+
+type bitrateWindow struct {
+	start time.Time
+	bytes int
+	busy  time.Duration
+}

pkg/sfu/datachannel/bitrate_test.go

@@ -0,0 +1,130 @@
+package datachannel
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+var busyModes = []BitrateMode{BitrateModeBusyOnly, BitrateModeExcludeIdleDrain}
+
+// 1 KB written every second, each fully drained before the next write: never
+// backlogged, so there is no estimate.
+func TestBitrateCalculatorSparseFastAck(t *testing.T) {
+	for _, mode := range busyModes {
+		t.Run(mode.String(), func(t *testing.T) {
+			c := NewBitrateCalculator(BitrateDuration, BitrateWindow, mode)
+			require.NotNil(t, c)
+
+			t0 := time.Now()
+			for i := 0; i < 5; i++ {
+				ts := t0.Add(time.Duration(i) * time.Second)
+				c.AddBytes(1000, 1000, ts)
+				_, ok := c.Bitrate(ts)
+				require.False(t, ok)
+			}
+		})
+	}
+}
+
+// Standing backlog of 1000 bytes; every 100ms write 800 and 800 drain, sampled
+// buffer 1000+800=1800. 800 B / 100ms = 64000 bps.
+func TestBitrateCalculatorSustainedBacklog(t *testing.T) {
+	for _, mode := range busyModes {
+		t.Run(mode.String(), func(t *testing.T) {
+			c := NewBitrateCalculator(BitrateDuration, BitrateWindow, mode)
+
+			t0 := time.Now()
+			c.AddBytes(800, 1800, t0)
+			for i := 1; i <= 30; i++ {
+				c.AddBytes(800, 1800, t0.Add(time.Duration(i)*100*time.Millisecond))
+			}
+
+			bitrate, ok := c.Bitrate(t0.Add(3 * time.Second))
+			require.True(t, ok)
+			require.InDelta(t, 64000, bitrate, 2000)
+		})
+	}
+}
+
+// Buffer only climbs and nothing drains (stalled connection): the estimate
+// converges to ~0.
+func TestBitrateCalculatorStalledConnection(t *testing.T) {
+	for _, mode := range busyModes {
+		t.Run(mode.String(), func(t *testing.T) {
+			c := NewBitrateCalculator(BitrateDuration, BitrateWindow, mode)
+
+			t0 := time.Now()
+			buffered := 0
+			for i := 0; i < 20; i++ {
+				buffered += 1000 // wrote 1000, nothing drained
+				c.AddBytes(1000, buffered, t0.Add(time.Duration(i)*50*time.Millisecond))
+			}
+
+			bitrate, ok := c.Bitrate(t0.Add(time.Second))
+			require.True(t, ok)
+			require.Less(t, bitrate, 1000)
+		})
+	}
+}
+
+// A single write that drains to empty is not backlogged at the next sample, so
+// no estimate is produced.
+func TestBitrateCalculatorFlushNotCounted(t *testing.T) {
+	for _, mode := range busyModes {
+		t.Run(mode.String(), func(t *testing.T) {
+			c := NewBitrateCalculator(BitrateDuration, BitrateWindow, mode)
+
+			t0 := time.Now()
+			c.AddBytes(100, 100, t0)
+			ts := t0.Add(100 * time.Millisecond)
+			c.AddBytes(0, 0, ts)
+			_, ok := c.Bitrate(ts)
+			require.False(t, ok)
+		})
+	}
+}
+
+// An idle drain followed by a backlog: BusyOnly counts the idle-drained bytes
+// over busy time only and reports a higher rate than ExcludeIdleDrain.
+func TestBitrateCalculatorMixedIdleThenBacklog(t *testing.T) {
+	off := NewBitrateCalculator(BitrateDuration, BitrateWindow, BitrateModeBusyOnly)
+	on := NewBitrateCalculator(BitrateDuration, BitrateWindow, BitrateModeExcludeIdleDrain)
+
+	t0 := time.Now()
+	feed := func(written, buffered int, ts time.Time) {
+		off.AddBytes(written, buffered, ts)
+		on.AddBytes(written, buffered, ts)
+	}
+
+	// 5000 drains during an idle gap (not backlogged), then a sustained backlog
+	// of 800 every 100ms over a standing 5000.
+	feed(5000, 5000, t0)
+	feed(5000, 5000, t0.Add(100*time.Millisecond))
+	for i := 1; i <= 5; i++ {
+		feed(800, 5800, t0.Add(time.Duration(100+i*100)*time.Millisecond))
+	}
+	queryTs := t0.Add(700 * time.Millisecond)
+
+	rOff, okOff := off.Bitrate(queryTs)
+	rOn, okOn := on.Bitrate(queryTs)
+	require.True(t, okOff)
+	require.True(t, okOn)
+	require.Greater(t, rOff, rOn)
+}
+
+// No observable buffer: throughput over elapsed time. 1000 B every 100ms =
+// 80000 bps.
+func TestBitrateCalculatorWallClock(t *testing.T) {
+	c := NewBitrateCalculator(BitrateDuration, BitrateWindow, BitrateModeWallClock)
+
+	t0 := time.Now()
+	for i := 0; i < 20; i++ {
+		c.AddBytes(1000, 0, t0.Add(time.Duration(i)*100*time.Millisecond))
+	}
+
+	bitrate, ok := c.Bitrate(t0.Add(2 * time.Second))
+	require.True(t, ok)
+	require.InDelta(t, 80000, bitrate, 8000)
+}

pkg/sfu/datachannel/datachannel_writer.go

@@ -0,0 +1,152 @@
+package datachannel
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/pion/datachannel"
+
+	"github.com/livekit/protocol/utils/mono"
+)
+
+const (
+	singleWriteTimeout = 50 * time.Millisecond
+)
+
+var (
+	ErrDataDroppedBySlowReader         = errors.New("data dropped by slow reader")
+	ErrDataDroppedByHighBufferedAmount = errors.New("data dropped due to high buffered amount")
+)
+
+type BufferedAmountGetter interface {
+	BufferedAmount() uint64
+}
+
+type DataChannelWriter[T BufferedAmountGetter] struct {
+	bufferGetter      T
+	rawDC             datachannel.ReadWriteCloserDeadliner
+	slowThreshold     int
+	rate              *BitrateCalculator
+	reliable          bool
+	targetLatency     time.Duration
+	minBufferedAmount uint64
+}
+
+// NewDataChannelWriterReliable creates a new DataChannelWriter for reliable data channel by
+// detaching it, when writing to the datachanel times out, it will block and retry if the
+// receiver's bitrate is above the slowThreshold or drop the data if it's below the threshold.
+// If the slowThreshold is 0, it will never retry on write timeout.
+func NewDataChannelWriterReliable[T BufferedAmountGetter](bufferGetter T, rawDC datachannel.ReadWriteCloserDeadliner, slowThreshold int) *DataChannelWriter[T] {
+	var rate *BitrateCalculator
+	if slowThreshold > 0 {
+		rate = NewBitrateCalculator(BitrateDuration, BitrateWindow, BitrateModeBusyOnly)
+	}
+	return &DataChannelWriter[T]{
+		bufferGetter:  bufferGetter,
+		rawDC:         rawDC,
+		slowThreshold: slowThreshold,
+		rate:          rate,
+		reliable:      true,
+	}
+}
+
+// NewDataChannelWriterUnreliable creates a new DataChannelWriter for unreliable data channel.
+// It will drop data when the buffered amount is too high to maintain the target latency.
+// The latency is estimated based on the bitrate in past 1 second. If targetLatency is 0, no
+// buffering control is applied.
+func NewDataChannelWriterUnreliable[T BufferedAmountGetter](bufferGetter T, rawDC datachannel.ReadWriteCloserDeadliner, targetLatency time.Duration, minBufferedAmount uint64) *DataChannelWriter[T] {
+	var rate *BitrateCalculator
+	if targetLatency > 0 {
+		rate = NewBitrateCalculator(BitrateDuration, BitrateWindow, BitrateModeBusyOnly)
+	}
+	return &DataChannelWriter[T]{
+		bufferGetter:      bufferGetter,
+		rawDC:             rawDC,
+		rate:              rate,
+		targetLatency:     targetLatency,
+		minBufferedAmount: minBufferedAmount,
+		reliable:          false,
+	}
+}
+
+func (w *DataChannelWriter[T]) BufferedAmountGetter() T {
+	return w.bufferGetter
+}
+
+func (w *DataChannelWriter[T]) Write(p []byte) (n int, err error) {
+	if w.reliable {
+		return w.writeReliable(p)
+	} else {
+		return w.writeUnreliable(p)
+	}
+}
+
+func (w *DataChannelWriter[T]) writeReliable(p []byte) (n int, err error) {
+	for {
+		err = w.rawDC.SetWriteDeadline(time.Now().Add(singleWriteTimeout))
+		if err != nil {
+			return 0, err
+		}
+		n, err = w.rawDC.Write(p)
+		if w.slowThreshold == 0 {
+			return
+		}
+
+		now := mono.Now()
+		w.rate.AddBytes(n, int(w.bufferGetter.BufferedAmount()), now)
+		// retry if the write timed out on a non-slow receiver
+		if errors.Is(err, context.DeadlineExceeded) {
+			if bitrate, ok := w.rate.Bitrate(now); !ok || bitrate >= w.slowThreshold {
+				continue
+			} else {
+				err = fmt.Errorf("%w: bitrate %d, threshold %d", ErrDataDroppedBySlowReader, bitrate, w.slowThreshold)
+			}
+		}
+
+		return
+	}
+}
+
+func (w *DataChannelWriter[T]) writeUnreliable(p []byte) (n int, err error) {
+	if w.targetLatency == 0 {
+		err = w.rawDC.SetWriteDeadline(time.Now().Add(singleWriteTimeout))
+		if err != nil {
+			return 0, err
+		}
+		return w.rawDC.Write(p)
+	}
+
+	if bitrate, ok := w.rate.Bitrate(time.Now()); ok {
+		// control buffer latency to target
+		bufferedAmount := w.bufferGetter.BufferedAmount()
+		targetLatencyLimit := uint64(time.Duration(bitrate) * w.targetLatency / 8 / time.Second)
+		if bufferedAmount > targetLatencyLimit && bufferedAmount > w.minBufferedAmount {
+			return 0, fmt.Errorf(
+				"%w: bitrate %d, buffered amount %d, target latency limit %d, min buffered amount %d",
+				ErrDataDroppedByHighBufferedAmount,
+				bitrate,
+				bufferedAmount,
+				targetLatencyLimit,
+				w.minBufferedAmount,
+			)
+		}
+	}
+
+	err = w.rawDC.SetWriteDeadline(time.Now().Add(singleWriteTimeout))
+	if err != nil {
+		return 0, err
+	}
+	n, err = w.rawDC.Write(p)
+	// Track drained bytes on every write (not just failures) so the bitrate
+	// calculator has samples; otherwise Bitrate() has no data and the
+	// latency-based drop threshold above collapses to the static minBufferedAmount.
+	w.rate.AddBytes(n, int(w.bufferGetter.BufferedAmount()), mono.Now())
+
+	return
+}
+
+func (w *DataChannelWriter[T]) Close() error {
+	return w.rawDC.Close()
+}

pkg/sfu/datachannel/datachannel_writer_test.go

@@ -0,0 +1,160 @@
+package datachannel
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"go.uber.org/atomic"
+
+	"github.com/pion/datachannel"
+	"github.com/pion/transport/v4/deadline"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDataChannelWriter(t *testing.T) {
+	mockDC := newMockDataChannelWriter()
+	// slow threshold is 1000B/s
+	w := NewDataChannelWriterReliable(mockDC, mockDC, 8000)
+	require.Equal(t, mockDC, w.BufferedAmountGetter())
+	buf := make([]byte, 2000)
+	// write 2000 bytes so it should not drop in 2 seconds
+	t0 := time.Now()
+	n, err := w.Write(buf)
+	require.NoError(t, err)
+	require.Equal(t, 2000, n)
+
+	t1 := time.Now()
+	mockDC.SetNextWriteCompleteAt(t0.Add(time.Second * 3 / 2))
+	n, err = w.Write(buf[:10])
+	require.NoError(t, err)
+	require.Equal(t, 10, n)
+	require.GreaterOrEqual(t, time.Since(t1), time.Second)
+
+	// bitrate below slow threshold(2000bytes/3sec), should drop by timeout
+	mockDC.SetNextWriteCompleteAt(t0.Add(3 * time.Second))
+	n, err = w.Write(buf[:1000])
+	require.ErrorIs(t, err, ErrDataDroppedBySlowReader, err)
+	require.Equal(t, 0, n)
+}
+
+func TestDataChannelWriter_NoSlowThreshold(t *testing.T) {
+	mockDC := newMockDataChannelWriter()
+	w := NewDataChannelWriterReliable(mockDC, mockDC, 0)
+	buf := make([]byte, 2000)
+	n, err := w.Write(buf)
+	require.NoError(t, err)
+	require.Equal(t, 2000, n)
+	mockDC.SetNextWriteCompleteAt(time.Now().Add(singleWriteTimeout / 2))
+	n, err = w.Write(buf[:10])
+	require.NoError(t, err)
+	require.Equal(t, 10, n)
+
+	// slow threshold is 0, should not block & retry
+	mockDC.SetNextWriteCompleteAt(time.Now().Add(singleWriteTimeout * 2))
+	n, err = w.Write(buf[:1000])
+	require.ErrorIs(t, err, context.DeadlineExceeded, err)
+	require.Equal(t, 0, n)
+}
+
+func TestDataChannelWriter_Unreliable(t *testing.T) {
+	mockDC := newMockLossyDataChannelWriter(8192)
+	w := NewDataChannelWriterUnreliable(mockDC, mockDC, 100*time.Millisecond, 2000)
+	for range 10 {
+		buf := make([]byte, 128)
+		_, err := w.Write(buf)
+		require.NoError(t, err)
+		time.Sleep(100 * time.Millisecond)
+	}
+	buf := make([]byte, 4096)
+	_, err := w.Write(buf)
+	require.NoError(t, err)
+	// should drop due to high buffered amount
+	_, err = w.Write(buf)
+	require.ErrorIs(t, err, ErrDataDroppedByHighBufferedAmount)
+}
+
+// mockDataChannelWriter
+type mockDataChannelWriter struct {
+	datachannel.ReadWriteCloserDeadliner
+	nextWriteCompleteAt time.Time
+	deadline            *deadline.Deadline
+	buffered            atomic.Int64
+}
+
+func newMockDataChannelWriter() *mockDataChannelWriter {
+	return &mockDataChannelWriter{
+		deadline: deadline.New(),
+	}
+}
+
+func (m *mockDataChannelWriter) BufferedAmount() uint64 {
+	return uint64(m.buffered.Load())
+}
+
+func (m *mockDataChannelWriter) Write(b []byte) (int, error) {
+	// buffered while the write is in flight, cleared once it completes
+	m.buffered.Store(int64(len(b)))
+	wait := time.Until(m.nextWriteCompleteAt)
+	if wait <= 0 {
+		m.buffered.Store(0)
+		return len(b), nil
+	}
+	select {
+	case <-m.deadline.Done():
+		return 0, m.deadline.Err()
+	case <-time.After(wait):
+		m.buffered.Store(0)
+		return len(b), nil
+	}
+}
+
+func (m *mockDataChannelWriter) SetWriteDeadline(t time.Time) error {
+	m.deadline.Set(t)
+	return nil
+}
+
+func (m *mockDataChannelWriter) SetNextWriteCompleteAt(t time.Time) {
+	m.nextWriteCompleteAt = t
+}
+
+// mockLossyDataChannelWriter
+type mockLossyDataChannelWriter struct {
+	datachannel.ReadWriteCloserDeadliner
+	bufferedAmount atomic.Int64
+	targetBitrate  int
+	lastWriteAt    time.Time
+}
+
+func newMockLossyDataChannelWriter(targetBitrate int) *mockLossyDataChannelWriter {
+	return &mockLossyDataChannelWriter{
+		targetBitrate: targetBitrate,
+		lastWriteAt:   time.Now(),
+	}
+}
+
+func (m *mockLossyDataChannelWriter) BufferedAmount() uint64 {
+	return uint64(m.bufferedAmount.Load())
+}
+
+func (m *mockLossyDataChannelWriter) Write(b []byte) (int, error) {
+	m.bufferedAmount.Add(int64(len(b)))
+	if time.Now().Before(m.lastWriteAt) {
+		return len(b), nil
+	}
+
+	// drain buffer based on target bitrate
+	canWriteBytes := time.Since(m.lastWriteAt) * time.Duration(m.targetBitrate) / time.Second / 8
+	if m.bufferedAmount.Load() <= int64(canWriteBytes) {
+		m.lastWriteAt = m.lastWriteAt.Add(time.Duration(int64(time.Second) * int64(m.BufferedAmount()) / (int64(m.targetBitrate) / 8)))
+		m.bufferedAmount.Store(0)
+	} else {
+		m.lastWriteAt = time.Now()
+		m.bufferedAmount.Add(-int64(canWriteBytes))
+	}
+	return len(b), nil
+}
+
+func (m *mockLossyDataChannelWriter) SetWriteDeadline(t time.Time) error {
+	return nil
+}

pkg/sfu/downtrack_downstream_integration_test.go

@@ -0,0 +1,921 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sfu_test
+
+// This file contains real-pion integration tests for the DownTrack downstream
+// packet push path.
+//
+// Rather than mocking the pacer/write stream, these tests wire a real DownTrack
+// (optionally with a real StreamAllocator + Pacer + BWE) to a real pion
+// PeerConnection and push packets over an in-memory pion vnet to a second real
+// pion PeerConnection, which reads them back off the wire.
+//
+// The suite drives the four ways a DownTrack emits packets downstream and
+// verifies the on-wire RTP header and payload produced by each: SSRC, payload
+// type, sequence number, timestamp, marker, payload bytes, and the padding
+// bit / padding size.
+//
+//	WriteRTP          - forwarding translated media
+//	retransmitPacket  - retransmitting (RTX) media
+//	WritePaddingRTP   - padding-only packets
+//	WriteProbePackets - RTX probe packets
+//
+// pion's rtp.Packet.Unmarshal rejects a packet whose padding bit is set but whose
+// trailing padding-size byte is invalid/zero (errInvalidRTPPadding), so a
+// malformed packet is dropped by the receiver and never arrives. Arrival on the
+// far side is therefore itself evidence that the packet was serialized correctly.
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/pion/rtcp"
+	"github.com/pion/rtp"
+	"github.com/pion/webrtc/v4"
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/mediatransportutil/pkg/codec"
+	"github.com/livekit/protocol/codecs/mime"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+
+	. "github.com/livekit/livekit-server/pkg/sfu"
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/sfu/bwe"
+	"github.com/livekit/livekit-server/pkg/sfu/bwe/remotebwe"
+	"github.com/livekit/livekit-server/pkg/sfu/bwe/sendsidebwe"
+	"github.com/livekit/livekit-server/pkg/sfu/ccutils"
+	"github.com/livekit/livekit-server/pkg/sfu/pacer"
+	"github.com/livekit/livekit-server/pkg/sfu/packettrailer"
+	"github.com/livekit/livekit-server/pkg/sfu/sfufakes"
+	"github.com/livekit/livekit-server/pkg/sfu/streamallocator"
+	"github.com/livekit/livekit-server/pkg/sfu/testutils"
+	"github.com/livekit/livekit-server/pkg/testutils/vnettest"
+)
+
+// -----------------------------------------------------------------------------
+// packet capture on the far side
+// -----------------------------------------------------------------------------
+
+type packetCapture struct {
+	lock    sync.Mutex
+	packets []*rtp.Packet
+}
+
+func (c *packetCapture) add(p *rtp.Packet) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	cp := *p
+	c.packets = append(c.packets, &cp)
+}
+
+func (c *packetCapture) count() int {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return len(c.packets)
+}
+
+func (c *packetCapture) all() []*rtp.Packet {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return append([]*rtp.Packet(nil), c.packets...)
+}
+
+// captureTrack wires an OnTrack handler on the receiving PC that reads RTP packets
+// into the capture until the track ends.
+func captureTrack(pc *webrtc.PeerConnection) *packetCapture {
+	cap := &packetCapture{}
+	pc.OnTrack(func(track *webrtc.TrackRemote, _ *webrtc.RTPReceiver) {
+		go func() {
+			for {
+				pkt, _, err := track.ReadRTP()
+				if err != nil {
+					return
+				}
+				cap.add(pkt)
+			}
+		}()
+	})
+	return cap
+}
+
+// -----------------------------------------------------------------------------
+// fakes (from pkg/sfu/sfufakes) + a null BWE
+// -----------------------------------------------------------------------------
+
+// newFakeTrackReceiver returns a counterfeiter FakeTrackReceiver configured just
+// enough for a DownTrack to bind and forward: it reports the given codec and fires
+// its readiness callback immediately so Bind completes.
+func newFakeTrackReceiver(codecParams webrtc.RTPCodecParameters) *sfufakes.FakeTrackReceiver {
+	r := &sfufakes.FakeTrackReceiver{}
+	r.TrackIDReturns("fake-track")
+	r.StreamIDReturns("fake-stream")
+	r.CodecReturns(codecParams)
+	r.MimeReturns(mime.NormalizeMimeType(codecParams.MimeType))
+	r.VideoLayerModeReturns(livekit.VideoLayer_MODE_UNUSED)
+	r.HeaderExtensionsReturns(nil)
+	r.IsClosedReturns(false)
+	r.TrackInfoReturns(&livekit.TrackInfo{})
+	r.CodecStateReturns(ReceiverCodecStateNormal)
+	r.GetPrimaryReceiverForRedReturns(r)
+	r.GetRedReceiverReturns(r)
+	r.ReadRTPReturns(0, io.EOF)
+	// AddOnReady must invoke its callback immediately so Bind reaches the bound state.
+	r.AddOnReadyCalls(func(f func()) { f() })
+	return r
+}
+
+// nullBWE is a complete no-op bwe.BWE for tests that do not exercise estimation.
+// bwe.NullBWE is meant to be embedded and lacks a couple of interface methods.
+type nullBWE struct {
+	*bwe.NullBWE
+}
+
+func newNullBWE() *nullBWE        { return &nullBWE{NullBWE: &bwe.NullBWE{}} }
+func (nullBWE) Type() bwe.BWEType { return bwe.BWETypeNone }
+
+// -----------------------------------------------------------------------------
+// DownTrack harness
+// -----------------------------------------------------------------------------
+
+var (
+	opusCodecParams = webrtc.RTPCodecParameters{
+		RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2},
+		PayloadType:        111,
+	}
+	vp8CodecParams = webrtc.RTPCodecParameters{
+		RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8, ClockRate: 90000, RTCPFeedback: vnettest.VideoRTCPFeedback()},
+		PayloadType:        96,
+	}
+)
+
+type downTrackHarness struct {
+	dt       *DownTrack
+	receiver *sfufakes.FakeTrackReceiver
+	pacer    pacer.Pacer
+	capture  *packetCapture
+	sender   *webrtc.PeerConnection
+	sub      *webrtc.PeerConnection
+}
+
+// newBoundDownTrack builds a real DownTrack, attaches it to a real sender PC,
+// negotiates with a real subscriber PC over vnet, and makes it writable.
+func newBoundDownTrack(t *testing.T, h *vnettest.Hosts, factory *buffer.Factory, codecParams webrtc.RTPCodecParameters, p pacer.Pacer, meCfg vnettest.MediaEngineConfig) *downTrackHarness {
+	t.Helper()
+
+	sender := vnettest.NewPeerConnection(t, vnettest.PCConfig{Net: h.OfferNet, MediaEngine: meCfg, BufferFactory: factory.GetOrNew})
+	sub := vnettest.NewPeerConnection(t, vnettest.PCConfig{Net: h.AnswerNet, MediaEngine: meCfg, BufferFactory: factory.GetOrNew})
+	capture := captureTrack(sub)
+
+	rcv := newFakeTrackReceiver(codecParams)
+	dt, err := NewDownTrack(DownTrackParams{
+		Codecs:        []webrtc.RTPCodecParameters{codecParams},
+		Receiver:      rcv,
+		BufferFactory: factory,
+		Pacer:         p,
+		Logger:        logger.GetLogger(),
+		Listener:      &sfufakes.FakeDownTrackListener{},
+		StreamID:      "fake-stream",
+		SubID:         "fake-sub",
+		MaxTrack:      500,
+	})
+	require.NoError(t, err)
+
+	tr, err := sender.AddTransceiverFromTrack(dt, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendonly})
+	require.NoError(t, err)
+	dt.SetTransceiver(tr)
+
+	vnettest.SignalPair(t, sender, sub)
+	dt.SetConnected()
+
+	require.Eventually(t, func() bool {
+		return dt.IsWritableForTest()
+	}, 10*time.Second, 20*time.Millisecond, "downtrack did not become writable")
+
+	return &downTrackHarness{
+		dt:       dt,
+		receiver: rcv,
+		pacer:    p,
+		capture:  capture,
+		sender:   sender,
+		sub:      sub,
+	}
+}
+
+// distinctivePayload builds a per-packet payload with recognizable bytes so payload
+// fidelity can be asserted end-to-end.
+func distinctivePayload(seed byte, n int) []byte {
+	b := make([]byte, n)
+	for j := range b {
+		b[j] = seed + byte(j)
+	}
+	return b
+}
+
+// -----------------------------------------------------------------------------
+// spike: confirm vnet + real pion media forwarding works with this pion version
+// -----------------------------------------------------------------------------
+
+func TestPionVNetForwardingSpike(t *testing.T) {
+	h := vnettest.NewHosts(t)
+
+	sender := vnettest.NewPeerConnection(t, vnettest.PCConfig{Net: h.OfferNet, MediaEngine: vnettest.MediaEngineConfig{Video: true}})
+	receiver := vnettest.NewPeerConnection(t, vnettest.PCConfig{Net: h.AnswerNet, MediaEngine: vnettest.MediaEngineConfig{Video: true}})
+
+	track, err := webrtc.NewTrackLocalStaticRTP(
+		webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8, ClockRate: 90000},
+		"video", "pion-spike",
+	)
+	require.NoError(t, err)
+	_, err = sender.AddTrack(track)
+	require.NoError(t, err)
+
+	cap := captureTrack(receiver)
+
+	vnettest.SignalPair(t, sender, receiver)
+
+	// pump a few RTP packets from the sender track
+	go func() {
+		sn := uint16(1000)
+		ts := uint32(0)
+		for i := 0; i < 200; i++ {
+			_ = track.WriteRTP(&rtp.Packet{
+				Header: rtp.Header{
+					Version:        2,
+					PayloadType:    96,
+					SequenceNumber: sn,
+					Timestamp:      ts,
+					SSRC:           0x1234,
+				},
+				Payload: []byte{0x10, 0x00, 0x00, 0x01, 0x02, 0x03},
+			})
+			sn++
+			ts += 3000
+			time.Sleep(2 * time.Millisecond)
+		}
+	}()
+
+	require.Eventually(t, func() bool {
+		return cap.count() >= 5
+	}, 10*time.Second, 20*time.Millisecond, "expected to receive forwarded RTP over vnet")
+}
+
+// -----------------------------------------------------------------------------
+// WriteRTP: media forwarding
+// -----------------------------------------------------------------------------
+
+// TestDownTrackForwardsMedia exercises the WriteRTP media forwarding path and
+// verifies the full forwarded header/payload on the far side: SSRC, payload type,
+// contiguous sequence numbers, timestamps, payload bytes, and that the padding bit
+// is cleared regardless of the source packet's padding bit.
+func TestDownTrackForwardsMedia(t *testing.T) {
+	h := vnettest.NewHosts(t)
+	factory := buffer.NewFactoryOfBufferFactory(500, 500).CreateBufferFactory()
+	p := pacer.NewPassThrough(logger.GetLogger(), newNullBWE())
+	t.Cleanup(p.Stop)
+
+	dh := newBoundDownTrack(t, h, factory, opusCodecParams, p, vnettest.MediaEngineConfig{})
+
+	const numPackets = 20
+	sn := uint16(23333)
+	ts := uint32(0xabcdef)
+	wantPayload := map[uint16][]byte{}
+	wantTS := map[uint16]uint32{}
+	for i := 0; i < numPackets; i++ {
+		ep, err := testutils.GetTestExtPacket(&testutils.TestExtPacketParams{
+			SequenceNumber: sn,
+			Timestamp:      ts,
+			SSRC:           0x12345678,
+			PayloadType:    111,
+			PayloadSize:    40,
+		})
+		require.NoError(t, err)
+		// distinctive payload so payload fidelity can be checked on the far side
+		payload := distinctivePayload(byte(i+1), 40)
+		ep.Packet.Payload = payload
+		// every other source packet carries the padding bit; the forwarded packet
+		// must not inherit it.
+		if i%2 == 0 {
+			ep.Packet.Header.Padding = true
+		}
+		wantPayload[sn] = payload
+		wantTS[sn] = ts
+
+		require.Equal(t, int32(1), dh.dt.WriteRTP(ep, 0), "packet %d should be forwarded", i)
+		sn++
+		ts += 960
+	}
+
+	require.Eventually(t, func() bool {
+		return dh.capture.count() >= numPackets
+	}, 10*time.Second, 20*time.Millisecond, "expected all forwarded media packets to arrive")
+
+	pkts := dh.capture.all()
+	seen := map[uint16]bool{}
+	for _, pkt := range pkts {
+		require.EqualValues(t, dh.dt.SSRC(), pkt.SSRC, "forwarded packet on wrong SSRC")
+		require.EqualValues(t, 111, pkt.PayloadType, "forwarded packet has wrong payload type")
+		require.False(t, pkt.Header.Padding, "forwarded media packet must not carry padding bit (sn=%d)", pkt.SequenceNumber)
+
+		wp, ok := wantPayload[pkt.SequenceNumber]
+		require.True(t, ok, "received unexpected sequence number %d", pkt.SequenceNumber)
+		require.Equal(t, wp, pkt.Payload, "forwarded payload mismatch (sn=%d)", pkt.SequenceNumber)
+		require.Equal(t, wantTS[pkt.SequenceNumber], pkt.Timestamp, "forwarded timestamp mismatch (sn=%d)", pkt.SequenceNumber)
+		seen[pkt.SequenceNumber] = true
+	}
+
+	// every source packet must have been forwarded, and the forwarded sequence
+	// numbers must be contiguous.
+	for want := uint16(23333); want < 23333+numPackets; want++ {
+		require.True(t, seen[want], "sequence number %d was not forwarded", want)
+	}
+}
+
+// -----------------------------------------------------------------------------
+// retransmitPacket: RTX / retransmission
+// -----------------------------------------------------------------------------
+
+// TestDownTrackRetransmitsPacketsAsIs exercises the retransmit path when no RTX
+// stream is negotiated, so the packet is re-sent as-is on the primary SSRC/payload
+// type. It verifies the retransmitted header/payload on the far side: SSRC, payload
+// type, sequence number, timestamp, payload bytes, and that the padding bit is
+// cleared regardless of the source packet's padding bit.
+func TestDownTrackRetransmitsPacketsAsIs(t *testing.T) {
+	h := vnettest.NewHosts(t)
+	factory := buffer.NewFactoryOfBufferFactory(500, 500).CreateBufferFactory()
+	p := pacer.NewPassThrough(logger.GetLogger(), newNullBWE())
+	t.Cleanup(p.Stop)
+
+	dh := newBoundDownTrack(t, h, factory, opusCodecParams, p, vnettest.MediaEngineConfig{})
+
+	const numPackets = 10
+	targetSN := uint16(40000)
+	wantPayload := map[uint16][]byte{}
+	wantTS := map[uint16]uint32{}
+	for i := 0; i < numPackets; i++ {
+		// build a source packet that carries the padding bit (valid padding trailer)
+		// and a distinctive payload.
+		payload := distinctivePayload(byte(i+100), 30)
+		src := &rtp.Packet{
+			Header: rtp.Header{
+				Version:        2,
+				Padding:        true,
+				PaddingSize:    20,
+				PayloadType:    111,
+				SequenceNumber: uint16(1000 + i),
+				Timestamp:      uint32(90000 + i*960),
+				SSRC:           0x22222222,
+			},
+			Payload: payload,
+		}
+		raw, err := src.Marshal()
+		require.NoError(t, err)
+
+		ts := uint32(500000 + i*960)
+		wantPayload[targetSN] = payload
+		wantTS[targetSN] = ts
+
+		_, err = dh.dt.RetransmitForTest(
+			uint64(src.SequenceNumber),
+			targetSN,
+			ts,
+			uint64(ts),
+			0,
+			raw,
+			0,
+		)
+		require.NoError(t, err)
+		targetSN++
+	}
+
+	require.Eventually(t, func() bool {
+		return dh.capture.count() >= numPackets
+	}, 10*time.Second, 20*time.Millisecond, "expected all retransmitted packets to arrive")
+
+	seen := map[uint16]bool{}
+	for _, pkt := range dh.capture.all() {
+		require.EqualValues(t, dh.dt.SSRC(), pkt.SSRC, "retransmitted packet on wrong SSRC")
+		require.EqualValues(t, 111, pkt.PayloadType, "retransmitted packet has wrong payload type")
+		require.False(t, pkt.Header.Padding, "retransmitted packet must not carry padding bit (sn=%d)", pkt.SequenceNumber)
+
+		wp, ok := wantPayload[pkt.SequenceNumber]
+		require.True(t, ok, "received unexpected sequence number %d", pkt.SequenceNumber)
+		require.Equal(t, wp, pkt.Payload, "retransmitted payload mismatch (sn=%d)", pkt.SequenceNumber)
+		require.Equal(t, wantTS[pkt.SequenceNumber], pkt.Timestamp, "retransmitted timestamp mismatch (sn=%d)", pkt.SequenceNumber)
+		seen[pkt.SequenceNumber] = true
+	}
+
+	for want := uint16(40000); want < 40000+numPackets; want++ {
+		require.True(t, seen[want], "retransmitted sequence number %d did not arrive", want)
+	}
+}
+
+// TestDownTrackRetransmitsPacketsViaRTX exercises the retransmit path when an RTX
+// stream is negotiated. Each retransmitted packet must be emitted on the RTX SSRC
+// with the RTX payload type, carry the original media sequence number as the OSN
+// prefix (the first two payload bytes), draw its sequence number from the DownTrack's
+// RTX sequence space, and clear the padding bit.
+//
+// RTX packets are captured at the pacer as the DownTrack builds them: pion's
+// receiver-side RTX handling de-RTXes / consumes them, so the RTX-specific header
+// fields are not observable on the far side. The packets are still handed to the real
+// pacer (and written over pion).
+func TestDownTrackRetransmitsPacketsViaRTX(t *testing.T) {
+	h := vnettest.NewHosts(t)
+	factory := buffer.NewFactoryOfBufferFactory(500, 500).CreateBufferFactory()
+	cp := &capturingPacer{inner: pacer.NewPassThrough(logger.GetLogger(), newNullBWE())}
+	t.Cleanup(cp.Stop)
+
+	// VP8 registers an RTX codec, so the bound DownTrack has an RTX SSRC / payload type.
+	dh := newBoundDownTrack(t, h, factory, vp8CodecParams, cp, vnettest.MediaEngineConfig{Video: true})
+	require.NotZero(t, dh.dt.SSRCRTX(), "RTX ssrc should be negotiated")
+	require.NotZero(t, dh.dt.PayloadTypeRTXForTest(), "RTX payload type should be negotiated")
+
+	type want struct {
+		payload []byte
+		ts      uint32
+	}
+	const numPackets = 10
+	osnBase := uint16(50000)
+	wants := map[uint16]want{} // keyed by OSN (the original media sequence number)
+	targetSN := osnBase
+	for i := 0; i < numPackets; i++ {
+		// build a source packet that carries the padding bit (valid padding trailer)
+		// and a distinctive payload.
+		payload := distinctivePayload(byte(i+100), 30)
+		src := &rtp.Packet{
+			Header: rtp.Header{
+				Version:        2,
+				Padding:        true,
+				PaddingSize:    20,
+				PayloadType:    96,
+				SequenceNumber: uint16(2000 + i),
+				Timestamp:      uint32(180000 + i*3000),
+				SSRC:           0x33333333,
+			},
+			Payload: payload,
+		}
+		raw, err := src.Marshal()
+		require.NoError(t, err)
+
+		ts := uint32(700000 + i*3000)
+		wants[targetSN] = want{payload: payload, ts: ts}
+
+		_, err = dh.dt.RetransmitForTest(uint64(src.SequenceNumber), targetSN, ts, uint64(ts), 0, raw, 0)
+		require.NoError(t, err)
+		targetSN++
+	}
+
+	require.Eventually(t, func() bool {
+		return len(cp.rtxPackets()) >= numPackets
+	}, 5*time.Second, 20*time.Millisecond, "expected RTX packets to be emitted")
+
+	seen := map[uint16]bool{}
+	rtxSeqs := make([]uint16, 0, numPackets)
+	for _, pr := range cp.rtxPackets() {
+		require.EqualValues(t, dh.dt.SSRCRTX(), pr.hdr.SSRC, "retransmitted packet must be on the RTX SSRC")
+		require.EqualValues(t, dh.dt.PayloadTypeRTXForTest(), pr.hdr.PayloadType, "retransmitted packet must use the RTX payload type")
+		require.False(t, pr.hdr.Padding, "retransmitted packet must not carry padding bit")
+		require.GreaterOrEqual(t, len(pr.payload), 2, "RTX payload must contain the OSN prefix")
+
+		osn := binary.BigEndian.Uint16(pr.payload[0:2])
+		w, ok := wants[osn]
+		require.True(t, ok, "unexpected OSN %d", osn)
+		require.Equal(t, w.payload, pr.payload[2:], "RTX media payload mismatch (osn=%d)", osn)
+		require.Equal(t, w.ts, pr.hdr.Timestamp, "RTX timestamp mismatch (osn=%d)", osn)
+		seen[osn] = true
+		rtxSeqs = append(rtxSeqs, pr.hdr.SequenceNumber)
+	}
+
+	for osn := osnBase; osn < osnBase+numPackets; osn++ {
+		require.True(t, seen[osn], "OSN %d was not retransmitted via RTX", osn)
+	}
+
+	// RTX-stream sequence numbers come from the DownTrack's own RTX sequence space
+	// and must advance by one per packet.
+	require.Len(t, rtxSeqs, numPackets)
+	for i := 1; i < len(rtxSeqs); i++ {
+		require.EqualValues(t, rtxSeqs[i-1]+1, rtxSeqs[i], "RTX sequence numbers must be contiguous")
+	}
+}
+
+// TestDownTrackReplaysPacketTrailerStripOnRetransmit verifies that a packet whose
+// packet trailer was stripped when it was forwarded is retransmitted with the same
+// bytes removed, i. e. the retransmitted media payload is byte identical to the
+// original transmission.
+func TestDownTrackReplaysPacketTrailerStripOnRetransmit(t *testing.T) {
+	h := vnettest.NewHosts(t)
+	factory := buffer.NewFactoryOfBufferFactory(500, 500).CreateBufferFactory()
+	cp := &capturingPacer{inner: pacer.NewPassThrough(logger.GetLogger(), newNullBWE())}
+	t.Cleanup(cp.Stop)
+
+	dh := newBoundDownTrack(t, h, factory, vp8CodecParams, cp, vnettest.MediaEngineConfig{Video: true})
+	require.NotZero(t, dh.dt.SSRCRTX(), "RTX ssrc should be negotiated")
+
+	video := distinctivePayload(11, 40)
+	trailer := lktsTrailer()
+	src := &rtp.Packet{
+		Header: rtp.Header{
+			Version:        2,
+			PayloadType:    96,
+			SequenceNumber: 3000,
+			Timestamp:      270000,
+			SSRC:           0x44444444,
+		},
+		Payload: append(append([]byte{}, video...), trailer...),
+	}
+	raw, err := src.Marshal()
+	require.NoError(t, err)
+
+	osn := uint16(60000)
+	_, err = dh.dt.RetransmitForTest(
+		uint64(src.SequenceNumber),
+		osn,
+		src.Timestamp,
+		uint64(src.Timestamp),
+		0,
+		raw,
+		uint8(len(trailer)),
+	)
+	require.NoError(t, err)
+
+	require.Eventually(t, func() bool {
+		return len(cp.rtxPackets()) >= 1
+	}, 5*time.Second, 20*time.Millisecond, "expected RTX packet to be emitted")
+
+	pr := cp.rtxPackets()[0]
+	require.EqualValues(t, osn, binary.BigEndian.Uint16(pr.payload[0:2]))
+	require.Equal(t, video, pr.payload[2:], "retransmitted payload must not carry the packet trailer")
+}
+
+// lktsTrailer builds a 15-byte LKTS packet trailer carrying a user timestamp TLV.
+func lktsTrailer() []byte {
+	trailer := []byte{0x01 ^ 0xFF, 8 ^ 0xFF}
+	for i := 0; i < 8; i++ {
+		trailer = append(trailer, byte(i)^0xFF)
+	}
+	trailer = append(trailer, 15^0xFF)
+	return append(trailer, packettrailer.Magic[:]...)
+}
+
+// -----------------------------------------------------------------------------
+// WritePaddingRTP: padding-only packets
+// -----------------------------------------------------------------------------
+
+// TestDownTrackSendsPaddingOnlyPackets exercises the WritePaddingRTP padding-only
+// path. It first forwards a video keyframe (verifying the media header/payload),
+// then emits padding-only packets and verifies each one on the far side: it is on
+// the primary SSRC/payload type, carries the padding bit, and declares a full
+// padding block.
+//
+// A padding-only packet is entirely padding; pion/srtp serializes the padding block
+// from header.PaddingSize (via rtp.MarshalPacketTo), and the pion receiver strips it
+// and reports both the padding bit and the padding size, so the received payload
+// length equals the declared padding size (RTPPaddingMaxPayloadSize).
+func TestDownTrackSendsPaddingOnlyPackets(t *testing.T) {
+	h := vnettest.NewHosts(t)
+	factory := buffer.NewFactoryOfBufferFactory(500, 500).CreateBufferFactory()
+	p := pacer.NewPassThrough(logger.GetLogger(), newNullBWE())
+	t.Cleanup(p.Stop)
+
+	dh := newBoundDownTrack(t, h, factory, vp8CodecParams, p, vnettest.MediaEngineConfig{Video: true})
+
+	// force a valid target/current layer so the video forwarder will forward (test seam
+	// also used by forwarder_test.go's disable()).
+	dh.dt.ForceForwardLayerForTest(buffer.VideoLayer{Spatial: 0, Temporal: 0})
+
+	// forward a keyframe so rtpStats becomes active and the stream is running
+	vp8 := &codec.VP8{
+		FirstByte:  0x10,
+		S:          true,
+		PictureID:  1,
+		IsKeyFrame: true,
+	}
+	keyframePayload := distinctivePayload(0x40, 50)
+	ep, err := testutils.GetTestExtPacketVP8(&testutils.TestExtPacketParams{
+		SequenceNumber: 5000,
+		Timestamp:      180000,
+		SSRC:           0x33333333,
+		PayloadType:    96,
+		PayloadSize:    50,
+		IsKeyFrame:     true,
+		Marker:         true, // end of frame, so padding can be emitted on the frame boundary
+	}, vp8)
+	require.NoError(t, err)
+	ep.Packet.Payload = keyframePayload
+	require.Equal(t, int32(1), dh.dt.WriteRTP(ep, 0), "keyframe should forward")
+
+	require.True(t, dh.dt.RTPStatsActiveForTest(), "rtpStats should be active after forwarding")
+
+	// wait for the media keyframe to arrive, then take the baseline count.
+	require.Eventually(t, func() bool {
+		return dh.capture.count() >= 1
+	}, 10*time.Second, 20*time.Millisecond, "expected media keyframe to arrive")
+	baseline := dh.capture.count()
+
+	// emit padding-only packets. paddingOnMute=true bypasses the RR/active gating so
+	// this stays deterministic; the header construction (Padding + PaddingSize) is the
+	// same code path either way.
+	const numPadding = 3
+	sent := 0
+	require.Eventually(t, func() bool {
+		if dh.dt.WritePaddingRTP(500, true, false) > 0 {
+			sent++
+		}
+		return sent >= numPadding
+	}, 5*time.Second, 20*time.Millisecond, "expected WritePaddingRTP to emit padding packets")
+
+	// the padding-only packets must arrive on the far side.
+	require.Eventually(t, func() bool {
+		return dh.capture.count() >= baseline+numPadding
+	}, 10*time.Second, 20*time.Millisecond, "expected padding-only packets to arrive on the far side")
+
+	sawKeyframe := 0
+	sawPadding := 0
+	for _, pkt := range dh.capture.all() {
+		require.EqualValues(t, dh.dt.SSRC(), pkt.SSRC, "packet on wrong SSRC")
+		require.EqualValues(t, 96, pkt.PayloadType, "packet has wrong payload type")
+
+		if !pkt.Header.Padding {
+			// the forwarded media keyframe
+			require.True(t, pkt.Header.Marker, "forwarded keyframe should carry the marker bit")
+			// the forwarder prepends the VP8 payload descriptor, so the media bytes
+			// appear as a suffix of the forwarded payload.
+			require.True(t, bytes.HasSuffix(pkt.Payload, keyframePayload), "forwarded keyframe payload mismatch")
+			sawKeyframe++
+			continue
+		}
+
+		// a padding-only packet: it must declare a full padding block.
+		require.EqualValues(t, RTPPaddingMaxPayloadSize, pkt.Header.PaddingSize,
+			"padding-only packet must declare full padding size (sn=%d)", pkt.SequenceNumber)
+		require.Equal(t, RTPPaddingMaxPayloadSize, len(pkt.Payload),
+			"padding-only packet must serialize a full padding block (sn=%d)", pkt.SequenceNumber)
+		sawPadding++
+	}
+	require.Equal(t, 1, sawKeyframe, "expected exactly one forwarded media keyframe")
+	require.GreaterOrEqual(t, sawPadding, numPadding, "expected the padding-only packets to be observed")
+}
+
+// -----------------------------------------------------------------------------
+// WriteProbePackets: RTX probe packets (with StreamAllocator + Pacer + BWE)
+// -----------------------------------------------------------------------------
+
+// capturingProbe records what the DownTrack handed the pacer for a probe packet.
+type capturingProbe struct {
+	hdr        rtp.Header
+	payloadLen int
+	lastByte   byte
+}
+
+// capturedRTX records what the DownTrack handed the pacer for an RTX packet.
+type capturedRTX struct {
+	hdr     rtp.Header
+	payload []byte
+}
+
+// capturingPacer wraps a real pacer.Pacer, records probe and RTX packets as the
+// DownTrack constructs them (before the real pacer patches/marshals them), and
+// delegates everything to the wrapped pacer so packets are still sent over real pion.
+type capturingPacer struct {
+	inner pacer.Pacer
+
+	lock   sync.Mutex
+	probes []capturingProbe
+	rtx    []capturedRTX
+}
+
+func (c *capturingPacer) Enqueue(p *pacer.Packet) {
+	if p.Header != nil {
+		switch {
+		case p.IsProbe:
+			cp := capturingProbe{hdr: *p.Header, payloadLen: len(p.Payload)}
+			if len(p.Payload) > 0 {
+				cp.lastByte = p.Payload[len(p.Payload)-1]
+			}
+			c.lock.Lock()
+			c.probes = append(c.probes, cp)
+			c.lock.Unlock()
+		case p.IsRTX:
+			c.lock.Lock()
+			c.rtx = append(c.rtx, capturedRTX{hdr: *p.Header, payload: append([]byte(nil), p.Payload...)})
+			c.lock.Unlock()
+		}
+	}
+	c.inner.Enqueue(p)
+}
+
+func (c *capturingPacer) probePackets() []capturingProbe {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return append([]capturingProbe(nil), c.probes...)
+}
+
+func (c *capturingPacer) rtxPackets() []capturedRTX {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return append([]capturedRTX(nil), c.rtx...)
+}
+
+func (c *capturingPacer) Stop()                       { c.inner.Stop() }
+func (c *capturingPacer) SetInterval(i time.Duration) { c.inner.SetInterval(i) }
+func (c *capturingPacer) SetBitrate(b int)            { c.inner.SetBitrate(b) }
+func (c *capturingPacer) TimeSinceLastSentPacket() time.Duration {
+	return c.inner.TimeSinceLastSentPacket()
+}
+func (c *capturingPacer) SetPacerProbeObserverListener(l pacer.PacerProbeObserverListener) {
+	c.inner.SetPacerProbeObserverListener(l)
+}
+func (c *capturingPacer) StartProbeCluster(pci ccutils.ProbeClusterInfo) {
+	c.inner.StartProbeCluster(pci)
+}
+func (c *capturingPacer) EndProbeCluster(id ccutils.ProbeClusterId) ccutils.ProbeClusterInfo {
+	return c.inner.EndProbeCluster(id)
+}
+
+// feedReceiverReport hands the DownTrack a synthetic RTCP receiver report for its
+// primary SSRC via the same handleRTCP path production uses. This deterministically
+// sets LastReceiverReportTime() (a precondition for WriteProbePackets) without
+// depending on the timing of the far side's periodic RRs.
+func feedReceiverReport(t *testing.T, dt *DownTrack, lastSeqNo uint16) {
+	t.Helper()
+	rr := &rtcp.ReceiverReport{
+		SSRC: 0x9999,
+		Reports: []rtcp.ReceptionReport{{
+			SSRC:               dt.SSRC(),
+			LastSenderReport:   0,
+			LastSequenceNumber: uint32(lastSeqNo),
+		}},
+	}
+	raw, err := rr.Marshal()
+	require.NoError(t, err)
+	dt.HandleRTCPForTest(raw)
+	require.NotZero(t, dt.LastReceiverReportTimeForTest(), "synthetic RR should set LastReceiverReportTime")
+}
+
+// TestDownTrackSendsProbePackets exercises the WriteProbePackets RTX probe path
+// while wiring the DownTrack to a real StreamAllocator + Pacer + BWE. It runs
+// against BOTH bandwidth estimators (send-side / remote).
+//
+// The RTX probe packets are captured at the pacer as the DownTrack builds them (they
+// are padding-only on the RTX stream and are not surfaced as a media track by the
+// pion receiver, so far-side capture is not reliable for them). Each probe is
+// verified to be on the RTX SSRC/payload type, carry the padding bit, declare a full
+// padding size, and serialize a full padding block whose trailing byte is the
+// padding count.
+//
+// The media path is exercised end-to-end over real pion with the allocator, the
+// pacer and the specific BWE wired in; BWE estimate values themselves are
+// intentionally not asserted (they are timing dependent) — the test verifies the
+// pipeline stays healthy and the probe serialization is correct with each estimator.
+func TestDownTrackSendsProbePackets(t *testing.T) {
+	cases := []struct {
+		name    string
+		makeBWE func() bwe.BWE
+	}{
+		{
+			name: "sendside",
+			makeBWE: func() bwe.BWE {
+				return sendsidebwe.NewSendSideBWE(sendsidebwe.SendSideBWEParams{
+					Config: sendsidebwe.DefaultSendSideBWEConfig,
+					Logger: logger.GetLogger(),
+				})
+			},
+		},
+		{
+			name: "remote",
+			makeBWE: func() bwe.BWE {
+				return remotebwe.NewRemoteBWE(remotebwe.RemoteBWEParams{
+					Config: remotebwe.DefaultRemoteBWEConfig,
+					Logger: logger.GetLogger(),
+				})
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			h := vnettest.NewHosts(t)
+			factory := buffer.NewFactoryOfBufferFactory(500, 500).CreateBufferFactory()
+
+			b := tc.makeBWE()
+			cp := &capturingPacer{inner: pacer.NewPassThrough(logger.GetLogger(), b)}
+			t.Cleanup(cp.Stop)
+
+			// header extensions (abs-send-time for remote BWE, transport-cc for send-side)
+			// are needed for the DownTrack to pick up an ext id, which WriteProbePackets
+			// requires.
+			dh := newBoundDownTrack(t, h, factory, vp8CodecParams, cp, vnettest.MediaEngineConfig{
+				Video:            true,
+				HeaderExtensions: true,
+			})
+
+			require.NotZero(t, dh.dt.SSRCRTX(), "RTX ssrc should be negotiated")
+			require.NotZero(t, dh.dt.PayloadTypeRTXForTest(), "RTX payload type should be negotiated")
+
+			sa := streamallocator.NewStreamAllocator(streamallocator.StreamAllocatorParams{
+				Config:    streamallocator.DefaultStreamAllocatorConfig,
+				BWE:       b,
+				Pacer:     cp,
+				RTTGetter: func() (float64, bool) { return 0, false },
+				Logger:    logger.GetLogger(),
+			}, true, false)
+			// AddTrack wires the allocator as the DownTrack's stream-allocator listener
+			// (which is what supplies the BWE type used to resolve the abs-send-time /
+			// transport-cc ext id that WriteProbePackets requires). We intentionally do
+			// NOT Start() the allocator's event loop: its asynchronous re-allocation would
+			// race with (and override) the layer we force below, making the forwarding
+			// non-deterministic. The probe path is driven directly instead.
+			sa.AddTrack(dh.dt, streamallocator.AddTrackParams{
+				Source:      livekit.TrackSource_CAMERA,
+				PublisherID: "fake-pub",
+			})
+
+			// now that the StreamAllocator listener is set and negotiation is complete,
+			// (re)resolve the negotiated header extension ids.
+			dh.dt.SetRTPHeaderExtensionsForTest()
+			require.Eventually(t, func() bool {
+				abs, twcc := dh.dt.ExtIDsForTest()
+				return abs != 0 || twcc != 0
+			}, 3*time.Second, 20*time.Millisecond, "expected an abs-send-time or transport-cc ext id to be negotiated")
+
+			// force a valid target/current layer so the video forwarder will forward.
+			dh.dt.ForceForwardLayerForTest(buffer.VideoLayer{Spatial: 0, Temporal: 0})
+
+			// forward a keyframe end-to-end (exercises DownTrack + allocator + pacer + BWE)
+			// and verify it arrives with the expected header/payload.
+			vp8 := &codec.VP8{FirstByte: 0x10, S: true, PictureID: 1, IsKeyFrame: true}
+			keyframePayload := distinctivePayload(0x50, 60)
+			ep, err := testutils.GetTestExtPacketVP8(&testutils.TestExtPacketParams{
+				SequenceNumber: 6000,
+				Timestamp:      270000,
+				SSRC:           0x44444444,
+				PayloadType:    96,
+				PayloadSize:    60,
+				IsKeyFrame:     true,
+				Marker:         true,
+			}, vp8)
+			require.NoError(t, err)
+			ep.Packet.Payload = keyframePayload
+			require.Equal(t, int32(1), dh.dt.WriteRTP(ep, 0), "keyframe should forward")
+			require.True(t, dh.dt.RTPStatsActiveForTest())
+
+			require.Eventually(t, func() bool {
+				return dh.capture.count() >= 1
+			}, 10*time.Second, 20*time.Millisecond, "expected forwarded media to arrive over the full pipeline")
+
+			media := dh.capture.all()[0]
+			require.EqualValues(t, dh.dt.SSRC(), media.SSRC, "forwarded media on wrong SSRC")
+			require.EqualValues(t, 96, media.PayloadType, "forwarded media has wrong payload type")
+			require.False(t, media.Header.Padding, "forwarded media must not carry padding bit")
+			// the forwarder prepends the VP8 payload descriptor, so the media bytes
+			// appear as a suffix of the forwarded payload.
+			require.True(t, bytes.HasSuffix(media.Payload, keyframePayload), "forwarded media payload mismatch")
+
+			// satisfy the RR precondition deterministically, referencing the
+			// subscriber-facing sequence number of the forwarded keyframe.
+			feedReceiverReport(t, dh.dt, media.SequenceNumber)
+
+			// emit RTX probe packets.
+			const numProbe = 3
+			require.Eventually(t, func() bool {
+				dh.dt.WriteProbePackets(2000, true)
+				return len(cp.probePackets()) >= numProbe
+			}, 5*time.Second, 20*time.Millisecond, "expected WriteProbePackets to emit RTX probe packets")
+
+			// every probe packet must be on the RTX stream, carry the padding bit, declare
+			// the full padding size, and serialize a full padding block ending in the
+			// padding count byte.
+			for _, pr := range cp.probePackets() {
+				require.EqualValues(t, 2, pr.hdr.Version, "probe packet version")
+				require.EqualValues(t, dh.dt.SSRCRTX(), pr.hdr.SSRC, "probe packet must be on the RTX stream")
+				require.EqualValues(t, dh.dt.PayloadTypeRTXForTest(), pr.hdr.PayloadType, "probe packet must use the RTX payload type")
+				require.True(t, pr.hdr.Padding, "probe packet must carry padding bit")
+				require.EqualValues(t, RTPPaddingMaxPayloadSize, pr.hdr.PaddingSize, "probe packet must set header.PaddingSize")
+				require.Equal(t, RTPPaddingMaxPayloadSize, pr.payloadLen, "probe packet must carry a full padding block")
+				require.EqualValues(t, RTPPaddingMaxPayloadSize, pr.lastByte, "probe packet trailing byte must be the padding count")
+			}
+
+			// the BWE is wired and exercised (the pacer records every send into it);
+			// its state must be queryable without panicking. Exact estimate values are
+			// timing dependent and intentionally not asserted.
+			require.NotPanics(t, func() { _ = b.CongestionState() })
+		})
+	}
+}

pkg/sfu/errors.go

@@ -0,0 +1,15 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sfu

pkg/sfu/export_test.go

@@ -0,0 +1,73 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sfu
+
+import "github.com/livekit/livekit-server/pkg/sfu/buffer"
+
+// Test-only seams that expose DownTrack internals to the external sfu_test package.
+// This file is only compiled into the test binary (its name ends in _test.go), so it
+// does not affect the production API. The external test package is required because the
+// integration tests import the streamallocator package, which itself imports sfu (an
+// import cycle that an in-package test cannot resolve).
+
+func (d *DownTrack) IsWritableForTest() bool { return d.writable.Load() }
+
+func (d *DownTrack) PayloadTypeRTXForTest() uint32 { return d.payloadTypeRTX.Load() }
+
+func (d *DownTrack) RTPStatsActiveForTest() bool { return d.rtpStats.IsActive() }
+
+func (d *DownTrack) LastReceiverReportTimeForTest() int64 {
+	return d.rtpStats.LastReceiverReportTime()
+}
+
+func (d *DownTrack) ExtIDsForTest() (absSendTime int, transportWide int) {
+	return d.absSendTimeExtID, d.transportWideExtID
+}
+
+func (d *DownTrack) HandleRTCPForTest(pkt []byte) { d.handleRTCP(pkt) }
+
+func (d *DownTrack) SetRTPHeaderExtensionsForTest() { d.setRTPHeaderExtensions() }
+
+// ForceForwardLayerForTest forces the forwarder onto a valid current/target layer so a
+// (non-simulcast) video packet will be forwarded without driving full allocation.
+func (d *DownTrack) ForceForwardLayerForTest(layer buffer.VideoLayer) {
+	d.forwarder.ForceLayerForTest(layer)
+}
+
+// RetransmitForTest drives the (unexported) retransmitPacket path with an in-package
+// extPacketMeta built from exported values, so the external test can exercise the
+// retransmit path.
+func (d *DownTrack) RetransmitForTest(
+	sourceSeqNo uint64,
+	targetSeqNo uint16,
+	timestamp uint32,
+	extTimestamp uint64,
+	layer int8,
+	sourcePkt []byte,
+	trailerStripped uint8,
+) (int, error) {
+	epm := extPacketMeta{
+		packetMeta: packetMeta{
+			sourceSeqNo:     sourceSeqNo,
+			targetSeqNo:     targetSeqNo,
+			timestamp:       timestamp,
+			layer:           layer,
+			trailerStripped: trailerStripped,
+		},
+		extSequenceNumber: uint64(targetSeqNo),
+		extTimestamp:      extTimestamp,
+	}
+	return d.retransmitPacket(&epm, sourcePkt, false)
+}

pkg/sfu/forwardstats.go

@@ -0,0 +1,306 @@
+package sfu
+
+import (
+	"math"
+	"sync"
+	"time"
+
+	"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
+	"github.com/livekit/protocol/logger"
+	"go.uber.org/atomic"
+)
+
+const (
+	cHighForwardingLatency = 20 * time.Millisecond
+	cSkewFactor            = 10
+)
+
+const (
+	// A summary interval's worth of samples across all tracks must fit without
+	// dropping (ForwardStats is a singleton). Shard count spreads the per-packet
+	// atomic; shard capacity bounds memory (numShards*shardCap*16 bytes = 2MiB).
+	forwardSampleNumShards = 16
+	forwardSampleShardCap  = 8192
+	forwardSampleShardMask = forwardSampleShardCap - 1
+	forwardSampleShardSel  = forwardSampleNumShards - 1
+)
+
+// forwardSampleShard is a ring of transit samples with multiple producers and a
+// single consumer. A producer reserves a slot, stores the value, then publishes
+// the slot's epoch (reserved index + 1). The consumer reads a slot only once its
+// epoch marks the value committed for that index.
+type forwardSampleShard struct {
+	writeIdx atomic.Uint64 // advanced by producers to reserve a slot
+	readIdx  uint64        // consumer-only cursor
+	ring     [forwardSampleShardCap]atomic.Int64
+	seq      [forwardSampleShardCap]atomic.Uint64 // per-slot publish epoch
+}
+
+// forwardSampleBuffer holds per-packet transit samples produced on the packet
+// path and consumed by the background worker, which performs metric emission.
+type forwardSampleBuffer struct {
+	shards  [forwardSampleNumShards]forwardSampleShard
+	dropped atomic.Uint64
+}
+
+// push records a sample: reserve a slot, store the value, then publish the
+// slot's epoch. The shard is selected from arrival time bits.
+func (b *forwardSampleBuffer) push(arrival, transitNs int64) {
+	sh := &b.shards[(uint64(arrival)>>6)&forwardSampleShardSel]
+	i := sh.writeIdx.Add(1) - 1
+	slot := i & forwardSampleShardMask
+	sh.ring[slot].Store(transitNs)
+	sh.seq[slot].Store(i + 1)
+}
+
+// drain passes every committed sample to fn and advances the read cursor. Only
+// the background worker calls this.
+//
+// A slot holds index r's value once its epoch equals r+1. If the slot at the
+// cursor is still uncommitted (a producer reserved it but has not published),
+// draining stops and resumes from there on the next call, so no sample is read
+// stale or skipped. When producers get a shard's capacity ahead, or overwrite a
+// slot before it is read, the affected samples are counted as dropped.
+func (b *forwardSampleBuffer) drain(fn func(transitNs int64)) {
+	for si := range b.shards {
+		sh := &b.shards[si]
+		w := sh.writeIdx.Load()
+		r := sh.readIdx
+		if w-r > forwardSampleShardCap {
+			b.dropped.Add(w - r - forwardSampleShardCap)
+			r = w - forwardSampleShardCap
+		}
+		for r < w {
+			slot := r & forwardSampleShardMask
+			if sh.seq[slot].Load() < r+1 {
+				// reserved but not yet published; resume here next drain
+				break
+			}
+			v := sh.ring[slot].Load()
+			if sh.seq[slot].Load() != r+1 {
+				// overwritten by a newer sample during the read; original lost
+				b.dropped.Add(1)
+				r++
+				continue
+			}
+			fn(v)
+			r++
+		}
+		sh.readIdx = r
+	}
+}
+
+func (b *forwardSampleBuffer) takeDropped() uint64 {
+	return b.dropped.Swap(0)
+}
+
+// forwardSummary is a mergeable summary of forwarding transit over an interval.
+// The sum of squares is kept in microseconds so it does not overflow int64.
+type forwardSummary struct {
+	count   int64
+	sumUs   int64
+	sumSqUs int64
+	minNs   int64
+	maxNs   int64
+}
+
+func (s forwardSummary) addSample(transitNs int64) forwardSummary {
+	us := transitNs / 1000
+	if s.count == 0 {
+		return forwardSummary{count: 1, sumUs: us, sumSqUs: us * us, minNs: transitNs, maxNs: transitNs}
+	}
+
+	s.count++
+	s.sumUs += us
+	s.sumSqUs += us * us
+	if transitNs < s.minNs {
+		s.minNs = transitNs
+	}
+	if transitNs > s.maxNs {
+		s.maxNs = transitNs
+	}
+	return s
+}
+
+func (s forwardSummary) merge(o forwardSummary) forwardSummary {
+	if o.count == 0 {
+		return s
+	}
+	if s.count == 0 {
+		return o
+	}
+	return forwardSummary{
+		count:   s.count + o.count,
+		sumUs:   s.sumUs + o.sumUs,
+		sumSqUs: s.sumSqUs + o.sumSqUs,
+		minNs:   min(s.minNs, o.minNs),
+		maxNs:   max(s.maxNs, o.maxNs),
+	}
+}
+
+func (s forwardSummary) meanStdDev() (mean, stdDev time.Duration) {
+	if s.count == 0 {
+		return 0, 0
+	}
+
+	meanUs := float64(s.sumUs) / float64(s.count)
+	mean = time.Duration(meanUs * float64(time.Microsecond))
+	if s.count < 2 {
+		return mean, 0
+	}
+
+	// sample variance (divisor count-1)
+	m2 := float64(s.sumSqUs) - float64(s.sumUs)*meanUs
+	varUs2 := m2 / float64(s.count-1)
+	if varUs2 < 0 {
+		// floating point rounding can push a (near-zero) variance slightly negative
+		varUs2 = 0
+	}
+	stdDev = time.Duration(math.Sqrt(varUs2) * float64(time.Microsecond))
+	return mean, stdDev
+}
+
+type ForwardStats struct {
+	samples forwardSampleBuffer
+
+	// ring of per-summary-interval summaries covering the report window.
+	// written by the background worker (flush) and read both by the worker
+	// (report) and by external callers (GetStats), so it is guarded by lock.
+	lock     sync.Mutex
+	ring     []forwardSummary
+	ringHead int
+	ringLen  int
+
+	summaryInterval time.Duration
+	reportInterval  time.Duration
+
+	closeCh chan struct{}
+}
+
+func NewForwardStats(summaryInterval, reportInterval, reportWindow time.Duration) *ForwardStats {
+	ringCap := int((reportWindow + summaryInterval - 1) / summaryInterval)
+	if ringCap < 1 {
+		ringCap = 1
+	}
+
+	s := &ForwardStats{
+		ring:            make([]forwardSummary, ringCap),
+		summaryInterval: summaryInterval,
+		reportInterval:  reportInterval,
+		closeCh:         make(chan struct{}),
+	}
+
+	go s.run()
+	return s
+}
+
+// Update records a forwarded packet's transit latency. It buffers the sample
+// and returns the transit and whether it exceeds the high-latency threshold.
+// The sample is aggregated and emitted by the background worker.
+func (s *ForwardStats) Update(arrival, left int64) (int64, bool) {
+	transit := left - arrival
+	s.samples.push(arrival, transit)
+	return transit, time.Duration(transit) > cHighForwardingLatency
+}
+
+func (s *ForwardStats) Stop() {
+	close(s.closeCh)
+}
+
+func (s *ForwardStats) run() {
+	summaryTicker := time.NewTicker(s.summaryInterval)
+	defer summaryTicker.Stop()
+	reportTicker := time.NewTicker(s.reportInterval)
+	defer reportTicker.Stop()
+
+	for {
+		select {
+		case <-s.closeCh:
+			return
+
+		case <-summaryTicker.C:
+			s.flush()
+
+		case <-reportTicker.C:
+			// the summary ticker keeps the window ring current to within one
+			// summary interval; report over it without advancing the ring.
+			s.report()
+		}
+	}
+}
+
+// flush drains the buffered samples, observes each into the Prometheus
+// histogram, and folds the interval summary into the window ring used for the
+// latency/jitter gauges.
+func (s *ForwardStats) flush() {
+	var summ forwardSummary
+	s.samples.drain(func(transitNs int64) {
+		prometheus.RecordForwardLatencySample(transitNs)
+		summ = summ.addSample(transitNs)
+	})
+
+	s.lock.Lock()
+	s.ring[s.ringHead] = summ
+	s.ringHead = (s.ringHead + 1) % len(s.ring)
+	if s.ringLen < len(s.ring) {
+		s.ringLen++
+	}
+	s.lock.Unlock()
+}
+
+// summarize merges the ring summaries covering the most recent window. A
+// window <= 0 (or >= the report window) covers the entire ring.
+func (s *ForwardStats) summarize(window time.Duration) forwardSummary {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	n := s.ringLen
+	if window > 0 && s.summaryInterval > 0 {
+		want := int((window + s.summaryInterval - 1) / s.summaryInterval)
+		if want < 1 {
+			want = 1
+		}
+		if want < n {
+			n = want
+		}
+	}
+
+	// walk backwards from the most recent entry (ringHead-1) over n entries.
+	var w forwardSummary
+	for i := 0; i < n; i++ {
+		idx := (s.ringHead - 1 - i + len(s.ring)) % len(s.ring)
+		w = w.merge(s.ring[idx])
+	}
+	return w
+}
+
+// GetStats returns the mean latency and jitter (std dev) of the forwarding
+// transit over the most recent duration. The duration is rounded up to a whole
+// number of summary intervals (the smallest bucket span that covers it). A
+// duration <= 0, or one that meets/exceeds the report window, covers the full
+// window.
+func (s *ForwardStats) GetStats(duration time.Duration) (time.Duration, time.Duration) {
+	return s.summarize(duration).meanStdDev()
+}
+
+func (s *ForwardStats) report() {
+	w := s.summarize(0)
+
+	latency, jitter := w.meanStdDev()
+	if dropped := s.samples.takeDropped(); dropped > 0 {
+		logger.Warnw("forward stats sample buffer overflow", nil, "dropped", dropped)
+	}
+	if w.count > 0 && jitter > latency*cSkewFactor {
+		logger.Infow(
+			"high jitter in forwarding path",
+			"lowest", time.Duration(w.minNs),
+			"highest", time.Duration(w.maxNs),
+			"count", w.count,
+			"latency", latency,
+			"jitter", jitter,
+		)
+	}
+
+	prometheus.RecordForwardJitter(uint32(jitter.Nanoseconds()))
+	prometheus.RecordForwardLatency(uint32(latency.Nanoseconds()))
+}

pkg/sfu/forwardstats_test.go

@@ -0,0 +1,365 @@
+package sfu
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"go.uber.org/atomic"
+
+	"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
+	"github.com/livekit/protocol/livekit"
+)
+
+// initPrometheus initializes the global forward-latency collectors so that the
+// worker's metric emission has non-nil targets. Init returns early if already
+// initialized, so it is safe to call from multiple tests.
+func initPrometheus(t *testing.T) {
+	t.Helper()
+	require.NoError(t, prometheus.Init("test", livekit.NodeType_SERVER))
+}
+
+// ---------------------------------------------------------------------------
+// forwardSummary
+// ---------------------------------------------------------------------------
+
+func TestForwardSummary_AddSample(t *testing.T) {
+	var s forwardSummary
+
+	// empty summary
+	require.Equal(t, int64(0), s.count)
+
+	// microsecond-aligned transits so the /1000 truncation is exact
+	s = s.addSample(3000) // 3us
+	s = s.addSample(1000) // 1us
+	s = s.addSample(2000) // 2us
+
+	require.Equal(t, int64(3), s.count)
+	require.Equal(t, int64(1+2+3), s.sumUs)
+	require.Equal(t, int64(1+4+9), s.sumSqUs)
+	require.Equal(t, int64(1000), s.minNs)
+	require.Equal(t, int64(3000), s.maxNs)
+}
+
+func TestForwardSummary_Merge(t *testing.T) {
+	var empty forwardSummary
+	a := forwardSummary{}.addSample(1000).addSample(2000)
+	b := forwardSummary{}.addSample(5000).addSample(3000)
+
+	// merging with empty is identity, in both directions
+	require.Equal(t, a, a.merge(empty))
+	require.Equal(t, a, empty.merge(a))
+
+	m := a.merge(b)
+	require.Equal(t, int64(4), m.count)
+	require.Equal(t, a.sumUs+b.sumUs, m.sumUs)
+	require.Equal(t, a.sumSqUs+b.sumSqUs, m.sumSqUs)
+	require.Equal(t, int64(1000), m.minNs)
+	require.Equal(t, int64(5000), m.maxNs)
+}
+
+func TestForwardSummary_MeanStdDev(t *testing.T) {
+	// empty -> zero
+	mean, stdDev := forwardSummary{}.meanStdDev()
+	require.Zero(t, mean)
+	require.Zero(t, stdDev)
+
+	// single sample -> mean set, stddev zero (needs >= 2 for variance)
+	mean, stdDev = forwardSummary{}.addSample(4000).meanStdDev()
+	require.Equal(t, 4*time.Microsecond, mean)
+	require.Zero(t, stdDev)
+
+	// identical samples -> zero variance
+	s := forwardSummary{}.addSample(2000).addSample(2000).addSample(2000)
+	mean, stdDev = s.meanStdDev()
+	require.Equal(t, 2*time.Microsecond, mean)
+	require.Zero(t, stdDev)
+
+	// known dataset [1us, 2us, 3us]: mean 2us, sample variance 1us^2 -> stddev 1us
+	s = forwardSummary{}.addSample(1000).addSample(2000).addSample(3000)
+	mean, stdDev = s.meanStdDev()
+	require.Equal(t, 2*time.Microsecond, mean)
+	require.InDelta(t, float64(time.Microsecond), float64(stdDev), float64(50*time.Nanosecond))
+}
+
+// ---------------------------------------------------------------------------
+// forwardSampleBuffer
+// ---------------------------------------------------------------------------
+
+func shardOf(arrival int64) int {
+	return int((uint64(arrival) >> 6) & forwardSampleShardSel)
+}
+
+// arrivalForShard returns the n-th arrival value that maps to a fixed shard.
+// Incrementing arrival by (1<<10) advances (arrival>>6) by 16, leaving the low
+// 4 selection bits unchanged.
+func arrivalForShard(n int) int64 {
+	return int64(n) << 10
+}
+
+func TestForwardSampleBuffer_PushDrain(t *testing.T) {
+	var b forwardSampleBuffer
+
+	const n = 1000
+	for i := 0; i < n; i++ {
+		b.push(int64(i), int64((i+1)*1000))
+	}
+
+	got := map[int64]int{}
+	total := 0
+	b.drain(func(v int64) {
+		got[v]++
+		total++
+	})
+	require.Equal(t, n, total)
+	require.Equal(t, uint64(0), b.dropped.Load())
+	for i := 0; i < n; i++ {
+		require.Equal(t, 1, got[int64((i+1)*1000)], "sample %d missing", i)
+	}
+
+	// draining again yields nothing (read cursor advanced)
+	total = 0
+	b.drain(func(v int64) { total++ })
+	require.Equal(t, 0, total)
+}
+
+func TestForwardSampleBuffer_Overflow(t *testing.T) {
+	var b forwardSampleBuffer
+
+	const extra = 100
+	const n = forwardSampleShardCap + extra
+
+	// pin every push to a single shard so it overflows
+	for i := 0; i < n; i++ {
+		b.push(arrivalForShard(i), int64(i)*1000)
+	}
+	require.Equal(t, 0, shardOf(arrivalForShard(0)))
+	require.Equal(t, shardOf(arrivalForShard(0)), shardOf(arrivalForShard(n-1)))
+
+	var drained []int64
+	b.drain(func(v int64) { drained = append(drained, v) })
+
+	// exactly a shard's worth survives; the oldest `extra` are dropped and counted
+	require.Len(t, drained, forwardSampleShardCap)
+	require.Equal(t, uint64(extra), b.dropped.Load())
+
+	// survivors are the most recent cap samples, in order
+	for j, v := range drained {
+		require.Equal(t, int64(extra+j)*1000, v)
+	}
+}
+
+func TestForwardSampleBuffer_DefersUncommitted(t *testing.T) {
+	var b forwardSampleBuffer
+	sh := &b.shards[0]
+
+	// simulate a producer that reserved index 0 but has not published its value
+	sh.writeIdx.Store(1)
+
+	got := 0
+	b.drain(func(int64) { got++ })
+	require.Equal(t, 0, got, "uncommitted slot must not be read")
+	require.Equal(t, uint64(0), sh.readIdx, "cursor must not advance past an uncommitted slot")
+	require.Equal(t, uint64(0), b.dropped.Load())
+
+	// producer publishes the value; next drain picks it up
+	sh.ring[0].Store(1234)
+	sh.seq[0].Store(1)
+
+	var vals []int64
+	b.drain(func(v int64) { vals = append(vals, v) })
+	require.Equal(t, []int64{1234}, vals)
+	require.Equal(t, uint64(1), sh.readIdx)
+	require.Equal(t, uint64(0), b.dropped.Load())
+}
+
+func TestForwardSampleBuffer_Concurrent(t *testing.T) {
+	var b forwardSampleBuffer
+	var stop atomic.Bool
+
+	var consumed int64
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for !stop.Load() {
+			b.drain(func(int64) { consumed++ })
+			time.Sleep(time.Millisecond)
+		}
+		b.drain(func(int64) { consumed++ }) // final sweep
+	}()
+
+	const producers = 8
+	const perProducer = 100_000
+	var wg sync.WaitGroup
+	for p := 0; p < producers; p++ {
+		wg.Add(1)
+		go func(seed int64) {
+			defer wg.Done()
+			for i := int64(0); i < perProducer; i++ {
+				b.push(seed*7+i, (i%50)*int64(time.Microsecond))
+			}
+		}(int64(p))
+	}
+	wg.Wait()
+	stop.Store(true)
+	<-done
+
+	// with a consumer keeping pace no samples should be lost
+	require.Equal(t, int64(producers*perProducer), consumed+int64(b.dropped.Load()))
+}
+
+// ---------------------------------------------------------------------------
+// ForwardStats
+// ---------------------------------------------------------------------------
+
+func TestForwardStats_Update(t *testing.T) {
+	s := &ForwardStats{ring: make([]forwardSummary, 1)}
+
+	// below threshold
+	transit, isHigh := s.Update(1000, 1000+int64(5*time.Millisecond))
+	require.Equal(t, int64(5*time.Millisecond), transit)
+	require.False(t, isHigh)
+
+	// above threshold
+	transit, isHigh = s.Update(1000, 1000+int64(25*time.Millisecond))
+	require.Equal(t, int64(25*time.Millisecond), transit)
+	require.True(t, isHigh)
+
+	// exactly at threshold is not "high" (strictly greater)
+	_, isHigh = s.Update(0, int64(cHighForwardingLatency))
+	require.False(t, isHigh)
+}
+
+func TestForwardStats_Flush(t *testing.T) {
+	initPrometheus(t)
+
+	s := &ForwardStats{ring: make([]forwardSummary, 4)}
+
+	for i := 0; i < 10; i++ {
+		s.Update(0, int64((i+1)*1000)) // 1us..10us
+	}
+
+	s.flush()
+
+	require.Equal(t, 1, s.ringLen)
+	summ := s.ring[0]
+	require.Equal(t, int64(10), summ.count)
+	require.Equal(t, int64(1000), summ.minNs)
+	require.Equal(t, int64(10000), summ.maxNs)
+	require.Equal(t, uint64(0), s.samples.dropped.Load())
+
+	// a subsequent flush with no new samples appends an empty summary
+	s.flush()
+	require.Equal(t, 2, s.ringLen)
+	require.Equal(t, int64(0), s.ring[1].count)
+}
+
+func TestForwardStats_ReportWindow(t *testing.T) {
+	initPrometheus(t)
+
+	// window of 3 summary buckets
+	s := &ForwardStats{ring: make([]forwardSummary, 3)}
+
+	s.Update(0, 1000)
+	s.flush()
+	s.Update(0, 3000)
+	s.flush()
+
+	// report merges the whole window without panicking and reflects both samples
+	var w forwardSummary
+	for i := 0; i < s.ringLen; i++ {
+		w = w.merge(s.ring[i])
+	}
+	require.Equal(t, int64(2), w.count)
+	require.Equal(t, int64(1000), w.minNs)
+	require.Equal(t, int64(3000), w.maxNs)
+
+	require.NotPanics(t, s.report)
+}
+
+func TestForwardStats_GetStats(t *testing.T) {
+	initPrometheus(t)
+
+	// 5 buckets, each covering one 100ms summary interval.
+	s := &ForwardStats{ring: make([]forwardSummary, 5), summaryInterval: 100 * time.Millisecond}
+
+	// fold five 100ms buckets, one sample each: 1ms, 2ms, 3ms, 4ms, 5ms.
+	for i := 1; i <= 5; i++ {
+		s.Update(0, int64(i)*int64(time.Millisecond))
+		s.flush()
+	}
+	require.Equal(t, 5, s.ringLen)
+
+	// a duration <= 0 covers the whole window: mean of 1..5ms == 3ms.
+	latency, jitter := s.GetStats(0)
+	require.InDelta(t, float64(3*time.Millisecond), float64(latency), float64(50*time.Microsecond))
+	require.Greater(t, jitter, time.Duration(0))
+
+	// a duration meeting/exceeding the window also covers it.
+	fullLatency, _ := s.GetStats(time.Second)
+	require.InDelta(t, float64(3*time.Millisecond), float64(fullLatency), float64(50*time.Microsecond))
+
+	// ~200ms rounds up to the two most recent buckets (4ms, 5ms): mean == 4.5ms.
+	shortLatency, _ := s.GetStats(200 * time.Millisecond)
+	require.InDelta(t, float64(4500*time.Microsecond), float64(shortLatency), float64(50*time.Microsecond))
+
+	// a sub-interval duration still yields at least the most recent bucket (5ms).
+	lastLatency, _ := s.GetStats(time.Nanosecond)
+	require.InDelta(t, float64(5*time.Millisecond), float64(lastLatency), float64(50*time.Microsecond))
+}
+
+func TestForwardStats_Lifecycle(t *testing.T) {
+	initPrometheus(t)
+
+	s := NewForwardStats(5*time.Millisecond, 20*time.Millisecond, 100*time.Millisecond)
+	for i := 0; i < 1000; i++ {
+		s.Update(int64(i), int64(i)+int64(time.Millisecond))
+	}
+	time.Sleep(60 * time.Millisecond) // let the worker flush/report a few times
+	require.NotPanics(t, s.Stop)
+}
+
+func TestNewForwardStats_RingSizing(t *testing.T) {
+	// ringCap = ceil(window / summaryInterval)
+	s := NewForwardStats(100*time.Millisecond, time.Second, time.Second)
+	require.Equal(t, 10, len(s.ring))
+	s.Stop()
+
+	// rounds up a partial interval
+	s = NewForwardStats(100*time.Millisecond, time.Second, 250*time.Millisecond)
+	require.Equal(t, 3, len(s.ring))
+	s.Stop()
+
+	// never smaller than one bucket, even if window < summaryInterval
+	s = NewForwardStats(time.Second, time.Second, 100*time.Millisecond)
+	require.Equal(t, 1, len(s.ring))
+	s.Stop()
+}
+
+// ---------------------------------------------------------------------------
+// benchmark: per-packet cost of Update (run with -cpu 1,8).
+// ---------------------------------------------------------------------------
+
+// benchArrival advances the arrival timestamp by 64ns per packet so that
+// consecutive packets from one goroutine map to successive shards
+// ((arrival>>6)&mask increments each step). A distinct per-goroutine base
+// spreads goroutines across shards.
+func benchArrival(base, i int64) int64 {
+	return base + i*64
+}
+
+func BenchmarkForwardStatsUpdate(b *testing.B) {
+	s := &ForwardStats{ring: make([]forwardSummary, 1)}
+
+	var gid atomic.Int64
+	b.RunParallel(func(pb *testing.PB) {
+		base := gid.Add(1) * 1_000_003
+		var i int64
+		for pb.Next() {
+			i++
+			arrival := benchArrival(base, i)
+			s.Update(arrival, arrival+int64(2*time.Millisecond))
+		}
+	})
+}

pkg/sfu/interceptor/rtx.go

@@ -0,0 +1,202 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package interceptor
+
+import (
+	"sync"
+
+	"github.com/pion/interceptor"
+	"github.com/pion/sdp/v3"
+	"github.com/pion/webrtc/v4"
+
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/sfu/utils"
+	"github.com/livekit/protocol/logger"
+)
+
+const (
+	SDESRepairRTPStreamIDURI = "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id"
+
+	rtxProbeCount = 10
+)
+
+// StreamInfoProber installs a bounded probe that identifies a remote stream from the
+// mid/rid/rsid header extensions of its packets. Implemented by buffer.Factory.
+type StreamInfoProber interface {
+	SetStreamInfoProbe(ssrc uint32, probe *buffer.StreamInfoProbe) bool
+}
+
+type streamInfo struct {
+	mid  string
+	rid  string
+	rsid string
+}
+
+type RTXInfoExtractorFactory struct {
+	onStreamFound  func(*interceptor.StreamInfo)
+	onRTXPairFound func(repair, base uint32, rsid string)
+	prober         StreamInfoProber
+	lock           sync.Mutex
+	streams        map[uint32]streamInfo
+	paired         map[uint32]struct{}
+	logger         logger.Logger
+}
+
+func NewRTXInfoExtractorFactory(
+	onStreamFound func(*interceptor.StreamInfo),
+	onRTXPairFound func(repair, base uint32, rsid string),
+	prober StreamInfoProber,
+	simTracks map[uint32]SimulcastTrackInfo,
+	logger logger.Logger,
+) *RTXInfoExtractorFactory {
+	f := &RTXInfoExtractorFactory{
+		onStreamFound:  onStreamFound,
+		onRTXPairFound: onRTXPairFound,
+		prober:         prober,
+		streams:        make(map[uint32]streamInfo),
+		paired:         make(map[uint32]struct{}),
+		logger:         logger,
+	}
+	f.seedSimulcastTracks(simTracks)
+	return f
+}
+
+// seedSimulcastTracks pairs migrated streams from the migration info. A migrated client
+// is mid-stream and stops sending rid/rsid, so the extensions never appear on the wire
+// and the pairing has to come from what is already known about the tracks.
+func (f *RTXInfoExtractorFactory) seedSimulcastTracks(simTracks map[uint32]SimulcastTrackInfo) {
+	for ssrc, info := range simTracks {
+		if info.Mid == "" || info.StreamID == "" {
+			continue
+		}
+
+		if info.IsRepairStream {
+			f.SetStreamInfo(ssrc, info.Mid, "", info.StreamID)
+			continue
+		}
+
+		f.SetStreamInfo(ssrc, info.Mid, info.StreamID, "")
+		if info.RepairSSRC != 0 {
+			f.SetStreamInfo(info.RepairSSRC, info.Mid, "", info.StreamID)
+		}
+	}
+}
+
+func (f *RTXInfoExtractorFactory) NewInterceptor(id string) (interceptor.Interceptor, error) {
+	return &RTXInfoExtractor{
+		factory: f,
+		logger:  f.logger,
+	}, nil
+}
+
+func (f *RTXInfoExtractorFactory) SetStreamInfo(ssrc uint32, mid, rid, rsid string) {
+	var repairSsrc, baseSsrc uint32
+	var repairSid string
+	f.lock.Lock()
+
+	if mid == "" || (rid == "" && rsid == "") {
+		f.lock.Unlock()
+		return
+	}
+
+	// the same stream can be reported by both the packet probe and the migration info
+	if _, ok := f.paired[ssrc]; ok {
+		f.lock.Unlock()
+		return
+	}
+
+	if rsid != "" {
+		// repair stream found, find base stream
+		for base, info := range f.streams {
+			if info.mid == mid && info.rid == rsid {
+				repairSsrc = ssrc
+				baseSsrc = base
+				repairSid = rsid
+				delete(f.streams, base)
+				break
+			}
+		}
+	} else {
+		// base stream found, find repair stream
+		for repair, info := range f.streams {
+			if info.mid == mid && info.rsid == rid {
+				repairSsrc = repair
+				baseSsrc = ssrc
+				repairSid = rid
+				delete(f.streams, repair)
+				break
+			}
+		}
+	}
+
+	// no rtx pair found, save it for later
+	if repairSsrc == 0 || baseSsrc == 0 {
+		f.streams[ssrc] = streamInfo{
+			mid:  mid,
+			rid:  rid,
+			rsid: rsid,
+		}
+		f.lock.Unlock()
+		return
+	}
+
+	f.paired[repairSsrc] = struct{}{}
+	f.paired[baseSsrc] = struct{}{}
+	f.lock.Unlock()
+
+	f.onRTXPairFound(repairSsrc, baseSsrc, repairSid)
+}
+
+// ------------------------------------------
+
+type RTXInfoExtractor struct {
+	interceptor.NoOp
+
+	factory *RTXInfoExtractorFactory
+	logger  logger.Logger
+}
+
+func (u *RTXInfoExtractor) BindRemoteStream(info *interceptor.StreamInfo, reader interceptor.RTPReader) interceptor.RTPReader {
+	u.factory.onStreamFound(info)
+
+	midExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESMidURI})
+	streamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESRTPStreamIDURI})
+	repairStreamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: SDESRepairRTPStreamIDURI})
+	if midExtensionID == 0 || streamIDExtensionID == 0 || repairStreamIDExtensionID == 0 {
+		return reader
+	}
+
+	// Probe on the buffer write path rather than by wrapping this reader. Remote streams
+	// are consumed through SettingEngine.BufferFactory, so nothing here reads the
+	// interceptor chain. pion used to drive the repair stream reader, but since
+	// pion/webrtc#3470 it only does so when the application reads the TrackRemote or
+	// when no BufferFactory is set, neither of which holds.
+	ok := u.factory.prober.SetStreamInfoProbe(info.SSRC, &buffer.StreamInfoProbe{
+		MidExtID:  uint8(midExtensionID),
+		RidExtID:  uint8(streamIDExtensionID),
+		RsidExtID: uint8(repairStreamIDExtensionID),
+		Tries:     rtxProbeCount,
+		OnFound:   u.factory.SetStreamInfo,
+	})
+	if !ok {
+		u.logger.Warnw(
+			"could not install stream info probe, rtx pairing will not work", nil,
+			"ssrc", info.SSRC,
+			"mime", info.MimeType,
+		)
+	}
+
+	return reader
+}

pkg/sfu/interceptor/unhandlesimulcast.go

@@ -0,0 +1,183 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package interceptor
+
+import (
+	"github.com/pion/interceptor"
+	"github.com/pion/rtp"
+	"github.com/pion/sdp/v3"
+	"github.com/pion/webrtc/v4"
+	"go.uber.org/zap/zapcore"
+
+	"github.com/livekit/livekit-server/pkg/sfu/utils"
+	"github.com/livekit/protocol/logger"
+)
+
+const (
+	simulcastProbeCount = 10
+)
+
+type SimulcastTrackInfo struct {
+	Mid            string
+	StreamID       string
+	RepairSSRC     uint32 // set only when `IsRepairStream: false`, i. e. RTX SSRC for the primary stream
+	IsRepairStream bool
+}
+
+func (s *SimulcastTrackInfo) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	e.AddString("Mid", s.Mid)
+	e.AddString("StreamID", s.StreamID)
+	e.AddUint32("RepairSSRC", s.RepairSSRC)
+	e.AddBool("IsRepairStream", s.IsRepairStream)
+	return nil
+}
+
+// -------------------------------------------------------------------
+
+type UnhandleSimulcastOption func(u *UnhandleSimulcastInterceptor) error
+
+func UnhandleSimulcastTracks(logger logger.Logger, tracks map[uint32]SimulcastTrackInfo) UnhandleSimulcastOption {
+	return func(u *UnhandleSimulcastInterceptor) error {
+		u.logger = logger
+		u.simTracks = tracks
+		return nil
+	}
+}
+
+type UnhandleSimulcastInterceptorFactory struct {
+	opts []UnhandleSimulcastOption
+}
+
+func (f *UnhandleSimulcastInterceptorFactory) NewInterceptor(id string) (interceptor.Interceptor, error) {
+	i := &UnhandleSimulcastInterceptor{simTracks: map[uint32]SimulcastTrackInfo{}}
+	for _, o := range f.opts {
+		if err := o(i); err != nil {
+			return nil, err
+		}
+	}
+	return i, nil
+}
+
+func NewUnhandleSimulcastInterceptorFactory(opts ...UnhandleSimulcastOption) (*UnhandleSimulcastInterceptorFactory, error) {
+	return &UnhandleSimulcastInterceptorFactory{opts: opts}, nil
+}
+
+type unhandleSimulcastRTPReader struct {
+	SimulcastTrackInfo
+	logger    logger.Logger
+	tryTimes  int
+	reader    interceptor.RTPReader
+	midExtID  uint8
+	ridExtID  uint8
+	rsidExtID uint8
+}
+
+func (u *unhandleSimulcastRTPReader) Read(b []byte, a interceptor.Attributes) (int, interceptor.Attributes, error) {
+	n, a, err := u.reader.Read(b, a)
+	if u.tryTimes < 0 || err != nil {
+		return n, a, err
+	}
+
+	header := rtp.Header{}
+	hsize, err := header.Unmarshal(b[:n])
+	if err != nil {
+		return n, a, nil
+	}
+	var mid, rid, rsid string
+	if payload := header.GetExtension(u.midExtID); payload != nil {
+		mid = string(payload)
+	}
+
+	if payload := header.GetExtension(u.ridExtID); payload != nil {
+		rid = string(payload)
+	}
+
+	if payload := header.GetExtension(u.rsidExtID); payload != nil {
+		rid = string(payload)
+	}
+
+	if mid != "" && (rid != "" || rsid != "") {
+		u.logger.Debugw(
+			"unhandle stream found",
+			"mid", mid,
+			"rid", rid,
+			"rsid", rsid,
+			"ssrc", header.SSRC,
+			"simulcastTrackInfo", u.SimulcastTrackInfo,
+		)
+		u.tryTimes = -1
+		return n, a, nil
+	} else {
+		// ignore padding only packet for probe count
+		if !header.Padding || n-header.MarshalSize()-int(b[n-1]) != 0 {
+			u.tryTimes--
+		}
+	}
+
+	if mid == "" {
+		header.SetExtension(u.midExtID, []byte(u.Mid))
+	}
+	if rid == "" && !u.IsRepairStream {
+		header.SetExtension(u.ridExtID, []byte(u.StreamID))
+	}
+	if rsid == "" && u.IsRepairStream {
+		header.SetExtension(u.rsidExtID, []byte(u.StreamID))
+	}
+
+	hsize2 := header.MarshalSize()
+
+	if hsize2-hsize+n > len(b) { // no enough buf to set extension
+		return n, a, nil
+	}
+	copy(b[hsize2:], b[hsize:n])
+	header.MarshalTo(b)
+	u.logger.Debugw(
+		"unhandle stream injecting",
+		"mid", mid,
+		"rid", rid,
+		"rsid", rsid,
+		"ssrc", header.SSRC,
+		"simulcastTrackInfo", u.SimulcastTrackInfo,
+	)
+	return hsize2 - hsize + n, a, nil
+}
+
+type UnhandleSimulcastInterceptor struct {
+	interceptor.NoOp
+	logger    logger.Logger
+	simTracks map[uint32]SimulcastTrackInfo
+}
+
+func (u *UnhandleSimulcastInterceptor) BindRemoteStream(info *interceptor.StreamInfo, reader interceptor.RTPReader) interceptor.RTPReader {
+	if t, ok := u.simTracks[info.SSRC]; ok {
+		midExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESMidURI})
+		streamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESRTPStreamIDURI})
+		repairStreamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESRepairRTPStreamIDURI})
+		if midExtensionID == 0 || streamIDExtensionID == 0 || repairStreamIDExtensionID == 0 {
+			return reader
+		}
+
+		return &unhandleSimulcastRTPReader{
+			SimulcastTrackInfo: t,
+			logger:             u.logger,
+			reader:             reader,
+			tryTimes:           simulcastProbeCount,
+			midExtID:           uint8(midExtensionID),
+			ridExtID:           uint8(streamIDExtensionID),
+			rsidExtID:          uint8(repairStreamIDExtensionID),
+		}
+	}
+	return reader
+}

pkg/sfu/interfaces.go

@@ -0,0 +1,189 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sfu
+
+import (
+	"time"
+
+	"github.com/pion/rtcp"
+	"github.com/pion/webrtc/v4"
+
+	"github.com/livekit/mediatransportutil/pkg/codec"
+	"github.com/livekit/protocol/codecs/mime"
+	"github.com/livekit/protocol/livekit"
+
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/sfu/bwe"
+)
+
+//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate
+
+// TrackReceiver defines an interface receive media from remote peer
+//
+//counterfeiter:generate . TrackReceiver
+type TrackReceiver interface {
+	TrackID() livekit.TrackID
+	StreamID() string
+
+	// returns the initial codec of the receiver, it is determined by the track's codec
+	// and will not change if the codec changes during the session (publisher changes codec)
+	Codec() webrtc.RTPCodecParameters
+	Mime() mime.MimeType
+	VideoLayerMode() livekit.VideoLayer_Mode
+	HeaderExtensions() []webrtc.RTPHeaderExtensionParameter
+	IsClosed() bool
+
+	ReadRTP(buf []byte, layer uint8, esn uint64) (int, error)
+	GetLayeredBitrate() ([]int32, Bitrates)
+
+	GetAudioLevel() (float64, bool)
+
+	SendPLI(layer int32, force bool)
+
+	SetMaxExpectedSpatialLayer(layer int32)
+
+	AddDownTrack(track TrackSender) error
+	DeleteDownTrack(participantID livekit.ParticipantID)
+	GetDownTracks() []TrackSender
+
+	DebugInfo() map[string]any
+
+	TrackInfo() *livekit.TrackInfo
+	UpdateTrackInfo(ti *livekit.TrackInfo)
+
+	// Get primary receiver if this receiver represents a RED codec; otherwise it will return itself
+	GetPrimaryReceiverForRed() TrackReceiver
+
+	// Get red receiver for primary codec, used by forward red encodings for opus only codec
+	GetRedReceiver() TrackReceiver
+
+	GetTemporalLayerFpsForSpatial(layer int32) []float32
+
+	GetTrackStats() *livekit.RTPStats
+
+	// AddOnReady adds a function to be called when the receiver is ready, the callback
+	// could be called immediately if the receiver is ready when the callback is added
+	AddOnReady(func())
+
+	AddOnCodecStateChange(func(webrtc.RTPCodecParameters, ReceiverCodecState))
+	CodecState() ReceiverCodecState
+
+	// VideoSizes returns the video size parsed from rtp packet for each spatial layer.
+	VideoSizes() []codec.VideoSize
+
+	// closes all associated buffers and issues a resync to all attached downtracks so that
+	// they can resync and have proper sequncing without gaps in sequence numbers / timestamps
+	Restart(reason string)
+}
+
+// --------------------------------------
+
+//counterfeiter:generate . REDTransformer
+type REDTransformer interface {
+	TrackReceiver
+	SetLBThreshold(lbThreshold int)
+	ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int32
+	ForwardRTCPSenderReport(
+		payloadType webrtc.PayloadType,
+		layer int32,
+		publisherSRData *livekit.RTCPSenderReportState,
+	)
+	GetDownTracks() []TrackSender
+	ResyncDownTracks()
+	OnStreamRestart()
+	CanClose() bool
+	Close()
+}
+
+// --------------------------------------
+
+// TrackSender defines an interface send media to remote peer
+//
+//counterfeiter:generate . TrackSender
+type TrackSender interface {
+	UpTrackLayersChange()
+	UpTrackBitrateAvailabilityChange()
+	UpTrackMaxPublishedLayerChange(maxPublishedLayer int32)
+	UpTrackMaxTemporalLayerSeenChange(maxTemporalLayerSeen int32)
+	UpTrackBitrateReport(availableLayers []int32, bitrates Bitrates)
+	WriteRTP(p *buffer.ExtPacket, layer int32) int32
+	Close()
+	IsClosed() bool
+	// ID is the globally unique identifier for this Track.
+	ID() string
+	SubscriberID() livekit.ParticipantID
+	HandleRTCPSenderReportData(
+		payloadType webrtc.PayloadType,
+		layer int32,
+		publisherSRData *livekit.RTCPSenderReportState,
+	) error
+	Resync()
+	SetReceiver(TrackReceiver)
+	ReceiverRestart(TrackReceiver)
+}
+
+// -------------------------------------------------------------------
+
+//counterfeiter:generate . DownTrackStreamAllocatorListener
+type DownTrackStreamAllocatorListener interface {
+	// RTCP received
+	OnREMB(dt *DownTrack, remb *rtcp.ReceiverEstimatedMaximumBitrate)
+	OnTransportCCFeedback(dt *DownTrack, cc *rtcp.TransportLayerCC)
+
+	// video layer availability changed
+	OnAvailableLayersChanged(dt *DownTrack)
+
+	// video layer bitrate availability changed
+	OnBitrateAvailabilityChanged(dt *DownTrack)
+
+	// max published spatial layer changed
+	OnMaxPublishedSpatialChanged(dt *DownTrack)
+
+	// max published temporal layer changed
+	OnMaxPublishedTemporalChanged(dt *DownTrack)
+
+	// subscription changed - mute/unmute
+	OnSubscriptionChanged(dt *DownTrack)
+
+	// subscribed max video layer changed
+	OnSubscribedLayerChanged(dt *DownTrack, layers buffer.VideoLayer)
+
+	// stream resumed
+	OnResume(dt *DownTrack)
+
+	// check if track should participate in BWE
+	IsBWEEnabled(dt *DownTrack) bool
+
+	// get the BWE type in use
+	BWEType() bwe.BWEType
+
+	// check if subscription mute can be applied
+	IsSubscribeMutable(dt *DownTrack) bool
+}
+
+// -------------------------------------------------------------------
+
+//counterfeiter:generate . DownTrackListener
+type DownTrackListener interface {
+	OnBindAndConnected()
+	OnStatsUpdate(stat *livekit.AnalyticsStat)
+	OnMaxSubscribedLayerChanged(layer int32)
+	OnRttUpdate(rtt uint32)
+	OnCodecNegotiated(webrtc.RTPCodecCapability)
+	OnDownTrackClose(isExpectedToResume bool)
+	OnStreamStarted(elapsed time.Duration)
+}
+
+// -------------------------------------------------------------------

pkg/sfu/pacer/base.go

@@ -0,0 +1,138 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pacer
+
+import (
+	"errors"
+	"io"
+	"time"
+
+	"github.com/livekit/livekit-server/pkg/sfu/bwe"
+	"github.com/livekit/mediatransportutil"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils/mono"
+	"github.com/pion/rtp"
+	"go.uber.org/atomic"
+)
+
+type Base struct {
+	logger logger.Logger
+
+	bwe bwe.BWE
+
+	lastPacketSentAt atomic.Int64
+
+	*ProbeObserver
+}
+
+func NewBase(logger logger.Logger, bwe bwe.BWE) *Base {
+	return &Base{
+		logger:        logger,
+		bwe:           bwe,
+		ProbeObserver: NewProbeObserver(logger),
+	}
+}
+
+func (b *Base) SetInterval(_interval time.Duration) {
+}
+
+func (b *Base) SetBitrate(_bitrate int) {
+}
+
+func (b *Base) TimeSinceLastSentPacket() time.Duration {
+	return time.Duration(mono.UnixNano() - b.lastPacketSentAt.Load())
+}
+
+func (b *Base) SendPacket(p *Packet) (int, error) {
+	defer func() {
+		if p.HeaderPool != nil && p.Header != nil {
+			*p.Header = rtp.Header{}
+			p.HeaderPool.Put(p.Header)
+		}
+
+		if p.Pool != nil && p.PoolEntity != nil {
+			p.Pool.Put(p.PoolEntity)
+		}
+
+		*p = Packet{}
+		PacketFactory.Put(p)
+	}()
+
+	err := b.patchRTPHeaderExtensions(p)
+	if err != nil {
+		b.logger.Errorw("patching rtp header extensions err", err)
+		return 0, err
+	}
+
+	var written int
+	written, err = p.WriteStream.WriteRTP(p.Header, p.Payload)
+	if err != nil {
+		if !errors.Is(err, io.ErrClosedPipe) {
+			b.logger.Errorw("write rtp packet failed", err)
+		}
+		return 0, err
+	}
+
+	return written, nil
+}
+
+// patch just abs-send-time and transport-cc extensions if applicable
+func (b *Base) patchRTPHeaderExtensions(p *Packet) error {
+	sendingAt := mono.Now()
+	if p.AbsSendTimeExtID != 0 {
+		absSendTimeExt := rtp.AbsSendTimeExtension{
+			Timestamp: uint64(mediatransportutil.ToNtpTime(sendingAt) >> 14),
+		}
+		absSendTimeBytes, err := absSendTimeExt.Marshal()
+		if err != nil {
+			return err
+		}
+
+		if err = p.Header.SetExtension(p.AbsSendTimeExtID, absSendTimeBytes); err != nil {
+			return err
+		}
+
+		b.lastPacketSentAt.Store(sendingAt.UnixNano())
+	}
+
+	packetSize := p.HeaderSize + len(p.Payload)
+	if p.TransportWideExtID != 0 && b.bwe != nil {
+		twccSN := b.bwe.RecordPacketSendAndGetSequenceNumber(
+			sendingAt.UnixMicro(),
+			packetSize,
+			p.IsRTX,
+			p.ProbeClusterId,
+			p.IsProbe,
+		)
+		twccExt := rtp.TransportCCExtension{
+			TransportSequence: twccSN,
+		}
+		twccExtBytes, err := twccExt.Marshal()
+		if err != nil {
+			return err
+		}
+
+		if err = p.Header.SetExtension(p.TransportWideExtID, twccExtBytes); err != nil {
+			return err
+		}
+
+		b.lastPacketSentAt.Store(sendingAt.UnixNano())
+	}
+
+	b.ProbeObserver.RecordPacket(packetSize, p.IsRTX, p.ProbeClusterId, p.IsProbe)
+	return nil
+}
+
+// ------------------------------------------------

pkg/sfu/pacer/leaky_bucket.go

@@ -0,0 +1,142 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pacer
+
+import (
+	"sync"
+	"time"
+
+	"github.com/frostbyte73/core"
+	"github.com/gammazero/deque"
+	"github.com/livekit/livekit-server/pkg/sfu/bwe"
+	"github.com/livekit/protocol/logger"
+)
+
+const (
+	maxOvershootFactor = 2.0
+)
+
+type LeakyBucket struct {
+	*Base
+
+	logger logger.Logger
+
+	lock     sync.RWMutex
+	packets  deque.Deque[*Packet]
+	interval time.Duration
+	bitrate  int
+	stop     core.Fuse
+}
+
+func NewLeakyBucket(logger logger.Logger, bwe bwe.BWE, interval time.Duration, bitrate int) *LeakyBucket {
+	l := &LeakyBucket{
+		Base:     NewBase(logger, bwe),
+		logger:   logger,
+		interval: interval,
+		bitrate:  bitrate,
+	}
+	l.packets.SetBaseCap(512)
+
+	go l.sendWorker()
+	return l
+}
+
+func (l *LeakyBucket) SetInterval(interval time.Duration) {
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	l.interval = interval
+}
+
+func (l *LeakyBucket) SetBitrate(bitrate int) {
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	l.bitrate = bitrate
+}
+
+func (l *LeakyBucket) Stop() {
+	l.stop.Break()
+}
+
+func (l *LeakyBucket) Enqueue(p *Packet) {
+	l.lock.Lock()
+	l.packets.PushBack(p)
+	l.lock.Unlock()
+}
+
+func (l *LeakyBucket) sendWorker() {
+	l.lock.RLock()
+	interval := l.interval
+	l.lock.RUnlock()
+	var bitrate int
+
+	timer := time.NewTimer(interval)
+	overage := 0
+
+	for {
+		<-timer.C
+
+		l.lock.RLock()
+		interval = l.interval
+		bitrate = l.bitrate
+		l.lock.RUnlock()
+
+		// calculate number of bytes that can be sent in this interval
+		// adjusting for overage.
+		intervalBytes := int(interval.Seconds() * float64(bitrate) / 8.0)
+		maxOvershootBytes := int(float64(intervalBytes) * maxOvershootFactor)
+		toSendBytes := intervalBytes - overage
+		if toSendBytes < 0 {
+			// too much overage, wait for next interval
+			overage = -toSendBytes
+			timer.Reset(interval)
+			continue
+		}
+
+		// do not allow too much overshoot in an interval
+		if toSendBytes > maxOvershootBytes {
+			toSendBytes = maxOvershootBytes
+		}
+
+		for {
+			if l.stop.IsBroken() {
+				return
+			}
+
+			l.lock.Lock()
+			if l.packets.Len() == 0 {
+				l.lock.Unlock()
+				// allow overshoot in next interval with shortage in this interval
+				overage = -toSendBytes
+				timer.Reset(interval)
+				break
+			}
+			p := l.packets.PopFront()
+			l.lock.Unlock()
+
+			written, _ := l.Base.SendPacket(p)
+			toSendBytes -= written
+			if toSendBytes < 0 {
+				// overage, wait for next interval
+				overage = -toSendBytes
+				timer.Reset(interval)
+				break
+			}
+		}
+	}
+}
+
+// ------------------------------------------------

pkg/sfu/pacer/no_queue.go

@@ -0,0 +1,90 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pacer
+
+import (
+	"sync"
+
+	"github.com/frostbyte73/core"
+	"github.com/gammazero/deque"
+	"github.com/livekit/livekit-server/pkg/sfu/bwe"
+	"github.com/livekit/protocol/logger"
+)
+
+type NoQueue struct {
+	*Base
+
+	logger logger.Logger
+
+	lock    sync.RWMutex
+	packets deque.Deque[*Packet]
+	wake    chan struct{}
+	stop    core.Fuse
+}
+
+func NewNoQueue(logger logger.Logger, bwe bwe.BWE) *NoQueue {
+	n := &NoQueue{
+		Base:   NewBase(logger, bwe),
+		logger: logger,
+		wake:   make(chan struct{}, 1),
+	}
+	n.packets.SetBaseCap(512)
+
+	go n.sendWorker()
+	return n
+}
+
+func (n *NoQueue) Stop() {
+	n.stop.Break()
+
+	select {
+	case n.wake <- struct{}{}:
+	default:
+	}
+}
+
+func (n *NoQueue) Enqueue(p *Packet) {
+	n.lock.Lock()
+	n.packets.PushBack(p)
+	n.lock.Unlock()
+
+	select {
+	case n.wake <- struct{}{}:
+	default:
+	}
+}
+
+func (n *NoQueue) sendWorker() {
+	for {
+		<-n.wake
+		for {
+			if n.stop.IsBroken() {
+				return
+			}
+
+			n.lock.Lock()
+			if n.packets.Len() == 0 {
+				n.lock.Unlock()
+				break
+			}
+			p := n.packets.PopFront()
+			n.lock.Unlock()
+
+			n.Base.SendPacket(p)
+		}
+	}
+}
+
+// ------------------------------------------------

pkg/sfu/pacer/pacer.go

@@ -0,0 +1,77 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pacer
+
+import (
+	"sync"
+	"time"
+
+	"github.com/livekit/livekit-server/pkg/sfu/ccutils"
+	"github.com/pion/rtp"
+	"github.com/pion/webrtc/v4"
+)
+
+var (
+	PacketFactory = &sync.Pool{
+		New: func() any {
+			return &Packet{}
+		},
+	}
+)
+
+// --------------------------------------
+
+type PacerBehavior string
+
+const (
+	PacerBehaviorPassThrough PacerBehavior = "pass-through"
+	PacerBehaviorNoQueue     PacerBehavior = "no-queue"
+	PacerBehaviorLeakybucket PacerBehavior = "leaky-bucket"
+)
+
+type Packet struct {
+	Header             *rtp.Header
+	HeaderPool         *sync.Pool
+	HeaderSize         int
+	Payload            []byte
+	IsRTX              bool
+	ProbeClusterId     ccutils.ProbeClusterId
+	IsProbe            bool
+	AbsSendTimeExtID   uint8
+	TransportWideExtID uint8
+	WriteStream        webrtc.TrackLocalWriter
+	Pool               *sync.Pool
+	PoolEntity         *[]byte
+}
+
+type Pacer interface {
+	Enqueue(p *Packet)
+	Stop()
+
+	SetInterval(interval time.Duration)
+	SetBitrate(bitrate int)
+
+	TimeSinceLastSentPacket() time.Duration
+
+	SetPacerProbeObserverListener(listener PacerProbeObserverListener)
+	StartProbeCluster(pci ccutils.ProbeClusterInfo)
+	EndProbeCluster(probeClusterId ccutils.ProbeClusterId) ccutils.ProbeClusterInfo
+}
+
+type PacerProbeObserverListener interface {
+	OnPacerProbeObserverClusterComplete(probeClusterId ccutils.ProbeClusterId)
+}
+
+// ------------------------------------------------

pkg/sfu/pacer/pass_through.go

@@ -0,0 +1,39 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pacer
+
+import (
+	"github.com/livekit/livekit-server/pkg/sfu/bwe"
+	"github.com/livekit/protocol/logger"
+)
+
+type PassThrough struct {
+	*Base
+}
+
+func NewPassThrough(logger logger.Logger, bwe bwe.BWE) *PassThrough {
+	return &PassThrough{
+		Base: NewBase(logger, bwe),
+	}
+}
+
+func (p *PassThrough) Stop() {
+}
+
+func (p *PassThrough) Enqueue(pkt *Packet) {
+	p.Base.SendPacket(pkt)
+}
+
+// ------------------------------------------------

pkg/sfu/pacer/probe_observer.go

@@ -0,0 +1,143 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pacer
+
+import (
+	"sync"
+	"time"
+
+	"go.uber.org/atomic"
+
+	"github.com/livekit/livekit-server/pkg/sfu/ccutils"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils/mono"
+)
+
+type ProbeObserver struct {
+	logger logger.Logger
+
+	listener PacerProbeObserverListener
+
+	isInProbe atomic.Bool
+
+	lock sync.Mutex
+	pci  ccutils.ProbeClusterInfo
+}
+
+func NewProbeObserver(logger logger.Logger) *ProbeObserver {
+	return &ProbeObserver{
+		logger: logger,
+	}
+}
+
+func (po *ProbeObserver) SetPacerProbeObserverListener(listener PacerProbeObserverListener) {
+	po.listener = listener
+}
+
+func (po *ProbeObserver) StartProbeCluster(pci ccutils.ProbeClusterInfo) {
+	if po.isInProbe.Load() {
+		po.logger.Warnw(
+			"ignoring start of a new probe cluster when already active", nil,
+			"probeClusterInfo", pci,
+		)
+		return
+	}
+
+	po.lock.Lock()
+	defer po.lock.Unlock()
+
+	po.pci = pci
+	po.pci.Result = ccutils.ProbeClusterResult{
+		StartTime: mono.UnixNano(),
+	}
+
+	po.isInProbe.Store(true)
+}
+
+func (po *ProbeObserver) EndProbeCluster(probeClusterId ccutils.ProbeClusterId) ccutils.ProbeClusterInfo {
+	if !po.isInProbe.Load() {
+		// probe not active
+		if probeClusterId != ccutils.ProbeClusterIdInvalid {
+			po.logger.Debugw(
+				"ignoring end of a probe cluster when not active",
+				"probeClusterId", probeClusterId,
+			)
+		}
+		return ccutils.ProbeClusterInfoInvalid
+	}
+
+	po.lock.Lock()
+	defer po.lock.Unlock()
+
+	if po.pci.Id != probeClusterId {
+		// probe cluster id not active
+		po.logger.Warnw(
+			"ignoring end of a probe cluster of a non-active one", nil,
+			"probeClusterId", probeClusterId,
+			"active", po.pci.Id,
+		)
+		return ccutils.ProbeClusterInfoInvalid
+	}
+
+	if po.pci.Result.EndTime == 0 {
+		po.pci.Result.EndTime = mono.UnixNano()
+	}
+
+	po.isInProbe.Store(false)
+
+	return po.pci
+}
+
+func (po *ProbeObserver) RecordPacket(size int, isRTX bool, probeClusterId ccutils.ProbeClusterId, isProbe bool) {
+	if !po.isInProbe.Load() {
+		return
+	}
+
+	po.lock.Lock()
+	if probeClusterId != po.pci.Id || po.pci.Result.EndTime != 0 {
+		po.lock.Unlock()
+		return
+	}
+
+	if isProbe {
+		po.pci.Result.PacketsProbe++
+		po.pci.Result.BytesProbe += size
+	} else {
+		if isRTX {
+			po.pci.Result.PacketsNonProbeRTX++
+			po.pci.Result.BytesNonProbeRTX += size
+		} else {
+			po.pci.Result.PacketsNonProbePrimary++
+			po.pci.Result.BytesNonProbePrimary += size
+		}
+	}
+
+	notify := false
+	var clusterId ccutils.ProbeClusterId
+	if po.pci.Result.EndTime == 0 && ((po.pci.Result.Bytes() >= po.pci.Goal.DesiredBytes) && time.Duration(mono.UnixNano()-po.pci.Result.StartTime) >= po.pci.Goal.Duration) {
+		po.pci.Result.EndTime = mono.UnixNano()
+		po.pci.Result.IsCompleted = true
+
+		notify = true
+		clusterId = po.pci.Id
+	}
+	po.lock.Unlock()
+
+	if notify && po.listener != nil {
+		po.listener.OnPacerProbeObserverClusterComplete(clusterId)
+	}
+}
+
+// ------------------------------------------------

pkg/sfu/packettrailer/packet_trailer.go

@@ -0,0 +1,48 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package packettrailer
+
+var Magic = [4]byte{'L', 'K', 'T', 'S'}
+
+const (
+	xorByte = 0xFF
+
+	envelopeSize = 5 // 1B trailer_len + 4B magic
+)
+
+// StripTrailer returns the number of bytes to strip from the end of an RTP
+// payload if it contains an LKTS trailer. The trailer is located by checking
+// for the "LKTS" magic suffix and then reading the XORed trailer_len byte
+// immediately before it. isEndOfFrame must be set only for packets ending an
+// encoded frame, i. e. where a trailer could have been appended. Returns 0 if
+// absent or ineligible.
+func StripTrailer(payload []byte, isEndOfFrame bool) int {
+	if !isEndOfFrame || len(payload) < envelopeSize {
+		return 0
+	}
+
+	tail := payload[len(payload)-4:]
+	if tail[0] != Magic[0] || tail[1] != Magic[1] ||
+		tail[2] != Magic[2] || tail[3] != Magic[3] {
+		return 0
+	}
+
+	trailerLen := int(payload[len(payload)-5] ^ xorByte)
+	if trailerLen < envelopeSize || trailerLen > len(payload) {
+		return 0
+	}
+
+	return trailerLen
+}

pkg/sfu/packettrailer/packet_trailer_test.go

@@ -0,0 +1,182 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package packettrailer
+
+import (
+	"encoding/binary"
+	"testing"
+)
+
+const (
+	tagTimestampUs = 0x01
+	tagFrameID     = 0x02
+)
+
+// appendTLV appends a single XORed TLV element to dst.
+func appendTLV(dst []byte, tag byte, value []byte) []byte {
+	dst = append(dst, tag^xorByte, byte(len(value))^xorByte)
+	for _, b := range value {
+		dst = append(dst, b^xorByte)
+	}
+	return dst
+}
+
+// appendEnvelope appends the 5-byte envelope (XORed trailer_len + magic).
+func appendEnvelope(dst []byte, trailerLen byte) []byte {
+	dst = append(dst, trailerLen^xorByte)
+	dst = append(dst, Magic[:]...)
+	return dst
+}
+
+// makeTrailer builds a complete LKTS trailer with both timestamp and frame_id TLVs.
+func makeTrailer(timestampUs int64, frameID uint32) []byte {
+	var trailer []byte
+
+	var tsBuf [8]byte
+	binary.BigEndian.PutUint64(tsBuf[:], uint64(timestampUs))
+	trailer = appendTLV(trailer, tagTimestampUs, tsBuf[:])
+
+	var fidBuf [4]byte
+	binary.BigEndian.PutUint32(fidBuf[:], frameID)
+	trailer = appendTLV(trailer, tagFrameID, fidBuf[:])
+
+	trailerLen := byte(len(trailer) + envelopeSize)
+	trailer = appendEnvelope(trailer, trailerLen)
+	return trailer
+}
+
+// makePayloadWithTrailer builds a fake video payload followed by a full LKTS trailer.
+func makePayloadWithTrailer(videoLen int, timestampUs int64, frameID uint32) []byte {
+	video := make([]byte, videoLen)
+	for i := range video {
+		video[i] = byte(i)
+	}
+	return append(video, makeTrailer(timestampUs, frameID)...)
+}
+
+// makeTimestampOnlyTrailer builds a trailer with only the timestamp TLV.
+func makeTimestampOnlyTrailer(timestampUs int64) []byte {
+	var trailer []byte
+	var tsBuf [8]byte
+	binary.BigEndian.PutUint64(tsBuf[:], uint64(timestampUs))
+	trailer = appendTLV(trailer, tagTimestampUs, tsBuf[:])
+	trailerLen := byte(len(trailer) + envelopeSize)
+	trailer = appendEnvelope(trailer, trailerLen)
+	return trailer
+}
+
+func TestStripTrailer(t *testing.T) {
+	fullTrailerSize := 21 // (1+1+8) + (1+1+4) + 5
+	tsOnlyTrailerSize := 15 // (1+1+8) + 5
+
+	tests := []struct {
+		name      string
+		payload   []byte
+		marker    bool
+		wantStrip int
+	}{
+		{
+			name:      "marker set with full trailer (timestamp + frame_id)",
+			payload:   makePayloadWithTrailer(20, 1700000000000000, 42),
+			marker:    true,
+			wantStrip: fullTrailerSize,
+		},
+		{
+			name: "marker set with timestamp-only trailer",
+			payload: func() []byte {
+				video := make([]byte, 20)
+				return append(video, makeTimestampOnlyTrailer(1700000000000000)...)
+			}(),
+			marker:    true,
+			wantStrip: tsOnlyTrailerSize,
+		},
+		{
+			name:      "marker not set with valid trailer",
+			payload:   makePayloadWithTrailer(20, 1700000000000000, 42),
+			marker:    false,
+			wantStrip: 0,
+		},
+		{
+			name:      "marker set without magic",
+			payload:   make([]byte, 32),
+			marker:    true,
+			wantStrip: 0,
+		},
+		{
+			name:      "marker set but payload too short for envelope",
+			payload:   []byte{0x4C, 0x4B, 0x54, 0x53},
+			marker:    true,
+			wantStrip: 0,
+		},
+		{
+			name: "marker set with partial magic mismatch",
+			payload: func() []byte {
+				p := makePayloadWithTrailer(20, 1700000000000000, 42)
+				p[len(p)-1] = 'x'
+				return p
+			}(),
+			marker:    true,
+			wantStrip: 0,
+		},
+		{
+			name: "trailer_len exceeds payload length",
+			payload: func() []byte {
+				var trailer []byte
+				var tsBuf [8]byte
+				binary.BigEndian.PutUint64(tsBuf[:], 42)
+				trailer = appendTLV(trailer, tagTimestampUs, tsBuf[:])
+				trailer = appendEnvelope(trailer, 200)
+				return trailer
+			}(),
+			marker:    true,
+			wantStrip: 0,
+		},
+		{
+			name: "trailer_len smaller than envelope (invalid)",
+			payload: func() []byte {
+				video := make([]byte, 20)
+				var trailer []byte
+				var tsBuf [8]byte
+				binary.BigEndian.PutUint64(tsBuf[:], 42)
+				trailer = appendTLV(trailer, tagTimestampUs, tsBuf[:])
+				trailer = appendEnvelope(trailer, 3)
+				return append(video, trailer...)
+			}(),
+			marker:    true,
+			wantStrip: 0,
+		},
+		{
+			name:      "exactly envelope-only trailer",
+			payload:   appendEnvelope(nil, byte(envelopeSize)),
+			marker:    true,
+			wantStrip: envelopeSize,
+		},
+		{
+			name:      "empty payload",
+			payload:   []byte{},
+			marker:    true,
+			wantStrip: 0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := StripTrailer(tt.payload, tt.marker)
+			if got != tt.wantStrip {
+				t.Errorf("StripTrailer() = %d, want %d", got, tt.wantStrip)
+			}
+		})
+	}
+}

pkg/sfu/playoutdelay.go

@@ -0,0 +1,205 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sfu
+
+import (
+	"sync"
+	"time"
+
+	pd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/playoutdelay"
+	"github.com/livekit/livekit-server/pkg/sfu/rtpstats"
+	"github.com/livekit/protocol/logger"
+	"go.uber.org/atomic"
+	"go.uber.org/zap/zapcore"
+)
+
+const (
+	jitterMultiToDelay      = 10
+	targetDelayLogThreshold = 500
+
+	// limit max delay change to make it smoother for a/v sync
+	maxDelayChangePerSec = 80
+)
+
+// ----------------------------------------------------
+
+type PlayoutDelayState int32
+
+const (
+	PlayoutDelayStateChanged PlayoutDelayState = iota
+	PlayoutDelaySending
+	PlayoutDelayAcked
+)
+
+func (s PlayoutDelayState) String() string {
+	switch s {
+	case PlayoutDelayStateChanged:
+		return "StateChanged"
+	case PlayoutDelaySending:
+		return "Sending"
+	case PlayoutDelayAcked:
+		return "Acked"
+	}
+	return "Unknown"
+}
+
+// ----------------------------------------------------
+
+type PlayoutDelayControllerState struct {
+	SenderSnapshotID uint32
+}
+
+func (p PlayoutDelayControllerState) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	e.AddUint32("SenderSnapshotID", p.SenderSnapshotID)
+	return nil
+}
+
+// ----------------------------------------------------
+
+type PlayoutDelayController struct {
+	lock               sync.Mutex
+	state              atomic.Int32
+	minDelay, maxDelay uint32
+	currentDelay       uint32
+	extBytes           atomic.Value //[]byte
+	sendingAtSeq       uint16
+	sendingAtTime      time.Time
+	logger             logger.Logger
+	rtpStats           *rtpstats.RTPStatsSender
+	senderSnapshotID   uint32
+
+	highDelayCount atomic.Uint32
+}
+
+func NewPlayoutDelayController(minDelay, maxDelay uint32, logger logger.Logger, rtpStats *rtpstats.RTPStatsSender) (*PlayoutDelayController, error) {
+	if maxDelay == 0 && minDelay > 0 {
+		maxDelay = pd.MaxPlayoutDelayDefault
+	}
+	if maxDelay > pd.PlayoutDelayMaxValue {
+		maxDelay = pd.PlayoutDelayMaxValue
+	}
+	c := &PlayoutDelayController{
+		currentDelay:     minDelay,
+		minDelay:         minDelay,
+		maxDelay:         maxDelay,
+		logger:           logger,
+		rtpStats:         rtpStats,
+		senderSnapshotID: rtpStats.NewSenderSnapshotId(),
+	}
+
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return c, c.createExtDataLocked()
+}
+
+func (c *PlayoutDelayController) GetState() PlayoutDelayControllerState {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	return PlayoutDelayControllerState{
+		SenderSnapshotID: c.senderSnapshotID,
+	}
+}
+
+func (c *PlayoutDelayController) SeedState(pdcs PlayoutDelayControllerState) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	c.senderSnapshotID = pdcs.SenderSnapshotID
+}
+
+func (c *PlayoutDelayController) SetJitter(jitter uint32) {
+	c.lock.Lock()
+	deltaInfoSender, _ := c.rtpStats.DeltaInfoSender(c.senderSnapshotID)
+	var nackPercent uint32
+	if deltaInfoSender != nil && deltaInfoSender.Packets > 0 {
+		nackPercent = deltaInfoSender.Nacks * 100 / deltaInfoSender.Packets
+	}
+
+	targetDelay := jitter * jitterMultiToDelay
+	if nackPercent > 60 {
+		targetDelay += (nackPercent - 60) * 2
+	}
+
+	elapsed := time.Since(c.sendingAtTime)
+	delayChangeLimit := uint32(maxDelayChangePerSec * elapsed.Seconds())
+	if delayChangeLimit > maxDelayChangePerSec {
+		delayChangeLimit = maxDelayChangePerSec
+	}
+
+	if targetDelay > c.currentDelay+delayChangeLimit {
+		targetDelay = c.currentDelay + delayChangeLimit
+	} else if c.currentDelay > targetDelay+delayChangeLimit {
+		targetDelay = c.currentDelay - delayChangeLimit
+	}
+	if targetDelay < c.minDelay {
+		targetDelay = c.minDelay
+	}
+	if targetDelay > c.maxDelay {
+		targetDelay = c.maxDelay
+	}
+	if c.currentDelay == targetDelay {
+		c.lock.Unlock()
+		return
+	}
+	if targetDelay > targetDelayLogThreshold {
+		if c.highDelayCount.Add(1)%100 == 1 {
+			c.logger.Infow("high playout delay", "target", targetDelay, "jitter", jitter, "nackPercent", nackPercent, "current", c.currentDelay)
+		}
+	}
+	c.currentDelay = targetDelay
+	c.createExtDataLocked()
+	c.lock.Unlock()
+}
+
+func (c *PlayoutDelayController) OnSeqAcked(seq uint16) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	if PlayoutDelayState(c.state.Load()) == PlayoutDelaySending && (seq-c.sendingAtSeq) < 0x8000 {
+		c.state.Store(int32(PlayoutDelayAcked))
+	}
+}
+
+func (c *PlayoutDelayController) GetDelayExtension(seq uint16) []byte {
+	switch PlayoutDelayState(c.state.Load()) {
+	case PlayoutDelayStateChanged:
+		c.lock.Lock()
+		c.state.Store(int32(PlayoutDelaySending))
+		c.sendingAtSeq = seq
+		c.sendingAtTime = time.Now()
+		c.lock.Unlock()
+		return c.extBytes.Load().([]byte)
+	case PlayoutDelaySending:
+		return c.extBytes.Load().([]byte)
+	case PlayoutDelayAcked:
+		return nil
+	}
+	return nil
+}
+
+func (c *PlayoutDelayController) createExtDataLocked() error {
+	delay := pd.PlayoutDelayFromValue(
+		uint16(c.currentDelay),
+		uint16(c.maxDelay),
+	)
+	b, err := delay.Marshal()
+	if err == nil {
+		c.extBytes.Store(b)
+		c.state.Store(int32(PlayoutDelayStateChanged))
+	} else {
+		c.logger.Errorw("failed to marshal playout delay", err, "playoutDelay", delay)
+	}
+	return err
+}

pkg/sfu/playoutdelay_test.go

@@ -0,0 +1,81 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sfu
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	pd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/playoutdelay"
+	"github.com/livekit/livekit-server/pkg/sfu/rtpstats"
+	"github.com/livekit/protocol/logger"
+)
+
+func TestPlayoutDelay(t *testing.T) {
+	stats := rtpstats.NewRTPStatsSender(rtpstats.RTPStatsParams{}, 128)
+	stats.SetClockRate(90000)
+
+	c, err := NewPlayoutDelayController(100, 120, logger.GetLogger(), stats)
+	require.NoError(t, err)
+
+	ext := c.GetDelayExtension(100)
+	playoutDelayEqual(t, ext, 100, 120)
+
+	ext = c.GetDelayExtension(105)
+	playoutDelayEqual(t, ext, 100, 120)
+
+	// seq acked before delay changed
+	c.OnSeqAcked(65534)
+	ext = c.GetDelayExtension(105)
+	playoutDelayEqual(t, ext, 100, 120)
+
+	c.OnSeqAcked(90)
+	ext = c.GetDelayExtension(105)
+	playoutDelayEqual(t, ext, 100, 120)
+
+	// seq acked, no extension sent for new packet
+	c.OnSeqAcked(103)
+	ext = c.GetDelayExtension(106)
+	require.Nil(t, ext)
+
+	// delay on change(can't go below min), no extension sent
+	c.SetJitter(0)
+	ext = c.GetDelayExtension(107)
+	require.Nil(t, ext)
+
+	// delay changed, generate new extension to send
+	time.Sleep(200 * time.Millisecond)
+	c.SetJitter(50)
+	t.Log(c.currentDelay, c.state.Load())
+	ext = c.GetDelayExtension(108)
+	var delay pd.PlayOutDelay
+	require.NoError(t, delay.Unmarshal(ext))
+	require.Greater(t, delay.Min, uint16(100))
+
+	// can't go above max
+	time.Sleep(200 * time.Millisecond)
+	c.SetJitter(10000)
+	ext = c.GetDelayExtension(109)
+	playoutDelayEqual(t, ext, 120, 120)
+}
+
+func playoutDelayEqual(t *testing.T, data []byte, min, max uint16) {
+	var delay pd.PlayOutDelay
+	require.NoError(t, delay.Unmarshal(data))
+	require.Equal(t, min, delay.Min)
+	require.Equal(t, max, delay.Max)
+}

pkg/sfu/receiver.go

@@ -0,0 +1,370 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sfu
+
+import (
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/pion/rtcp"
+	"github.com/pion/webrtc/v4"
+
+	"github.com/livekit/protocol/codecs/mime"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/sfu/connectionquality"
+	"github.com/livekit/livekit-server/pkg/sfu/rtpstats"
+)
+
+var _ TrackReceiver = (*WebRTCReceiver)(nil)
+
+// WebRTCReceiver receives a media track
+type WebRTCReceiver struct {
+	*ReceiverBase
+
+	receiver       *webrtc.RTPReceiver
+	onCloseHandler func()
+
+	onRTCP func([]rtcp.Packet)
+
+	upTracksMu sync.Mutex
+	upTracks   [buffer.DefaultMaxLayerSpatial + 1]TrackRemote
+
+	connectionStats *connectionquality.ConnectionStats
+	onStatsUpdate   func(w *WebRTCReceiver, stat *livekit.AnalyticsStat)
+}
+
+type ReceiverOpts func(w *WebRTCReceiver) *WebRTCReceiver
+
+// WithPliThrottleConfig indicates minimum time(ms) between sending PLIs
+func WithPliThrottleConfig(pliThrottleConfig PLIThrottleConfig) ReceiverOpts {
+	return func(w *WebRTCReceiver) *WebRTCReceiver {
+		w.ReceiverBase.SetPLIThrottleConfig(pliThrottleConfig)
+		return w
+	}
+}
+
+// WithAudioConfig sets up parameters for active speaker detection
+func WithAudioConfig(audioConfig AudioConfig) ReceiverOpts {
+	return func(w *WebRTCReceiver) *WebRTCReceiver {
+		w.ReceiverBase.SetAudioConfig(audioConfig)
+		return w
+	}
+}
+
+func WithEnableRTPStreamRestartDetection(enable bool) ReceiverOpts {
+	return func(w *WebRTCReceiver) *WebRTCReceiver {
+		w.ReceiverBase.SetEnableRTPStreamRestartDetection(enable)
+		return w
+	}
+}
+
+// WithLoadBalanceThreshold enables parallelization of packet writes when downTracks exceeds threshold
+// Value should be between 3 and 150.
+// For a server handling a few large rooms, use a smaller value (required to handle very large (250+ participant) rooms).
+// For a server handling many small rooms, use a larger value or disable.
+// Set to 0 (disabled) by default.
+func WithLoadBalanceThreshold(downTracks int) ReceiverOpts {
+	return func(w *WebRTCReceiver) *WebRTCReceiver {
+		w.ReceiverBase.SetLBThreshold(downTracks)
+		return w
+	}
+}
+
+func WithForwardStats(forwardStats *ForwardStats) ReceiverOpts {
+	return func(w *WebRTCReceiver) *WebRTCReceiver {
+		w.ReceiverBase.SetForwardStats(forwardStats)
+		return w
+	}
+}
+
+// NewWebRTCReceiver creates a new webrtc track receiver
+func NewWebRTCReceiver(
+	receiver *webrtc.RTPReceiver,
+	track TrackRemote,
+	trackInfo *livekit.TrackInfo,
+	logger logger.Logger,
+	onRTCP func([]rtcp.Packet),
+	streamTrackerManagerConfig StreamTrackerManagerConfig,
+	opts ...ReceiverOpts,
+) *WebRTCReceiver {
+	w := &WebRTCReceiver{
+		receiver: receiver,
+		onRTCP:   onRTCP,
+	}
+
+	w.ReceiverBase = NewReceiverBase(
+		ReceiverBaseParams{
+			TrackID:                      livekit.TrackID(track.ID()),
+			StreamID:                     track.StreamID(),
+			Kind:                         track.Kind(),
+			Codec:                        track.Codec(),
+			HeaderExtensions:             receiver.GetParameters().HeaderExtensions,
+			Logger:                       logger,
+			StreamTrackerManagerConfig:   streamTrackerManagerConfig,
+			StreamTrackerManagerListener: w,
+			IsSelfClosing:                true,
+			OnClosed:                     w.onClosed,
+		},
+		trackInfo,
+		ReceiverCodecStateNormal,
+	)
+
+	for _, opt := range opts {
+		w = opt(w)
+	}
+
+	w.connectionStats = connectionquality.NewConnectionStats(connectionquality.ConnectionStatsParams{
+		ReceiverProvider: w,
+		Logger:           logger.WithValues("direction", "up"),
+	})
+	w.connectionStats.OnStatsUpdate(func(_cs *connectionquality.ConnectionStats, stat *livekit.AnalyticsStat) {
+		if w.onStatsUpdate != nil {
+			w.onStatsUpdate(w, stat)
+		}
+	})
+	codec := track.Codec()
+	w.connectionStats.Start(
+		mime.NormalizeMimeType(codec.MimeType),
+		// TODO: technically not correct to declare FEC on when RED. Need the primary codec's fmtp line to check.
+		mime.IsMimeTypeStringRED(codec.MimeType) || strings.Contains(strings.ToLower(codec.SDPFmtpLine), "useinbandfec=1"),
+	)
+
+	w.UpdateTrackInfo(trackInfo)
+
+	return w
+}
+
+func (w *WebRTCReceiver) OnStatsUpdate(fn func(w *WebRTCReceiver, stat *livekit.AnalyticsStat)) {
+	w.onStatsUpdate = fn
+}
+
+func (w *WebRTCReceiver) GetConnectionScoreAndQuality() (float32, livekit.ConnectionQuality) {
+	return w.connectionStats.GetScoreAndQuality()
+}
+
+func (w *WebRTCReceiver) ssrc(layer int) uint32 {
+	w.upTracksMu.Lock()
+	defer w.upTracksMu.Unlock()
+
+	if track := w.upTracks[layer]; track != nil {
+		return uint32(track.SSRC())
+	}
+	return 0
+}
+
+func (w *WebRTCReceiver) AddUpTrack(track TrackRemote, buff *buffer.Buffer) error {
+	if w.isClosed.Load() {
+		return ErrReceiverClosed
+	}
+
+	layer := int32(0)
+	if w.Kind() == webrtc.RTPCodecTypeVideo && w.videoLayerMode != livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM {
+		layer = buffer.GetSpatialLayerForRid(w.Mime(), track.RID(), w.ReceiverBase.TrackInfo())
+	}
+	if layer < 0 {
+		w.ReceiverBase.Logger().Warnw(
+			"invalid layer", nil,
+			"rid", track.RID(),
+			"trackInfo", logger.Proto(w.ReceiverBase.TrackInfo()),
+		)
+		return ErrInvalidLayer
+	}
+
+	w.upTracksMu.Lock()
+	if w.upTracks[layer] != nil {
+		w.upTracksMu.Unlock()
+		return ErrDuplicateLayer
+	}
+	w.upTracks[layer] = track
+	w.upTracksMu.Unlock()
+
+	w.ReceiverBase.AddBuffer(buff, layer)
+	buff.OnRtcpFeedback(w.sendRTCP)
+	w.ReceiverBase.StartBuffer(buff, layer)
+	return nil
+}
+
+func (w *WebRTCReceiver) NumUpTracks() int {
+	numUpTracks := 0
+
+	w.upTracksMu.Lock()
+	for _, track := range w.upTracks {
+		if track != nil {
+			numUpTracks++
+		}
+	}
+	w.upTracksMu.Unlock()
+
+	return numUpTracks
+}
+
+func (w *WebRTCReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) {
+	w.ReceiverBase.UpdateTrackInfo(ti)
+	w.connectionStats.UpdateMute(ti.GetMuted())
+}
+
+func (w *WebRTCReceiver) notifyMaxExpectedLayer(layer int32) {
+	ti := w.TrackInfo()
+	if ti == nil {
+		return
+	}
+
+	if w.Kind() == webrtc.RTPCodecTypeAudio || ti.Source == livekit.TrackSource_SCREEN_SHARE {
+		// screen share tracks have highly variable bitrate, do not use bit rate based quality for those
+		return
+	}
+
+	expectedBitrate := int64(0)
+	for _, vl := range buffer.GetVideoLayersForMimeType(w.Mime(), ti) {
+		if vl.SpatialLayer <= layer {
+			expectedBitrate += int64(vl.Bitrate)
+		}
+	}
+
+	w.connectionStats.AddBitrateTransition(expectedBitrate)
+}
+
+func (w *WebRTCReceiver) SetMaxExpectedSpatialLayer(layer int32) {
+	w.ReceiverBase.SetMaxExpectedSpatialLayer(layer)
+
+	w.notifyMaxExpectedLayer(layer)
+
+	if layer == buffer.InvalidLayerSpatial {
+		w.connectionStats.UpdateLayerMute(true)
+	} else {
+		w.connectionStats.UpdateLayerMute(false)
+		w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired())
+	}
+}
+
+// StreamTrackerManagerListener.OnAvailableLayersChanged
+func (w *WebRTCReceiver) OnAvailableLayersChanged() {
+	w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired())
+}
+
+// StreamTrackerManagerListener.OnBitrateAvailabilityChanged
+func (w *WebRTCReceiver) OnBitrateAvailabilityChanged() {
+}
+
+// StreamTrackerManagerListener.OnMaxPublishedLayerChanged
+func (w *WebRTCReceiver) OnMaxPublishedLayerChanged(maxPublishedLayer int32) {
+	w.notifyMaxExpectedLayer(maxPublishedLayer)
+	w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired())
+}
+
+// StreamTrackerManagerListener.OnMaxTemporalLayerSeenChanged
+func (w *WebRTCReceiver) OnMaxTemporalLayerSeenChanged(maxTemporalLayerSeen int32) {
+	w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired())
+}
+
+// StreamTrackerManagerListener.OnMaxAvailableLayerChanged
+func (w *WebRTCReceiver) OnMaxAvailableLayerChanged(maxAvailableLayer int32) {
+}
+
+// StreamTrackerManagerListener.OnBitrateReport
+func (w *WebRTCReceiver) OnBitrateReport(availableLayers []int32, bitrates Bitrates) {
+	w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired())
+}
+
+// OnCloseHandler method to be called on remote track removed
+func (w *WebRTCReceiver) OnCloseHandler(fn func()) {
+	w.onCloseHandler = fn
+}
+
+func (w *WebRTCReceiver) sendRTCP(packets []rtcp.Packet) {
+	if packets == nil || w.isClosed.Load() {
+		return
+	}
+
+	if w.onRTCP != nil {
+		w.onRTCP(packets)
+	}
+}
+
+func (w *WebRTCReceiver) GetDeltaStats() map[uint32]*buffer.StreamStatsWithLayers {
+	buffers := w.ReceiverBase.GetAllBuffers()
+	deltaStats := make(map[uint32]*buffer.StreamStatsWithLayers, len(buffers))
+	for layer, buff := range buffers {
+		if buff == nil {
+			continue
+		}
+
+		sswl := buff.GetDeltaStats()
+		if sswl == nil {
+			continue
+		}
+
+		// patch buffer stats with correct layer
+		patched := make(map[int32]*rtpstats.RTPDeltaInfo, 1)
+		patched[int32(layer)] = sswl.Layers[0]
+		sswl.Layers = patched
+
+		deltaStats[w.ssrc(layer)] = sswl
+	}
+
+	return deltaStats
+}
+
+func (w *WebRTCReceiver) GetLastSenderReportTime() time.Time {
+	buffers := w.ReceiverBase.GetAllBuffers()
+	latestSRTime := time.Time{}
+	for _, buff := range buffers {
+		if buff == nil {
+			continue
+		}
+
+		srAt := buff.GetLastSenderReportTime()
+		if srAt.After(latestSRTime) {
+			latestSRTime = srAt
+		}
+	}
+
+	return latestSRTime
+}
+
+func (w *WebRTCReceiver) onClosed() {
+	w.connectionStats.Close()
+
+	if w.onCloseHandler != nil {
+		w.onCloseHandler()
+	}
+}
+
+func (w *WebRTCReceiver) DebugInfo() map[string]any {
+	info := w.ReceiverBase.DebugInfo()
+
+	w.upTracksMu.Lock()
+	upTrackInfo := make([]map[string]any, 0, len(w.upTracks))
+	for layer, ut := range w.upTracks {
+		if ut != nil {
+			upTrackInfo = append(upTrackInfo, map[string]any{
+				"Layer": layer,
+				"SSRC":  ut.SSRC(),
+				"Msid":  ut.Msid(),
+				"RID":   ut.RID(),
+			})
+		}
+	}
+	w.upTracksMu.Unlock()
+	info["UpTracks"] = upTrackInfo
+
+	return info
+}
+
+// -----------------------------------------------------------

pkg/sfu/receiver_test.go

@@ -0,0 +1,213 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sfu
+
+import (
+	"fmt"
+	"hash/fnv"
+	"math/rand"
+	"runtime"
+	"sync"
+	"testing"
+
+	"github.com/gammazero/workerpool"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/atomic"
+)
+
+func TestWebRTCReceiver_OnCloseHandler(t *testing.T) {
+	type args struct {
+		fn func()
+	}
+	tests := []struct {
+		name string
+		args args
+	}{
+		{
+			name: "Must set on close handler function",
+			args: args{
+				fn: func() {},
+			},
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			w := &WebRTCReceiver{}
+			w.OnCloseHandler(tt.args.fn)
+			assert.NotNil(t, w.onCloseHandler)
+		})
+	}
+}
+
+func BenchmarkWriteRTP(b *testing.B) {
+	cases := []int{1, 2, 5, 10, 100, 250, 500}
+	workers := runtime.NumCPU()
+	wp := workerpool.New(workers)
+	for _, c := range cases {
+		// fills each bucket with a max of 50, i.e. []int{50, 50} for c=100
+		fill := make([]int, 0)
+		for i := 50; ; i += 50 {
+			if i > c {
+				fill = append(fill, c%50)
+				break
+			}
+
+			fill = append(fill, 50)
+			if i == c {
+				break
+			}
+		}
+
+		// splits c into numCPU buckets, i.e. []int{9, 9, 9, 9, 8, 8, 8, 8, 8, 8, 8, 8} for 12 cpus and c=100
+		split := make([]int, workers)
+		for i := range split {
+			split[i] = c / workers
+		}
+		for i := 0; i < c%workers; i++ {
+			split[i]++
+		}
+
+		b.Run(fmt.Sprintf("%d-Downtracks/Control", c), func(b *testing.B) {
+			benchmarkNoPool(b, c)
+		})
+		b.Run(fmt.Sprintf("%d-Downtracks/Pool(Fill)", c), func(b *testing.B) {
+			benchmarkPool(b, wp, fill)
+		})
+		b.Run(fmt.Sprintf("%d-Downtracks/Pool(Hash)", c), func(b *testing.B) {
+			benchmarkPool(b, wp, split)
+		})
+		b.Run(fmt.Sprintf("%d-Downtracks/Goroutines", c), func(b *testing.B) {
+			benchmarkGoroutine(b, split)
+		})
+		b.Run(fmt.Sprintf("%d-Downtracks/LoadBalanced", c), func(b *testing.B) {
+			benchmarkLoadBalanced(b, workers, 2, c)
+		})
+		b.Run(fmt.Sprintf("%d-Downtracks/LBPool", c), func(b *testing.B) {
+			benchmarkLoadBalancedPool(b, wp, workers, 2, c)
+		})
+	}
+}
+
+func benchmarkNoPool(b *testing.B, downTracks int) {
+	for b.Loop() {
+		for range downTracks {
+			writeRTP()
+		}
+	}
+}
+
+func benchmarkPool(b *testing.B, wp *workerpool.WorkerPool, buckets []int) {
+	for b.Loop() {
+		var wg sync.WaitGroup
+		for j := range buckets {
+			downTracks := buckets[j]
+			if downTracks == 0 {
+				continue
+			}
+			wg.Add(1)
+			wp.Submit(func() {
+				defer wg.Done()
+				for dt := 0; dt < downTracks; dt++ {
+					writeRTP()
+				}
+			})
+		}
+		wg.Wait()
+	}
+}
+
+func benchmarkGoroutine(b *testing.B, buckets []int) {
+	for b.Loop() {
+		var wg sync.WaitGroup
+		for j := range buckets {
+			downTracks := buckets[j]
+			if downTracks == 0 {
+				continue
+			}
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for dt := 0; dt < downTracks; dt++ {
+					writeRTP()
+				}
+			}()
+		}
+		wg.Wait()
+	}
+}
+
+func benchmarkLoadBalanced(b *testing.B, numProcs, step, downTracks int) {
+	for b.Loop() {
+		start := atomic.NewUint64(0)
+		step := uint64(step)
+		end := uint64(downTracks)
+
+		var wg sync.WaitGroup
+		wg.Add(numProcs)
+		for p := 0; p < numProcs; p++ {
+			go func() {
+				defer wg.Done()
+				for {
+					n := start.Add(step)
+					if n >= end+step {
+						return
+					}
+
+					for i := n - step; i < n && i < end; i++ {
+						writeRTP()
+					}
+				}
+			}()
+		}
+		wg.Wait()
+	}
+}
+
+func benchmarkLoadBalancedPool(b *testing.B, wp *workerpool.WorkerPool, numProcs, step, downTracks int) {
+	for b.Loop() {
+		start := atomic.NewUint64(0)
+		step := uint64(step)
+		end := uint64(downTracks)
+
+		var wg sync.WaitGroup
+		wg.Add(numProcs)
+		for p := 0; p < numProcs; p++ {
+			wp.Submit(func() {
+				defer wg.Done()
+				for {
+					n := start.Add(step)
+					if n >= end+step {
+						return
+					}
+
+					for i := n - step; i < n && i < end; i++ {
+						writeRTP()
+					}
+				}
+			})
+		}
+		wg.Wait()
+	}
+}
+
+func writeRTP() {
+	s := []byte("simulate some work")
+	stop := 1900 + rand.Intn(200)
+	for j := 0; j < stop; j++ {
+		h := fnv.New128()
+		s = h.Sum(s)
+	}
+}

pkg/sfu/redprimaryreceiver.go

@@ -0,0 +1,386 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sfu
+
+import (
+	"encoding/binary"
+	"errors"
+
+	"go.uber.org/atomic"
+
+	"github.com/pion/rtp"
+	"github.com/pion/webrtc/v4"
+
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/sfu/utils"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+)
+
+var _ REDTransformer = (*RedPrimaryReceiver)(nil)
+
+var (
+	ErrIncompleteRedHeader = errors.New("incomplete red block header")
+	ErrIncompleteRedBlock  = errors.New("incomplete red block payload")
+)
+
+type RedPrimaryReceiver struct {
+	TrackReceiver
+	downTrackSpreader *utils.DownTrackSpreader[TrackSender]
+	logger            logger.Logger
+	closed            atomic.Bool
+	redPT             uint8
+
+	firstPktReceived                  bool
+	lastSeq                           uint16
+	highestForwardedExtSequenceNumber uint64
+	highestForwardedExtTimestamp      uint64
+
+	// bitset for upstream packet receive history [lastSeq-8, lastSeq-1], bit 1 represents packet received
+	pktHistory byte
+}
+
+func NewRedPrimaryReceiver(receiver TrackReceiver, dsp utils.DownTrackSpreaderParams) REDTransformer {
+	return &RedPrimaryReceiver{
+		TrackReceiver:     receiver,
+		downTrackSpreader: utils.NewDownTrackSpreader[TrackSender](dsp),
+		logger:            dsp.Logger,
+		redPT:             uint8(receiver.Codec().PayloadType),
+	}
+}
+
+func (r *RedPrimaryReceiver) SetLBThreshold(lbThreshold int) {
+	r.downTrackSpreader.SetThreshold(lbThreshold)
+}
+
+func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int32 {
+	// extract primary payload from RED and forward to downtracks
+	if r.downTrackSpreader.DownTrackCount() == 0 {
+		return 0
+	}
+
+	if pkt.Packet.PayloadType != r.redPT {
+		// forward non-red packet directly
+		var writeCount atomic.Int32
+		r.downTrackSpreader.Broadcast(func(dt TrackSender) {
+			writeCount.Add(dt.WriteRTP(pkt, spatialLayer))
+		})
+		return writeCount.Load()
+	}
+
+	pkts, err := r.getSendPktsFromRed(pkt.Packet)
+	if err != nil {
+		r.logger.Errorw("get encoding for red failed", err, "payloadtype", pkt.Packet.PayloadType)
+		return 0
+	}
+
+	var writeCount atomic.Int32
+	for i, sendPkt := range pkts {
+		pPkt := *pkt
+		if i != len(pkts)-1 {
+			// patch extended sequence number and time stamp for all but the last packet,
+			// last packet is the primary payload
+			pPkt.ExtSequenceNumber -= uint64(pkts[len(pkts)-1].SequenceNumber - pkts[i].SequenceNumber)
+			pPkt.ExtTimestamp -= uint64(pkts[len(pkts)-1].Timestamp - pkts[i].Timestamp)
+		}
+		pPkt.Packet = sendPkt
+
+		if r.highestForwardedExtTimestamp != 0 {
+			if (pPkt.ExtSequenceNumber > r.highestForwardedExtSequenceNumber && pPkt.ExtTimestamp < r.highestForwardedExtTimestamp) ||
+				(pPkt.ExtSequenceNumber < r.highestForwardedExtSequenceNumber && pPkt.ExtTimestamp > r.highestForwardedExtTimestamp) {
+				r.logger.Warnw(
+					"sequence number OR timestamp inversion, dropping", nil,
+					"numPackets", len(pkts),
+					"primaryIncomingSN", pkt.Packet.Header.SequenceNumber,
+					"primaryIncomingTS", pkt.Packet.Header.Timestamp,
+					"primaryExtractedSN", pkts[len(pkts)-1].SequenceNumber,
+					"primaryExtractedTS", pkts[len(pkts)-1].Timestamp,
+					"primaryESN", pkt.ExtSequenceNumber,
+					"primaryETS", pkt.ExtTimestamp,
+					"packetIndex", i,
+					"packetExtractedSN", pkts[i].SequenceNumber,
+					"packetESN", pPkt.ExtSequenceNumber,
+					"packetExtractedTS", pkts[i].Timestamp,
+					"packetETS", pPkt.ExtTimestamp,
+					"pktHistory", r.pktHistory,
+					"redHeader", pkt.Packet.Payload[:min(len(pkt.Packet.Payload), 10)],
+					"payloadSize", len(pkt.Packet.Payload),
+					"lastSeq", r.lastSeq,
+					"highestFowardedExtSequenceNumber", r.highestForwardedExtSequenceNumber,
+					"highestFowardedExtTimestamp", r.highestForwardedExtTimestamp,
+				)
+				continue // drop the packet which causes the inversion
+			}
+		}
+		r.highestForwardedExtSequenceNumber = max(r.highestForwardedExtSequenceNumber, pPkt.ExtSequenceNumber)
+		r.highestForwardedExtTimestamp = max(r.highestForwardedExtTimestamp, pPkt.ExtTimestamp)
+
+		// not modify the ExtPacket.RawPacket here for performance since it is not used by the DownTrack,
+		// otherwise it should be set to the correct value (marshal the primary rtp packet)
+		r.downTrackSpreader.Broadcast(func(dt TrackSender) {
+			writeCount.Add(dt.WriteRTP(&pPkt, spatialLayer))
+		})
+	}
+	return writeCount.Load()
+}
+
+func (r *RedPrimaryReceiver) ForwardRTCPSenderReport(
+	payloadType webrtc.PayloadType,
+	layer int32,
+	publisherSRData *livekit.RTCPSenderReportState,
+) {
+	r.downTrackSpreader.Broadcast(func(dt TrackSender) {
+		_ = dt.HandleRTCPSenderReportData(payloadType, layer, publisherSRData)
+	})
+}
+
+func (r *RedPrimaryReceiver) AddDownTrack(track TrackSender) error {
+	if r.closed.Load() {
+		return ErrReceiverClosed
+	}
+
+	if r.downTrackSpreader.HasDownTrack(track.SubscriberID()) {
+		r.logger.Infow("subscriberID already exists, replacing downtrack", "subscriberID", track.SubscriberID())
+	}
+
+	r.downTrackSpreader.Store(track)
+	r.logger.Debugw("red primary receiver downtrack added", "subscriberID", track.SubscriberID())
+	return nil
+}
+
+func (r *RedPrimaryReceiver) DeleteDownTrack(subscriberID livekit.ParticipantID) {
+	if r.closed.Load() {
+		return
+	}
+
+	r.downTrackSpreader.Free(subscriberID)
+	r.logger.Debugw("red primary receiver downtrack deleted", "subscriberID", subscriberID)
+}
+
+func (r *RedPrimaryReceiver) GetDownTracks() []TrackSender {
+	return r.downTrackSpreader.GetDownTracks()
+}
+
+func (r *RedPrimaryReceiver) ResyncDownTracks() {
+	r.downTrackSpreader.Broadcast(func(dt TrackSender) {
+		dt.Resync()
+	})
+}
+
+func (r *RedPrimaryReceiver) OnStreamRestart() {
+	r.downTrackSpreader.Broadcast(func(dt TrackSender) {
+		dt.ReceiverRestart(r)
+	})
+}
+
+func (r *RedPrimaryReceiver) IsClosed() bool {
+	return r.closed.Load()
+}
+
+func (r *RedPrimaryReceiver) CanClose() bool {
+	return r.closed.Load() || r.downTrackSpreader.DownTrackCount() == 0
+}
+
+func (r *RedPrimaryReceiver) Close() {
+	r.closed.Store(true)
+	closeTrackSenders(r.downTrackSpreader.ResetAndGetDownTracks())
+}
+
+func (r *RedPrimaryReceiver) ReadRTP(buf []byte, layer uint8, esn uint64) (int, error) {
+	n, err := r.TrackReceiver.ReadRTP(buf, layer, esn)
+	if err != nil {
+		return n, err
+	}
+
+	var pkt rtp.Packet
+	pkt.Unmarshal(buf[:n])
+	payload, err := extractPrimaryEncodingForRED(pkt.Payload)
+	if err != nil {
+		return 0, err
+	}
+	pkt.Payload = payload
+
+	return pkt.MarshalTo(buf)
+}
+
+func (r *RedPrimaryReceiver) getSendPktsFromRed(rtp *rtp.Packet) ([]*rtp.Packet, error) {
+	var needRecover bool
+	if !r.firstPktReceived {
+		r.lastSeq = rtp.SequenceNumber
+		r.pktHistory = 0
+		r.firstPktReceived = true
+	} else {
+		diff := rtp.SequenceNumber - r.lastSeq
+		switch {
+		case diff == 0: // duplicate
+			break
+
+		case diff > 0x8000: // unorder
+			// in history
+			if 65535-diff < 8 {
+				r.pktHistory |= 1 << (65535 - diff)
+				needRecover = true
+			}
+
+		case diff > 8: // long jump
+			r.lastSeq = rtp.SequenceNumber
+			r.pktHistory = 0
+			needRecover = true
+
+		default:
+			r.lastSeq = rtp.SequenceNumber
+			r.pktHistory = (r.pktHistory << byte(diff)) | 1<<(diff-1)
+			needRecover = true
+		}
+	}
+
+	var recoverBits byte
+	if needRecover {
+		bitIndex := r.lastSeq - rtp.SequenceNumber
+		for i := range maxRedCount {
+			if bitIndex > 7 {
+				break
+			}
+			if r.pktHistory&byte(1<<bitIndex) == 0 {
+				recoverBits |= byte(1 << i)
+			}
+			bitIndex++
+		}
+	}
+
+	return extractPktsFromRed(rtp, recoverBits)
+}
+
+// ---------------------------------------
+
+type block struct {
+	tsOffset uint32
+	length   int
+	pt       uint8
+	primary  bool
+}
+
+func extractPktsFromRed(redPkt *rtp.Packet, recoverBits byte) ([]*rtp.Packet, error) {
+	payload := redPkt.Payload
+	var blocks []block
+	var blockLength int
+	for {
+		if len(payload) < 1 {
+			// illegal data, need at least one byte for primary encoding
+			return nil, ErrIncompleteRedHeader
+		}
+
+		if payload[0]&0x80 == 0 {
+			// last block is primary encoding data
+			pt := uint8(payload[0] & 0x7F)
+
+			blocks = append(blocks, block{pt: pt, primary: true})
+
+			payload = payload[1:]
+			break
+		} else {
+			if len(payload) < 4 {
+				// illegal data
+				return nil, ErrIncompleteRedHeader
+			}
+
+			blockHead := binary.BigEndian.Uint32(payload[0:])
+			length := int(blockHead & 0x03FF)
+			blockHead >>= 10
+			tsOffset := blockHead & 0x3FFF
+			blockHead >>= 14
+			pt := uint8(blockHead & 0x7F)
+
+			blocks = append(blocks, block{pt: pt, length: length, tsOffset: tsOffset})
+
+			blockLength += length
+			payload = payload[4:]
+		}
+	}
+
+	if len(payload) < blockLength {
+		return nil, ErrIncompleteRedBlock
+	}
+
+	pkts := make([]*rtp.Packet, 0, len(blocks))
+	for i, b := range blocks {
+		if b.primary {
+			header := redPkt.Header
+			header.PayloadType = b.pt
+			pkts = append(pkts, &rtp.Packet{Header: header, Payload: payload})
+			break
+		}
+
+		recoverIndex := len(blocks) - i - 1
+		if recoverIndex < 1 || recoverBits&(1<<(recoverIndex-1)) == 0 {
+			// skip past packet/block that does not need recovery
+			payload = payload[b.length:]
+			continue
+		}
+
+		// recover missing packet
+		header := redPkt.Header
+		header.SequenceNumber -= uint16(recoverIndex)
+		header.Timestamp -= b.tsOffset
+		header.PayloadType = b.pt
+		pkts = append(pkts, &rtp.Packet{Header: header, Payload: payload[:b.length]})
+
+		payload = payload[b.length:]
+	}
+
+	return pkts, nil
+}
+
+func extractPrimaryEncodingForRED(payload []byte) ([]byte, error) {
+
+	/* RED payload https://datatracker.ietf.org/doc/html/rfc2198#section-3
+		0                   1                    2                   3
+	    0 1 2 3 4 5 6 7 8 9 0 1 2 3  4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+	   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+	   |F|   block PT  |  timestamp offset         |   block length    |
+	   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+	   F: 1 bit First bit in header indicates whether another header block
+	       follows.  If 1 further header blocks follow, if 0 this is the
+	       last header block.
+	*/
+
+	var blockLength int
+	for {
+		if len(payload) < 1 {
+			// illegal data, need at least one byte for primary encoding
+			return nil, ErrIncompleteRedHeader
+		}
+
+		if payload[0]&0x80 == 0 {
+			// last block is primary encoding data
+			payload = payload[1:]
+			break
+		} else {
+			if len(payload) < 4 {
+				// illegal data
+				return nil, ErrIncompleteRedHeader
+			}
+
+			blockLength += int(binary.BigEndian.Uint16(payload[2:]) & 0x03FF)
+			payload = payload[4:]
+		}
+	}
+
+	if len(payload) < blockLength {
+		return nil, ErrIncompleteRedBlock
+	}
+
+	return payload[blockLength:], nil
+}

pkg/sfu/redreceiver.go

@@ -0,0 +1,254 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sfu
+
+import (
+	"encoding/binary"
+	"fmt"
+
+	"go.uber.org/atomic"
+
+	"github.com/pion/rtp"
+	"github.com/pion/webrtc/v4"
+
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/sfu/utils"
+	"github.com/livekit/mediatransportutil/pkg/bucket"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+)
+
+var _ REDTransformer = (*RedReceiver)(nil)
+
+const (
+	maxRedCount   = 2
+	mtuSize       = 1500
+	maxRedPayload = 1 << 10 // fit into 10 bits length field
+
+	// the RedReceiver is only for chrome / native webrtc now, we always negotiate opus payload to 111 with those clients,
+	// so it is safe to use a fixed payload 111 here for performance(avoid encoding red blocks for each downtrack that
+	// have a different opus payload type).
+	opusPT    = 111
+	opusRedPT = 63
+)
+
+type RedReceiver struct {
+	TrackReceiver
+	downTrackSpreader *utils.DownTrackSpreader[TrackSender]
+	logger            logger.Logger
+	closed            atomic.Bool
+	pktBuff           [maxRedCount]*rtp.Packet
+	redPayloadBuf     [mtuSize]byte
+}
+
+func NewRedReceiver(receiver TrackReceiver, dsp utils.DownTrackSpreaderParams) REDTransformer {
+	return &RedReceiver{
+		TrackReceiver:     receiver,
+		downTrackSpreader: utils.NewDownTrackSpreader[TrackSender](dsp),
+		logger:            dsp.Logger,
+	}
+}
+
+func (r *RedReceiver) SetLBThreshold(lbThreshold int) {
+	r.downTrackSpreader.SetThreshold(lbThreshold)
+}
+
+func (r *RedReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int32 {
+	// encode RED payload from primary payload and forward to downtracks
+	if r.downTrackSpreader.DownTrackCount() == 0 {
+		return 0
+	}
+
+	// fallback to primary codec if payload size exceeds redundant block length
+	if len(pkt.Packet.Payload) >= maxRedPayload {
+		var writeCount atomic.Int32
+		r.downTrackSpreader.Broadcast(func(dt TrackSender) {
+			writeCount.Add(dt.WriteRTP(pkt, spatialLayer))
+		})
+		return writeCount.Load()
+	}
+
+	redLen, err := r.encodeRedForPrimary(pkt.Packet, r.redPayloadBuf[:])
+	if err != nil {
+		r.logger.Errorw("red encoding failed", err)
+		return 0
+	}
+
+	pPkt := *pkt
+	redRtpPacket := *pkt.Packet
+	redRtpPacket.PayloadType = opusRedPT
+	redRtpPacket.Payload = r.redPayloadBuf[:redLen]
+	pPkt.Packet = &redRtpPacket
+
+	// not modify the ExtPacket.RawPacket here for performance since it is not used by the DownTrack,
+	// otherwise it should be set to the correct value (marshal the primary rtp packet)
+	var writeCount atomic.Int32
+	r.downTrackSpreader.Broadcast(func(dt TrackSender) {
+		writeCount.Add(dt.WriteRTP(&pPkt, spatialLayer))
+	})
+	return writeCount.Load()
+}
+
+func (r *RedReceiver) ForwardRTCPSenderReport(
+	payloadType webrtc.PayloadType,
+	layer int32,
+	publisherSRData *livekit.RTCPSenderReportState,
+) {
+	r.downTrackSpreader.Broadcast(func(dt TrackSender) {
+		_ = dt.HandleRTCPSenderReportData(payloadType, layer, publisherSRData)
+	})
+}
+
+func (r *RedReceiver) AddDownTrack(track TrackSender) error {
+	if r.closed.Load() {
+		return ErrReceiverClosed
+	}
+
+	if r.downTrackSpreader.HasDownTrack(track.SubscriberID()) {
+		r.logger.Infow("subscriberID already exists, replacing downtrack", "subscriberID", track.SubscriberID())
+	}
+
+	r.downTrackSpreader.Store(track)
+	r.logger.Debugw("red receiver downtrack added", "subscriberID", track.SubscriberID())
+	return nil
+}
+
+func (r *RedReceiver) DeleteDownTrack(subscriberID livekit.ParticipantID) {
+	if r.closed.Load() {
+		return
+	}
+
+	r.downTrackSpreader.Free(subscriberID)
+	r.logger.Debugw("red receiver downtrack deleted", "subscriberID", subscriberID)
+}
+
+func (r *RedReceiver) GetDownTracks() []TrackSender {
+	return r.downTrackSpreader.GetDownTracks()
+}
+
+func (r *RedReceiver) ResyncDownTracks() {
+	r.downTrackSpreader.Broadcast(func(dt TrackSender) {
+		dt.Resync()
+	})
+}
+
+func (r *RedReceiver) OnStreamRestart() {
+	r.downTrackSpreader.Broadcast(func(dt TrackSender) {
+		dt.ReceiverRestart(r)
+	})
+}
+
+func (r *RedReceiver) CanClose() bool {
+	return r.closed.Load() || r.downTrackSpreader.DownTrackCount() == 0
+}
+
+func (r *RedReceiver) IsClosed() bool {
+	return r.closed.Load()
+}
+
+func (r *RedReceiver) Close() {
+	r.closed.Store(true)
+	closeTrackSenders(r.downTrackSpreader.ResetAndGetDownTracks())
+}
+
+func (r *RedReceiver) ReadRTP(buf []byte, layer uint8, esn uint64) (int, error) {
+	// red encoding doesn't support nack
+	return 0, bucket.ErrPacketMismatch
+}
+
+func (r *RedReceiver) encodeRedForPrimary(pkt *rtp.Packet, redPayload []byte) (int, error) {
+	redLength := len(r.pktBuff)
+	redPkts := make([]*rtp.Packet, 0, redLength+1)
+	lastNilPkt := -1
+	for i := redLength - 1; i >= 0; i-- {
+		if r.pktBuff[i] == nil {
+			lastNilPkt = i
+			break
+		}
+
+	}
+
+	for _, prev := range r.pktBuff[lastNilPkt+1:] {
+		if pkt.SequenceNumber == prev.SequenceNumber ||
+			(pkt.SequenceNumber-prev.SequenceNumber) > uint16(redLength) ||
+			(pkt.Timestamp-prev.Timestamp) >= (1<<14) {
+			continue
+		}
+		redPkts = append(redPkts, prev)
+	}
+
+	// insert primary packet in history buffer
+	// NOTE: packet is copied from retransmission buffer and used in forwarding path. So, not making another
+	// copy here and just maintaining pointer to the packet as the forwarding path should not alter the packet.
+	for i := redLength - 1; i >= 0; i-- {
+		if r.pktBuff[i] == nil || // history is empty
+			pkt.SequenceNumber-r.pktBuff[i].SequenceNumber < (1<<15) { // received packet has more recent sequence number
+			// age out older ones
+			for j := 0; j < i; j++ {
+				r.pktBuff[j] = r.pktBuff[j+1]
+			}
+			r.pktBuff[i] = pkt
+			break
+		}
+	}
+
+	return encodeRedForPrimary(redPkts, pkt, redPayload)
+}
+
+func encodeRedForPrimary(redPkts []*rtp.Packet, primary *rtp.Packet, redPayload []byte) (int, error) {
+	payloadSize := len(primary.Payload) + 1
+	for _, p := range redPkts {
+		payloadSize += len(p.Payload) + 4
+	}
+
+	// if required payload size is larger than the redPayload buffer, encode the primary packet only
+	if payloadSize > len(redPayload) {
+		redPkts = redPkts[:0]
+	}
+
+	var index int
+	for _, p := range redPkts {
+		/* RED payload https://datatracker.ietf.org/doc/html/rfc2198#section-3
+			0                   1                    2                   3
+		    0 1 2 3 4 5 6 7 8 9 0 1 2 3  4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+		   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+		   |F|   block PT  |  timestamp offset         |   block length    |
+		   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+		   F: 1 bit First bit in header indicates whether another header block
+		       follows.  If 1 further header blocks follow, if 0 this is the
+		       last header block.
+		*/
+		header := uint32(0x80 | uint8(opusPT))
+		header <<= 14
+		header |= (primary.Timestamp - p.Timestamp) & 0x3FFF
+		header <<= 10
+		header |= uint32(len(p.Payload)) & 0x3FF
+		binary.BigEndian.PutUint32(redPayload[index:], header)
+		index += 4
+	}
+	// last block header
+	redPayload[index] = uint8(opusPT)
+	index++
+
+	// append data blocks
+	redPkts = append(redPkts, primary)
+	for _, p := range redPkts {
+		if copy(redPayload[index:], p.Payload) < len(p.Payload) {
+			return 0, fmt.Errorf("red payload don't have enough space, needsize %d", len(p.Payload))
+		}
+		index += len(p.Payload)
+	}
+	return index, nil
+}

pkg/sfu/redreceiver_test.go

@@ -0,0 +1,456 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sfu
+
+import (
+	"testing"
+
+	"github.com/pion/rtp"
+	"github.com/pion/webrtc/v4"
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/protocol/logger"
+)
+
+const (
+	tsStep = uint32(48000 / 1000 * 10)
+)
+
+type dummyDowntrack struct {
+	TrackSender
+	lastReceivedPkt *rtp.Packet
+	receivedPkts    []*rtp.Packet
+}
+
+func (dt *dummyDowntrack) WriteRTP(p *buffer.ExtPacket, _ int32) int32 {
+	dt.lastReceivedPkt = p.Packet
+	dt.receivedPkts = append(dt.receivedPkts, p.Packet)
+	return 1
+}
+
+func TestRedReceiver(t *testing.T) {
+	dt := &dummyDowntrack{TrackSender: &DownTrack{}}
+
+	t.Run("normal", func(t *testing.T) {
+		w := &WebRTCReceiver{
+			ReceiverBase: &ReceiverBase{
+				params: ReceiverBaseParams{
+					Kind:   webrtc.RTPCodecTypeAudio,
+					Logger: logger.GetLogger(),
+				},
+				isRED: true,
+			},
+		}
+		require.Equal(t, w.GetRedReceiver(), w.ReceiverBase)
+		w.isRED = false
+		red := w.GetRedReceiver().(*RedReceiver)
+		require.NotNil(t, red)
+		require.NoError(t, red.AddDownTrack(dt))
+
+		header := rtp.Header{SequenceNumber: 65534, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111}
+		expectPkt := make([]*rtp.Packet, 0, maxRedCount+1)
+		for _, pkt := range generatePkts(header, 10, tsStep) {
+			expectPkt = append(expectPkt, pkt)
+			if len(expectPkt) > maxRedCount+1 {
+				expectPkt = expectPkt[1:]
+			}
+			red.ForwardRTP(&buffer.ExtPacket{
+				Packet: pkt,
+			}, 0)
+			verifyRedEncodings(t, dt.lastReceivedPkt, expectPkt)
+		}
+	})
+
+	t.Run("packet lost and jump", func(t *testing.T) {
+		w := &WebRTCReceiver{
+			ReceiverBase: &ReceiverBase{
+				params: ReceiverBaseParams{
+					Kind:   webrtc.RTPCodecTypeAudio,
+					Logger: logger.GetLogger(),
+				},
+			},
+		}
+		red := w.GetRedReceiver().(*RedReceiver)
+		require.NoError(t, red.AddDownTrack(dt))
+
+		header := rtp.Header{SequenceNumber: 65534, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111}
+		expectPkt := make([]*rtp.Packet, 0, maxRedCount+1)
+		for i := range 10 {
+			if i%2 == 0 {
+				header.SequenceNumber++
+				header.Timestamp += tsStep
+				expectPkt = append(expectPkt, nil)
+				continue
+			}
+			hbuf, _ := header.Marshal()
+			pkt1 := &rtp.Packet{
+				Header:  header,
+				Payload: hbuf,
+			}
+			expectPkt = append(expectPkt, pkt1)
+			if len(expectPkt) > maxRedCount+1 {
+				expectPkt = expectPkt[len(expectPkt)-maxRedCount-1:]
+			}
+			red.ForwardRTP(&buffer.ExtPacket{
+				Packet: pkt1,
+			}, 0)
+			verifyRedEncodings(t, dt.lastReceivedPkt, expectPkt)
+			header.SequenceNumber++
+			header.Timestamp += tsStep
+		}
+
+		// jump
+		header.SequenceNumber += 10
+		header.Timestamp += 10 * tsStep
+
+		expectPkt = expectPkt[:0]
+		for _, pkt := range generatePkts(header, 3, tsStep) {
+			expectPkt = append(expectPkt, pkt)
+			if len(expectPkt) > maxRedCount+1 {
+				expectPkt = expectPkt[len(expectPkt)-maxRedCount-1:]
+			}
+			red.ForwardRTP(&buffer.ExtPacket{
+				Packet: pkt,
+			}, 0)
+			verifyRedEncodings(t, dt.lastReceivedPkt, expectPkt)
+		}
+	})
+
+	t.Run("unorder and repeat", func(t *testing.T) {
+		w := &WebRTCReceiver{
+			ReceiverBase: &ReceiverBase{
+				params: ReceiverBaseParams{
+					Kind:   webrtc.RTPCodecTypeAudio,
+					Logger: logger.GetLogger(),
+				},
+			},
+		}
+		red := w.GetRedReceiver().(*RedReceiver)
+		require.NoError(t, red.AddDownTrack(dt))
+
+		header := rtp.Header{SequenceNumber: 65534, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111}
+		var prevPkts []*rtp.Packet
+		for _, pkt := range generatePkts(header, 10, tsStep) {
+			red.ForwardRTP(&buffer.ExtPacket{
+				Packet: pkt,
+			}, 0)
+			prevPkts = append(prevPkts, pkt)
+		}
+
+		// old unorder data don't have red records
+		expectPkt := prevPkts[len(prevPkts)-3 : len(prevPkts)-2]
+		red.ForwardRTP(&buffer.ExtPacket{
+			Packet: expectPkt[0],
+		}, 0)
+		verifyRedEncodings(t, dt.lastReceivedPkt, expectPkt)
+
+		// repeat packet only have 1 red records
+		expectPkt = prevPkts[len(prevPkts)-2:]
+		red.ForwardRTP(&buffer.ExtPacket{
+			Packet: expectPkt[1],
+		}, 0)
+		verifyRedEncodings(t, dt.lastReceivedPkt, expectPkt)
+	})
+
+	t.Run("encoding exceed space", func(t *testing.T) {
+		w := &WebRTCReceiver{
+			ReceiverBase: &ReceiverBase{
+				params: ReceiverBaseParams{
+					Kind:   webrtc.RTPCodecTypeAudio,
+					Logger: logger.GetLogger(),
+				},
+				isRED: true,
+			},
+		}
+		require.Equal(t, w.GetRedReceiver(), w.ReceiverBase)
+		w.isRED = false
+		red := w.GetRedReceiver().(*RedReceiver)
+		require.NotNil(t, red)
+		require.NoError(t, red.AddDownTrack(dt))
+
+		header := rtp.Header{SequenceNumber: 65534, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111}
+		expectPkt := make([]*rtp.Packet, 0, maxRedCount+1)
+		for _, pkt := range generatePkts(header, 10, tsStep) {
+			// make sure red encodings don't have enough space to encoding redundant packet
+			pkt.Payload = make([]byte, 1000)
+			expectPkt = append(expectPkt[:0], pkt)
+			red.ForwardRTP(&buffer.ExtPacket{
+				Packet: pkt,
+			}, 0)
+			verifyRedEncodings(t, dt.lastReceivedPkt, expectPkt)
+		}
+	})
+
+	t.Run("large timestamp gap", func(t *testing.T) {
+		w := &WebRTCReceiver{
+			ReceiverBase: &ReceiverBase{
+				params: ReceiverBaseParams{
+					Kind:   webrtc.RTPCodecTypeAudio,
+					Logger: logger.GetLogger(),
+				},
+				isRED: true,
+			},
+		}
+		require.Equal(t, w.GetRedReceiver(), w.ReceiverBase)
+		w.isRED = false
+		red := w.GetRedReceiver().(*RedReceiver)
+		require.NotNil(t, red)
+		require.NoError(t, red.AddDownTrack(dt))
+
+		header := rtp.Header{SequenceNumber: 65534, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111}
+		// first few packets normal
+		expectPkt := make([]*rtp.Packet, 0, maxRedCount+1)
+		for _, pkt := range generatePkts(header, 4, tsStep) {
+			expectPkt = append(expectPkt, pkt)
+			if len(expectPkt) > maxRedCount+1 {
+				expectPkt = expectPkt[1:]
+			}
+			red.ForwardRTP(&buffer.ExtPacket{
+				Packet: pkt,
+			}, 0)
+			verifyRedEncodings(t, dt.lastReceivedPkt, expectPkt)
+		}
+
+		// and then a few packets with a large timestamp jump, should contain only primary
+		for _, pkt := range generatePkts(header, 4, 40*tsStep) {
+			red.ForwardRTP(&buffer.ExtPacket{
+				Packet: pkt,
+			}, 0)
+			verifyRedEncodings(t, dt.lastReceivedPkt, []*rtp.Packet{pkt})
+		}
+	})
+}
+
+func verifyRedEncodings(t *testing.T, red *rtp.Packet, redPkts []*rtp.Packet) {
+	solidPkts := make([]*rtp.Packet, 0, len(redPkts))
+	for _, pkt := range redPkts {
+		if pkt != nil {
+			solidPkts = append(solidPkts, pkt)
+		}
+	}
+	pktsFromRed, err := extractPktsFromRed(red, 0xFF)
+	require.NoError(t, err)
+	require.Len(t, pktsFromRed, len(solidPkts))
+	for i, pkt := range pktsFromRed {
+		verifyEncodingEqual(t, pkt, solidPkts[i])
+	}
+}
+
+func verifyPktsEqual(t *testing.T, p1s, p2s []*rtp.Packet) {
+	require.Len(t, p1s, len(p2s))
+	for i, pkt := range p1s {
+		verifyEncodingEqual(t, pkt, p2s[i])
+	}
+}
+
+func verifyEncodingEqual(t *testing.T, p1, p2 *rtp.Packet) {
+	require.Equal(t, p1.Header.Timestamp, p2.Header.Timestamp)
+	require.Equal(t, p1.PayloadType, p2.PayloadType)
+	require.EqualValues(t, p1.Payload, p2.Payload, "seq1 %s", p1.SequenceNumber)
+}
+
+func generatePkts(header rtp.Header, count int, tsStep uint32) []*rtp.Packet {
+	pkts := make([]*rtp.Packet, 0, count)
+	for range count {
+		hbuf, _ := header.Marshal()
+		pkts = append(pkts, &rtp.Packet{
+			Header:  header,
+			Payload: hbuf,
+		})
+		header.SequenceNumber++
+		header.Timestamp += tsStep
+	}
+	return pkts
+}
+
+func generateRedPkts(t *testing.T, pkts []*rtp.Packet, redCount int) []*rtp.Packet {
+	redPkts := make([]*rtp.Packet, 0, len(pkts))
+	for i, pkt := range pkts {
+		encodingPkts := make([]*rtp.Packet, 0, redCount)
+		for j := i - redCount; j < i; j++ {
+			if j < 0 {
+				continue
+			}
+			encodingPkts = append(encodingPkts, pkts[j])
+		}
+		buf := make([]byte, mtuSize)
+		redPkt := *pkt
+		redPkt.PayloadType = opusRedPT
+		encoded, err := encodeRedForPrimary(encodingPkts, pkt, buf)
+		require.NoError(t, err)
+		redPkt.Payload = buf[:encoded]
+		redPkts = append(redPkts, &redPkt)
+	}
+	return redPkts
+}
+
+func testRedRedPrimaryReceiver(t *testing.T, maxPktCount, redCount int, sendPktIdx, expectPktIdx []int) {
+	dt := &dummyDowntrack{TrackSender: &DownTrack{}}
+	w := &WebRTCReceiver{
+		ReceiverBase: &ReceiverBase{
+			params: ReceiverBaseParams{
+				Kind:   webrtc.RTPCodecTypeAudio,
+				Logger: logger.GetLogger(),
+				Codec:  webrtc.RTPCodecParameters{PayloadType: opusRedPT, RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: "audio/red"}},
+			},
+		},
+	}
+	require.Equal(t, w.GetPrimaryReceiverForRed(), w.ReceiverBase)
+	w.isRED = true
+	red := w.GetPrimaryReceiverForRed().(*RedPrimaryReceiver)
+	require.NotNil(t, red)
+	require.NoError(t, red.AddDownTrack(dt))
+
+	header := rtp.Header{SequenceNumber: 65530, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111}
+	primaryPkts := generatePkts(header, maxPktCount, tsStep)
+	redPkts := generateRedPkts(t, primaryPkts, redCount)
+
+	for _, i := range sendPktIdx {
+		red.ForwardRTP(&buffer.ExtPacket{
+			Packet: redPkts[i],
+		}, 0)
+	}
+
+	expectPkts := make([]*rtp.Packet, 0, len(expectPktIdx))
+	for _, i := range expectPktIdx {
+		expectPkts = append(expectPkts, primaryPkts[i])
+	}
+
+	verifyPktsEqual(t, expectPkts, dt.receivedPkts)
+}
+
+func TestRedPrimaryReceiver(t *testing.T) {
+	w := &WebRTCReceiver{
+		ReceiverBase: &ReceiverBase{
+			params: ReceiverBaseParams{
+				Kind:   webrtc.RTPCodecTypeAudio,
+				Logger: logger.GetLogger(),
+			},
+		},
+	}
+	require.Equal(t, w.GetPrimaryReceiverForRed(), w.ReceiverBase)
+	w.isRED = true
+	red := w.GetPrimaryReceiverForRed().(*RedPrimaryReceiver)
+	require.NotNil(t, red)
+
+	t.Run("packet should send only once", func(t *testing.T) {
+		maxPktCount := 19
+		var sendPktIndex []int
+		for i := range maxPktCount {
+			sendPktIndex = append(sendPktIndex, i)
+		}
+		testRedRedPrimaryReceiver(t, maxPktCount, maxRedCount, sendPktIndex, sendPktIndex)
+	})
+
+	t.Run("packet duplicate and unorder", func(t *testing.T) {
+		maxPktCount := 19
+		var sendPktIndex []int
+		for i := range maxPktCount {
+			sendPktIndex = append(sendPktIndex, i)
+			if i > 0 {
+				sendPktIndex = append(sendPktIndex, i-1)
+			}
+			sendPktIndex = append(sendPktIndex, i)
+		}
+		testRedRedPrimaryReceiver(t, maxPktCount, maxRedCount, sendPktIndex, sendPktIndex)
+	})
+
+	t.Run("full recover", func(t *testing.T) {
+		maxPktCount := 19
+		var sendPktIndex, recvPktIndex []int
+		for i := range maxPktCount {
+			recvPktIndex = append(recvPktIndex, i)
+
+			// drop packets covered by red encoding
+			if i%(maxRedCount+1) != 0 {
+				continue
+			}
+			sendPktIndex = append(sendPktIndex, i)
+		}
+
+		testRedRedPrimaryReceiver(t, maxPktCount, maxRedCount, sendPktIndex, recvPktIndex)
+	})
+
+	t.Run("lost 2 but red recover 1", func(t *testing.T) {
+		maxPktCount := 19
+		sendPktIndex := []int{0, 3, 6, 9, 12}
+		recvPktIndex := []int{0, 2, 3, 5, 6, 8, 9, 11, 12}
+		testRedRedPrimaryReceiver(t, maxPktCount, 1, sendPktIndex, recvPktIndex)
+	})
+
+	t.Run("part recover and long jump", func(t *testing.T) {
+		maxPktCount := 50
+		sendPktIndex := []int{0, 5, 12, 21 /*long jump*/, 24, 27}
+		recvPktIndex := []int{0, 3, 4, 5, 10, 11, 12, 19, 20, 21, 22, 23, 24, 25, 26, 27}
+		testRedRedPrimaryReceiver(t, maxPktCount, maxRedCount, sendPktIndex, recvPktIndex)
+	})
+
+	t.Run("unorder", func(t *testing.T) {
+		maxPktCount := 50
+		sendPktIndex := []int{20, 10 /*unorder can't recover*/, 25, 23, 34}
+		recvPktIndex := []int{20, 10, 23, 24, 25, 21, 22, 23, 32, 33, 34}
+		testRedRedPrimaryReceiver(t, maxPktCount, maxRedCount, sendPktIndex, recvPktIndex)
+	})
+
+	t.Run("mixed primary codec", func(t *testing.T) {
+		dt := &dummyDowntrack{TrackSender: &DownTrack{}}
+		w := &WebRTCReceiver{
+			ReceiverBase: &ReceiverBase{
+				params: ReceiverBaseParams{
+					Kind:   webrtc.RTPCodecTypeAudio,
+					Logger: logger.GetLogger(),
+					Codec:  webrtc.RTPCodecParameters{PayloadType: opusRedPT, RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: "audio/red"}},
+				},
+			},
+		}
+		require.Equal(t, w.GetPrimaryReceiverForRed(), w.ReceiverBase)
+		w.isRED = true
+		red := w.GetPrimaryReceiverForRed().(*RedPrimaryReceiver)
+		require.NotNil(t, red)
+		require.NoError(t, red.AddDownTrack(dt))
+
+		primaryPkt := &rtp.Packet{
+			Header:  rtp.Header{SequenceNumber: 65530, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111},
+			Payload: []byte{1, 3, 5, 7, 9},
+		}
+		red.ForwardRTP(&buffer.ExtPacket{
+			Packet: primaryPkt,
+		}, 0)
+
+		verifyPktsEqual(t, []*rtp.Packet{primaryPkt}, dt.receivedPkts)
+	})
+}
+
+func TestExtractPrimaryEncodingForRED(t *testing.T) {
+	header := rtp.Header{SequenceNumber: 65530, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111}
+	pkts := generatePkts(header, 10, tsStep)
+	redPkts := generateRedPkts(t, pkts, 2)
+
+	primaryPkts := make([]*rtp.Packet, 0, len(redPkts))
+
+	for _, redPkt := range redPkts {
+		payload, err := extractPrimaryEncodingForRED(redPkt.Payload)
+		require.NoError(t, err)
+		primaryHeader := redPkt.Header
+		primaryHeader.PayloadType = 111
+		primaryPkts = append(primaryPkts, &rtp.Packet{
+			Header:  primaryHeader,
+			Payload: payload,
+		})
+	}
+
+	verifyPktsEqual(t, pkts, primaryPkts)
+}

pkg/sfu/rtpextension/abscapturetime/abscapturetime.go

@@ -0,0 +1,114 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package abscapturetime
+
+import (
+	"encoding/binary"
+	"errors"
+	"time"
+
+	"github.com/livekit/mediatransportutil"
+)
+
+const (
+	AbsCaptureTimeURI = "http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time"
+)
+
+var (
+	errInvalidData = errors.New("invalid data")
+	errTooSmall    = errors.New("buffer too small")
+)
+
+// Reference: https://webrtc.googlesource.com/src/+/refs/heads/main/docs/native-code/rtp-hdrext/abs-capture-time/
+//
+// Data layout of the shortened version of abs-capture-time with a 1-byte header + 8 bytes of data:
+//
+//  0                   1                   2                   3
+//  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// |  ID   | len=7 |     absolute capture timestamp (bit 0-23)     |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// |             absolute capture timestamp (bit 24-55)            |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// |  ... (56-63)  |
+// +-+-+-+-+-+-+-+-+
+//
+//Data layout of the extended version of abs-capture-time with a 1-byte header + 16 bytes of data:
+//
+//  0                   1                   2                   3
+//  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// |  ID   | len=15|     absolute capture timestamp (bit 0-23)     |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// |             absolute capture timestamp (bit 24-55)            |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// |  ... (56-63)  |   estimated capture clock offset (bit 0-23)   |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// |           estimated capture clock offset (bit 24-55)          |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// |  ... (56-63)  |
+// +-+-+-+-+-+-+-+-+
+
+type AbsCaptureTime struct {
+	absoluteCaptureTimestamp    mediatransportutil.NtpTime
+	estimatedCaptureClockOffset int64
+}
+
+func AbsCaptureTimeFromValue(absoluteCaptureTimestamp uint64, estimatedCaptureClockOffset int64) *AbsCaptureTime {
+	return &AbsCaptureTime{
+		absoluteCaptureTimestamp:    mediatransportutil.NtpTime(absoluteCaptureTimestamp),
+		estimatedCaptureClockOffset: estimatedCaptureClockOffset,
+	}
+}
+
+func (a *AbsCaptureTime) Rewrite(offset time.Duration) error {
+	if a.absoluteCaptureTimestamp == 0 {
+		return errInvalidData
+	}
+
+	capturedAt := a.absoluteCaptureTimestamp.Time().Add(offset)
+	a.absoluteCaptureTimestamp = mediatransportutil.ToNtpTime(capturedAt)
+	a.estimatedCaptureClockOffset = 0
+	return nil
+}
+
+func (a *AbsCaptureTime) Marshal() ([]byte, error) {
+	if a.absoluteCaptureTimestamp == 0 {
+		return nil, errInvalidData
+	}
+
+	size := 8
+	if a.estimatedCaptureClockOffset != 0 {
+		size += 8
+	}
+	marshalled := make([]byte, size)
+	binary.BigEndian.PutUint64(marshalled, uint64(a.absoluteCaptureTimestamp))
+	if a.estimatedCaptureClockOffset != 0 {
+		binary.BigEndian.PutUint64(marshalled[8:], uint64(a.estimatedCaptureClockOffset))
+	}
+	return marshalled, nil
+}
+
+func (a *AbsCaptureTime) Unmarshal(marshalled []byte) error {
+	if len(marshalled) < 8 {
+		return errTooSmall
+	}
+
+	a.absoluteCaptureTimestamp = mediatransportutil.NtpTime(binary.BigEndian.Uint64(marshalled))
+	if len(marshalled) >= 16 {
+		a.estimatedCaptureClockOffset = int64(binary.BigEndian.Uint64(marshalled[8:]))
+	}
+	return nil
+}

pkg/sfu/rtpextension/dependencydescriptor/bitstreamreader.go

@@ -0,0 +1,138 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dependencydescriptor
+
+import (
+	"errors"
+	"io"
+)
+
+type BitStreamReader struct {
+	buf           []byte
+	pos           int
+	remainingBits int
+}
+
+func NewBitStreamReader(buf []byte) *BitStreamReader {
+	return &BitStreamReader{buf: buf, remainingBits: len(buf) * 8}
+}
+
+func (b *BitStreamReader) RemainingBits() int {
+	return b.remainingBits
+}
+
+// Reads `bits` from the bitstream. `bits` must be in range [0, 64].
+// Returns an unsigned integer in range [0, 2^bits - 1].
+// On failure sets `BitstreamReader` into the failure state and returns 0.
+func (b *BitStreamReader) ReadBits(bits int) (uint64, error) {
+	if bits < 0 || bits > 64 {
+		return 0, errors.New("invalid number of bits, expected 0-64")
+	}
+
+	if b.remainingBits < bits {
+		b.remainingBits -= bits
+		return 0, io.EOF
+	}
+
+	remainingBitsInFirstByte := b.remainingBits % 8
+	b.remainingBits -= bits
+	if bits < remainingBitsInFirstByte {
+		// Reading fewer bits than what's left in the current byte, just
+		// return the portion of this byte that is needed.
+		offset := remainingBitsInFirstByte - bits
+		return uint64((b.buf[b.pos] >> offset) & ((1 << bits) - 1)), nil
+	}
+	var result uint64
+	if remainingBitsInFirstByte > 0 {
+		// Read all bits that were left in the current byte and consume that byte.
+		bits -= remainingBitsInFirstByte
+		mask := byte((1 << remainingBitsInFirstByte) - 1)
+		result = uint64(b.buf[b.pos]&mask) << bits
+		b.pos++
+	}
+	// Read as many full bytes as we can.
+	for bits >= 8 {
+		bits -= 8
+		result |= uint64(b.buf[b.pos]) << bits
+		b.pos++
+	}
+
+	// Whatever is left to read is smaller than a byte, so grab just the needed
+	// bits and shift them into the lowest bits.
+	if bits > 0 {
+		result |= uint64(b.buf[b.pos] >> (8 - bits))
+	}
+	return result, nil
+}
+
+func (b *BitStreamReader) ReadBool() (bool, error) {
+	val, err := b.ReadBits(1)
+	return val != 0, err
+}
+
+func (b *BitStreamReader) Ok() bool {
+	return b.remainingBits >= 0
+}
+
+func (b *BitStreamReader) Invalidate() {
+	b.remainingBits = -1
+}
+
+// Reads value in range [0, `num_values` - 1].
+// This encoding is similar to ReadBits(val, Ceil(Log2(num_values)),
+// but reduces wastage incurred when encoding non-power of two value ranges
+// Non symmetric values are encoded as:
+// 1) n = bit_width(num_values)
+// 2) k = (1 << n) - num_values
+// Value v in range [0, k - 1] is encoded in (n-1) bits.
+// Value v in range [k, num_values - 1] is encoded as (v+k) in n bits.
+// https://aomediacodec.github.io/av1-spec/#nsn
+func (b *BitStreamReader) ReadNonSymmetric(numValues uint32) (uint32, error) {
+	if numValues >= (uint32(1) << 31) {
+		return 0, errors.New("invalid number of values, expected 0-2^31")
+	}
+
+	width := bitwidth(numValues)
+	numMinBitsValues := (uint32(1) << width) - numValues
+
+	val, err := b.ReadBits(width - 1)
+	if err != nil {
+		return 0, err
+	}
+	if val < uint64(numMinBitsValues) {
+		return uint32(val), nil
+	}
+	bit, err := b.ReadBits(1)
+	if err != nil {
+		return 0, err
+	}
+	return uint32((val << 1) + bit - uint64(numMinBitsValues)), nil
+}
+
+func (b *BitStreamReader) BytesRead() int {
+	if b.remainingBits%8 > 0 {
+		return b.pos + 1
+	}
+	return b.pos
+}
+
+func bitwidth(n uint32) int {
+	var w int
+	for n != 0 {
+		n >>= 1
+		w++
+	}
+	return w
+}

pkg/sfu/rtpextension/dependencydescriptor/bitstreamwriter.go

@@ -0,0 +1,132 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dependencydescriptor
+
+import (
+	"errors"
+	"fmt"
+)
+
+type BitStreamWriter struct {
+	buf       []byte
+	pos       int
+	bitOffset int // bit offset in the current byte
+}
+
+func NewBitStreamWriter(buf []byte) *BitStreamWriter {
+	return &BitStreamWriter{buf: buf}
+}
+
+func (w *BitStreamWriter) RemainingBits() int {
+	return (len(w.buf)-w.pos)*8 - w.bitOffset
+}
+
+func (w *BitStreamWriter) WriteBits(val uint64, bitCount int) error {
+	if bitCount > w.RemainingBits() {
+		return errors.New("insufficient space")
+	}
+
+	totalBits := bitCount
+
+	// push bits to the highest bits of uint64
+	val <<= 64 - bitCount
+
+	buf := w.buf[w.pos:]
+
+	// The first byte is relatively special; the bit offset to write to may put us
+	// in the middle of the byte, and the total bit count to write may require we
+	// save the bits at the end of the byte.
+	remainingBitsInCurrentByte := 8 - w.bitOffset
+	bitsInFirstByte := bitCount
+	if bitsInFirstByte > remainingBitsInCurrentByte {
+		bitsInFirstByte = remainingBitsInCurrentByte
+	}
+
+	buf[0] = w.writePartialByte(uint8(val>>56), bitsInFirstByte, buf[0], w.bitOffset)
+
+	if bitCount <= remainingBitsInCurrentByte {
+		// no bit left to write
+		return w.consumeBits(totalBits)
+	}
+
+	// write the rest of the bits
+	val <<= bitsInFirstByte
+	buf = buf[1:]
+	bitCount -= bitsInFirstByte
+	for bitCount >= 8 {
+		buf[0] = uint8(val >> 56)
+		buf = buf[1:]
+		val <<= 8
+		bitCount -= 8
+	}
+
+	// write the last bits
+	if bitCount > 0 {
+		buf[0] = w.writePartialByte(uint8(val>>56), bitCount, buf[0], 0)
+	}
+	return w.consumeBits(totalBits)
+}
+
+func (w *BitStreamWriter) consumeBits(bitCount int) error {
+	if bitCount > w.RemainingBits() {
+		return errors.New("insufficient space")
+	}
+
+	w.pos += (w.bitOffset + bitCount) / 8
+	w.bitOffset = (w.bitOffset + bitCount) % 8
+
+	return nil
+}
+
+func (w *BitStreamWriter) writePartialByte(source uint8, sourceBitCount int, target uint8, targetBitOffset int) uint8 {
+	// if !(targetBitOffset < 8 &&  sourceBitCount <= (8-targetBitOffset)) {
+	// 	return fmt.Errorf("invalid argument, source %d, sourceBitCount %d, target %d, targetBitOffset %d", source, sourceBitCount, target, targetBitOffset)
+	// }
+
+	// generate mask for bits to overwrite, shift source bits to highest bits, then position to target bit offset
+	mask := uint8(0xff<<(8-sourceBitCount)) >> uint8(targetBitOffset)
+
+	// clear target bits and write source bits
+	return (target & ^mask) | (source >> targetBitOffset)
+}
+
+func (w *BitStreamWriter) WriteNonSymmetric(val, numValues uint32) error {
+	if val >= numValues || numValues > 1<<31 {
+		return fmt.Errorf("invalid argument, val %d, numValues %d", val, numValues)
+	}
+	if numValues == 1 {
+		// When there is only one possible value, it requires zero bits to store it.
+		// But WriteBits doesn't support writing zero bits.
+		return nil
+	}
+
+	countBits := bitwidth(numValues)
+	numMinBitsValues := (uint32(1) << countBits) - numValues
+	if val < numMinBitsValues {
+		return w.WriteBits(uint64(val), countBits-1)
+	} else {
+		return w.WriteBits(uint64(val+numMinBitsValues), countBits)
+	}
+}
+
+func SizeNonSymmetricBits(val, numValues uint32) int {
+	countBits := bitwidth(numValues)
+	numMinBitsValues := (uint32(1) << countBits) - numValues
+	if val < numMinBitsValues {
+		return countBits - 1
+	} else {
+		return countBits
+	}
+}

pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorextension.go

@@ -0,0 +1,199 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dependencydescriptor
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+)
+
+// DependencyDescriptorExtension is a extension payload format in
+// https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension
+
+func formatBitmask(b *uint32) string {
+	if b == nil {
+		return "-"
+	}
+	return strconv.FormatInt(int64(*b), 2)
+}
+
+// ------------------------------------------------------------------------------
+
+type DependencyDescriptorExtension struct {
+	Descriptor *DependencyDescriptor
+	Structure  *FrameDependencyStructure
+}
+
+func (d *DependencyDescriptorExtension) Marshal() ([]byte, error) {
+	return d.MarshalWithActiveChains(^uint32(0))
+}
+
+func (d *DependencyDescriptorExtension) MarshalWithActiveChains(activeChains uint32) ([]byte, error) {
+	writer, err := NewDependencyDescriptorWriter(nil, d.Structure, activeChains, d.Descriptor)
+	if err != nil {
+		return nil, err
+	}
+	buf := make([]byte, int(math.Ceil(float64(writer.ValueSizeBits())/8)))
+	writer.ResetBuf(buf)
+	if err = writer.Write(); err != nil {
+		return nil, err
+	}
+	return buf, nil
+}
+
+func (d *DependencyDescriptorExtension) Unmarshal(buf []byte) (int, error) {
+	reader := NewDependencyDescriptorReader(buf, d.Structure, d.Descriptor)
+	return reader.Parse()
+}
+
+// ------------------------------------------------------------------------------
+
+const (
+	MaxSpatialIds    = 4
+	MaxTemporalIds   = 8
+	MaxDecodeTargets = 32
+	MaxTemplates     = 64
+
+	AllChainsAreActive = uint32(0)
+
+	ExtensionURI = "https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension"
+)
+
+// ------------------------------------------------------------------------------
+
+type DependencyDescriptor struct {
+	FirstPacketInFrame         bool
+	LastPacketInFrame          bool
+	FrameNumber                uint16
+	FrameDependencies          *FrameDependencyTemplate
+	Resolution                 *RenderResolution
+	ActiveDecodeTargetsBitmask *uint32
+	AttachedStructure          *FrameDependencyStructure
+}
+
+func (d *DependencyDescriptor) MarshalSize() (int, error) {
+	return d.MarshalSizeWithActiveChains(^uint32(0))
+}
+
+func (d *DependencyDescriptor) MarshalSizeWithActiveChains(activeChains uint32) (int, error) {
+	writer, err := NewDependencyDescriptorWriter(nil, d.AttachedStructure, activeChains, d)
+	if err != nil {
+		return 0, err
+	}
+	return int(math.Ceil(float64(writer.ValueSizeBits()) / 8)), nil
+}
+
+func (d *DependencyDescriptor) String() string {
+	resolution, dependencies := "-", "-"
+	if d.Resolution != nil {
+		resolution = fmt.Sprintf("%+v", *d.Resolution)
+	}
+	if d.FrameDependencies != nil {
+		dependencies = fmt.Sprintf("%+v", *d.FrameDependencies)
+	}
+	return fmt.Sprintf("DependencyDescriptor{FirstPacketInFrame: %v, LastPacketInFrame: %v, FrameNumber: %v, FrameDependencies: %s, Resolution: %s, ActiveDecodeTargetsBitmask: %v, AttachedStructure: %v}",
+		d.FirstPacketInFrame, d.LastPacketInFrame, d.FrameNumber, dependencies, resolution, formatBitmask(d.ActiveDecodeTargetsBitmask), d.AttachedStructure)
+}
+
+// ------------------------------------------------------------------------------
+
+// Relationship of a frame to a Decode target.
+type DecodeTargetIndication int
+
+const (
+	DecodeTargetNotPresent  DecodeTargetIndication = iota // DecodeTargetInfo symbol '-'
+	DecodeTargetDiscardable                               // DecodeTargetInfo symbol 'D'
+	DecodeTargetSwitch                                    // DecodeTargetInfo symbol 'S'
+	DecodeTargetRequired                                  // DecodeTargetInfo symbol 'R'
+)
+
+func (i DecodeTargetIndication) String() string {
+	switch i {
+	case DecodeTargetNotPresent:
+		return "-"
+	case DecodeTargetDiscardable:
+		return "D"
+	case DecodeTargetSwitch:
+		return "S"
+	case DecodeTargetRequired:
+		return "R"
+	default:
+		return "Unknown"
+	}
+}
+
+// ------------------------------------------------------------------------------
+
+type FrameDependencyTemplate struct {
+	SpatialId               int
+	TemporalId              int
+	DecodeTargetIndications []DecodeTargetIndication
+	FrameDiffs              []int
+	ChainDiffs              []int
+}
+
+func (t *FrameDependencyTemplate) Clone() *FrameDependencyTemplate {
+	t2 := &FrameDependencyTemplate{
+		SpatialId:  t.SpatialId,
+		TemporalId: t.TemporalId,
+	}
+
+	t2.DecodeTargetIndications = make([]DecodeTargetIndication, len(t.DecodeTargetIndications))
+	copy(t2.DecodeTargetIndications, t.DecodeTargetIndications)
+
+	t2.FrameDiffs = make([]int, len(t.FrameDiffs))
+	copy(t2.FrameDiffs, t.FrameDiffs)
+
+	t2.ChainDiffs = make([]int, len(t.ChainDiffs))
+	copy(t2.ChainDiffs, t.ChainDiffs)
+
+	return t2
+}
+
+// ------------------------------------------------------------------------------
+
+type FrameDependencyStructure struct {
+	StructureId      int
+	NumDecodeTargets int
+	NumChains        int
+	// If chains are used (num_chains > 0), maps decode target index into index of
+	// the chain protecting that target.
+	DecodeTargetProtectedByChain []int
+	Resolutions                  []RenderResolution
+	Templates                    []*FrameDependencyTemplate
+}
+
+func (f *FrameDependencyStructure) String() string {
+	str := fmt.Sprintf("FrameDependencyStructure{StructureId: %v, NumDecodeTargets: %v, NumChains: %v, DecodeTargetProtectedByChain: %v, Resolutions: %+v, Templates: [",
+		f.StructureId, f.NumDecodeTargets, f.NumChains, f.DecodeTargetProtectedByChain, f.Resolutions)
+
+	// templates
+	for _, t := range f.Templates {
+		str += fmt.Sprintf("%+v, ", t)
+	}
+	str += "]}"
+
+	return str
+}
+
+// ------------------------------------------------------------------------------
+
+type RenderResolution struct {
+	Width  int
+	Height int
+}
+
+// ------------------------------------------------------------------------------

pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorextension_test.go

@@ -0,0 +1,65 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dependencydescriptor
+
+import (
+	"encoding/hex"
+	"testing"
+)
+
+func TestDependencyDescriptorUnmarshal(t *testing.T) {
+
+	// hex bytes from traffic capture
+	hexes := []string{
+		"c1017280081485214eafffaaaa863cf0430c10c302afc0aaa0063c00430010c002a000a80006000040001d954926e082b04a0941b820ac1282503157f974000ca864330e222222eca8655304224230eca877530077004200ef008601df010d",
+		"86017340fc",
+		"46017340fc",
+		"c3017540fc",
+		"88017640fc",
+		"48017640fc",
+		"c2017840fc",
+		//
+		"c1017280081485214eafffaaaa863cf0430c10c302afc0aaa0063c00430010c002a000a80006000040001d954926e082b04a0941b820ac1282503157f974000ca864330e222222eca8655304224230eca877530077004200ef008601df010d",
+		"860173",
+		"460173",
+		"8b0174",
+		"0b0174",
+		"0b0174",
+		"c30175",
+	}
+
+	var structure *FrameDependencyStructure
+
+	for _, h := range hexes {
+		buf, err := hex.DecodeString(h)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		var ddVal DependencyDescriptor
+		var d = DependencyDescriptorExtension{
+			Structure:  structure,
+			Descriptor: &ddVal,
+		}
+		if _, err := d.Unmarshal(buf); err != nil {
+			t.Fatal(err)
+		}
+		if ddVal.AttachedStructure != nil {
+			structure = ddVal.AttachedStructure
+		}
+
+		t.Log(ddVal.String())
+	}
+}

pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorreader.go

@@ -0,0 +1,447 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dependencydescriptor
+
+import (
+	"errors"
+)
+
+var (
+	ErrDDReaderNoStructure              = errors.New("DependencyDescriptorReader: Structure is nil")
+	ErrDDReaderTemplateWithoutStructure = errors.New("DependencyDescriptorReader: has templateDependencyStructurePresentFlag but AttachedStructure is nil")
+	ErrDDReaderTooManyTemplates         = errors.New("DependencyDescriptorReader: too many templates")
+	ErrDDReaderTooManyTemporalLayers    = errors.New("DependencyDescriptorReader: too many temporal layers")
+	ErrDDReaderTooManySpatialLayers     = errors.New("DependencyDescriptorReader: too many spatial layers")
+	ErrDDReaderInvalidTemplateIndex     = errors.New("DependencyDescriptorReader: invalid template index")
+	ErrDDReaderInvalidSpatialLayer      = errors.New("DependencyDescriptorReader: invalid spatial layer, should be less than the number of resolutions")
+	ErrDDReaderNumDTIMismatch           = errors.New("DependencyDescriptorReader: decode target indications length mismatch with structure num decode targets")
+	ErrDDReaderNumChainDiffsMismatch    = errors.New("DependencyDescriptorReader: chain diffs length mismatch with structure num chains")
+)
+
+type DependencyDescriptorReader struct {
+	// Output.
+	descriptor *DependencyDescriptor
+
+	// Values that are needed while reading the descriptor, but can be discarded
+	// when reading is complete.
+	buffer                         *BitStreamReader
+	frameDependencyTemplateId      int
+	activeDecodeTargetsPresentFlag bool
+	customDtisFlag                 bool
+	customFdiffsFlag               bool
+	customChainsFlag               bool
+	structure                      *FrameDependencyStructure
+}
+
+func NewDependencyDescriptorReader(buf []byte, structure *FrameDependencyStructure, descriptor *DependencyDescriptor) *DependencyDescriptorReader {
+	buffer := NewBitStreamReader(buf)
+	return &DependencyDescriptorReader{
+		buffer:     buffer,
+		descriptor: descriptor,
+		structure:  structure,
+	}
+}
+
+func (r *DependencyDescriptorReader) Parse() (int, error) {
+	if err := r.readMandatoryFields(); err != nil {
+		return 0, err
+	}
+	if len(r.buffer.buf) > 3 {
+		err := r.readExtendedFields()
+		if err != nil {
+			return 0, err
+		}
+	}
+
+	if r.descriptor.AttachedStructure != nil {
+		r.structure = r.descriptor.AttachedStructure
+	}
+
+	if r.structure == nil {
+		r.buffer.Invalidate()
+		return 0, ErrDDReaderNoStructure
+	}
+
+	if r.activeDecodeTargetsPresentFlag {
+		bitmask, err := r.buffer.ReadBits(r.structure.NumDecodeTargets)
+		if err != nil {
+			return 0, err
+		}
+		mask := uint32(bitmask)
+		r.descriptor.ActiveDecodeTargetsBitmask = &mask
+	}
+
+	err := r.readFrameDependencyDefinition()
+	if err != nil {
+		return 0, err
+	}
+	return r.buffer.BytesRead(), nil
+}
+
+func (r *DependencyDescriptorReader) readMandatoryFields() error {
+	var err error
+	r.descriptor.FirstPacketInFrame, err = r.buffer.ReadBool()
+	if err != nil {
+		return err
+	}
+
+	r.descriptor.LastPacketInFrame, err = r.buffer.ReadBool()
+	if err != nil {
+		return err
+	}
+
+	templateID, err := r.buffer.ReadBits(6)
+	if err != nil {
+		return err
+	}
+	r.frameDependencyTemplateId = int(templateID)
+
+	frameNumber, err := r.buffer.ReadBits(16)
+	if err != nil {
+		return err
+	}
+	r.descriptor.FrameNumber = uint16(frameNumber)
+	return nil
+}
+
+func (r *DependencyDescriptorReader) readExtendedFields() error {
+	templateDependencyStructurePresentFlag, err := r.buffer.ReadBool()
+	if err != nil {
+		return err
+	}
+
+	flag, err := r.buffer.ReadBool()
+	if err != nil {
+		return err
+	}
+	r.activeDecodeTargetsPresentFlag = flag
+
+	flag, err = r.buffer.ReadBool()
+	if err != nil {
+		return err
+	}
+	r.customDtisFlag = flag
+
+	flag, err = r.buffer.ReadBool()
+	if err != nil {
+		return err
+	}
+	r.customFdiffsFlag = flag
+
+	flag, err = r.buffer.ReadBool()
+	if err != nil {
+		return err
+	}
+	r.customChainsFlag = flag
+
+	if templateDependencyStructurePresentFlag {
+		err = r.readTemplateDependencyStructure()
+		if err != nil {
+			return err
+		}
+		if r.descriptor.AttachedStructure == nil {
+			return ErrDDReaderTemplateWithoutStructure
+		}
+		bitmask := uint32((uint64(1) << r.descriptor.AttachedStructure.NumDecodeTargets) - 1)
+		r.descriptor.ActiveDecodeTargetsBitmask = &bitmask
+	}
+	return nil
+}
+
+func (r *DependencyDescriptorReader) readTemplateDependencyStructure() error {
+	r.descriptor.AttachedStructure = &FrameDependencyStructure{}
+	structureId, err := r.buffer.ReadBits(6)
+	if err != nil {
+		return err
+	}
+	r.descriptor.AttachedStructure.StructureId = int(structureId)
+
+	numDecodeTargets, err := r.buffer.ReadBits(5)
+	if err != nil {
+		return err
+	}
+	r.descriptor.AttachedStructure.NumDecodeTargets = int(numDecodeTargets) + 1
+
+	if err = r.readTemplateLayers(); err != nil {
+		return err
+	}
+	if err = r.readTemplateDtis(); err != nil {
+		return err
+	}
+	if err = r.readTemplateFdiffs(); err != nil {
+		return err
+	}
+	if err = r.readTemplateChains(); err != nil {
+		return err
+	}
+
+	flag, err := r.buffer.ReadBool()
+	if err != nil {
+		return err
+	}
+	if flag {
+		return r.readResolutions()
+	}
+	return nil
+}
+
+type nextLayerIdcType int
+
+const (
+	sameLayer nextLayerIdcType = iota
+	nextTemporalLayer
+	nextSpatialLayer
+	noMoreLayer
+	invalidLayer
+)
+
+func (r *DependencyDescriptorReader) readTemplateLayers() error {
+	var (
+		templates             []*FrameDependencyTemplate
+		temporalId, spatialId int
+		nextLayerIdc          nextLayerIdcType
+	)
+	for {
+		if len(templates) == MaxTemplates {
+			return ErrDDReaderTooManyTemplates
+		}
+
+		var lastTemplate FrameDependencyTemplate
+		templates = append(templates, &lastTemplate)
+		lastTemplate.TemporalId = temporalId
+		lastTemplate.SpatialId = spatialId
+
+		idc, err := r.buffer.ReadBits(2)
+		if err != nil {
+			return err
+		}
+		nextLayerIdc = nextLayerIdcType(idc)
+
+		switch nextLayerIdc {
+		case nextTemporalLayer:
+			temporalId++
+			if temporalId >= MaxTemporalIds {
+				return ErrDDReaderTooManyTemporalLayers
+			}
+		case nextSpatialLayer:
+			spatialId++
+			temporalId = 0
+			if spatialId >= MaxSpatialIds {
+				return ErrDDReaderTooManySpatialLayers
+			}
+		}
+
+		if nextLayerIdc == noMoreLayer || !r.buffer.Ok() {
+			break
+		}
+	}
+
+	r.descriptor.AttachedStructure.Templates = templates
+	return nil
+}
+
+func (r *DependencyDescriptorReader) readTemplateDtis() error {
+	structure := r.descriptor.AttachedStructure
+	for _, template := range structure.Templates {
+		if len(template.DecodeTargetIndications) < structure.NumDecodeTargets {
+			template.DecodeTargetIndications = append(template.DecodeTargetIndications, make([]DecodeTargetIndication, structure.NumDecodeTargets-len(template.DecodeTargetIndications))...)
+		} else {
+			template.DecodeTargetIndications = template.DecodeTargetIndications[:structure.NumDecodeTargets]
+		}
+
+		for i := range template.DecodeTargetIndications {
+			indication, err := r.buffer.ReadBits(2)
+			if err != nil {
+				return err
+			}
+			template.DecodeTargetIndications[i] = DecodeTargetIndication(indication)
+		}
+	}
+	return nil
+}
+
+func (r *DependencyDescriptorReader) readTemplateFdiffs() error {
+	for _, template := range r.descriptor.AttachedStructure.Templates {
+		for {
+			fdiffFollow, err := r.buffer.ReadBool()
+			if err != nil {
+				return err
+			}
+			if !fdiffFollow {
+				break
+			}
+			fDiffMinusOne, err := r.buffer.ReadBits(4)
+			if err != nil {
+				return err
+			}
+			template.FrameDiffs = append(template.FrameDiffs, int(fDiffMinusOne+1))
+		}
+	}
+
+	return nil
+}
+
+func (r *DependencyDescriptorReader) readTemplateChains() error {
+	structure := r.descriptor.AttachedStructure
+
+	numChains, err := r.buffer.ReadNonSymmetric(uint32(structure.NumDecodeTargets) + 1)
+	if err != nil {
+		return err
+	}
+	structure.NumChains = int(numChains)
+	if structure.NumChains == 0 {
+		return nil
+	}
+
+	for i := 0; i < structure.NumDecodeTargets; i++ {
+		protectedByChain, err := r.buffer.ReadNonSymmetric(uint32(structure.NumChains))
+		if err != nil {
+			return err
+		}
+		structure.DecodeTargetProtectedByChain = append(structure.DecodeTargetProtectedByChain, int(protectedByChain))
+	}
+
+	for _, frameTemplate := range structure.Templates {
+		for chainId := 0; chainId < structure.NumChains; chainId++ {
+			chainDiff, err := r.buffer.ReadBits(4)
+			if err != nil {
+				return err
+			}
+			frameTemplate.ChainDiffs = append(frameTemplate.ChainDiffs, int(chainDiff))
+		}
+	}
+
+	return nil
+}
+
+func (r *DependencyDescriptorReader) readResolutions() error {
+	structure := r.descriptor.AttachedStructure
+	// The way templates are bitpacked, they are always ordered by spatial_id.
+	spatialLayers := structure.Templates[len(structure.Templates)-1].SpatialId + 1
+	for sid := 0; sid < spatialLayers; sid++ {
+		widthMinus1, err := r.buffer.ReadBits(16)
+		if err != nil {
+			return err
+		}
+		heightMinus1, err := r.buffer.ReadBits(16)
+		if err != nil {
+			return err
+		}
+		structure.Resolutions = append(structure.Resolutions, RenderResolution{
+			Width:  int(widthMinus1 + 1),
+			Height: int(heightMinus1 + 1),
+		})
+	}
+
+	return nil
+}
+
+func (r *DependencyDescriptorReader) readFrameDependencyDefinition() error {
+	templateIndex := (r.frameDependencyTemplateId + MaxTemplates - r.structure.StructureId) % MaxTemplates
+
+	if templateIndex >= len(r.structure.Templates) {
+		r.buffer.Invalidate()
+		return ErrDDReaderInvalidTemplateIndex
+	}
+
+	// Copy all the fields from the matching template
+	r.descriptor.FrameDependencies = r.structure.Templates[templateIndex].Clone()
+
+	if r.customDtisFlag {
+		err := r.readFrameDtis()
+		if err != nil {
+			return err
+		}
+	}
+
+	if r.customFdiffsFlag {
+		err := r.readFrameFdiffs()
+		if err != nil {
+			return err
+		}
+	}
+
+	if r.customChainsFlag {
+		err := r.readFrameChains()
+		if err != nil {
+			return err
+		}
+	}
+
+	if len(r.structure.Resolutions) == 0 {
+		r.descriptor.Resolution = nil
+	} else {
+		// Format guarantees that if there were resolutions in the last structure,
+		// then each spatial layer got one.
+		if r.descriptor.FrameDependencies.SpatialId >= len(r.structure.Resolutions) {
+			r.buffer.Invalidate()
+			return ErrDDReaderInvalidSpatialLayer
+		}
+		res := r.structure.Resolutions[r.descriptor.FrameDependencies.SpatialId]
+		r.descriptor.Resolution = &res
+	}
+
+	return nil
+}
+
+func (r *DependencyDescriptorReader) readFrameDtis() error {
+	if len(r.descriptor.FrameDependencies.DecodeTargetIndications) != r.structure.NumDecodeTargets {
+		return ErrDDReaderNumDTIMismatch
+	}
+
+	for i := range r.descriptor.FrameDependencies.DecodeTargetIndications {
+		indication, err := r.buffer.ReadBits(2)
+		if err != nil {
+			return err
+		}
+		r.descriptor.FrameDependencies.DecodeTargetIndications[i] = DecodeTargetIndication(indication)
+	}
+	return nil
+}
+
+func (r *DependencyDescriptorReader) readFrameFdiffs() error {
+	r.descriptor.FrameDependencies.FrameDiffs = r.descriptor.FrameDependencies.FrameDiffs[:0]
+	for {
+		nexFdiffSize, err := r.buffer.ReadBits(2)
+		if err != nil {
+			return err
+		}
+		if nexFdiffSize == 0 {
+			break
+		}
+		fDiffMinusOne, err := r.buffer.ReadBits(int(nexFdiffSize * 4))
+		if err != nil {
+			return err
+		}
+		r.descriptor.FrameDependencies.FrameDiffs = append(r.descriptor.FrameDependencies.FrameDiffs, int(fDiffMinusOne+1))
+	}
+
+	return nil
+}
+
+func (r *DependencyDescriptorReader) readFrameChains() error {
+	if len(r.descriptor.FrameDependencies.ChainDiffs) != r.structure.NumChains {
+		return ErrDDReaderNumChainDiffsMismatch
+	}
+
+	for i := range r.descriptor.FrameDependencies.ChainDiffs {
+		chainDiff, err := r.buffer.ReadBits(8)
+		if err != nil {
+			return err
+		}
+		r.descriptor.FrameDependencies.ChainDiffs[i] = int(chainDiff)
+	}
+	return nil
+}

pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorwriter.go

@@ -0,0 +1,500 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dependencydescriptor
+
+import (
+	"fmt"
+	"slices"
+)
+
+type TemplateMatch struct {
+	TemplateIdx      int
+	NeedCustomDtis   bool
+	NeedCustomFdiffs bool
+	NeedCustomChains bool
+	// Size in bits to store frame-specific details, i.e.
+	// excluding mandatory fields and template dependency structure.
+	ExtraSizeBits int
+}
+
+type DependencyDescriptorWriter struct {
+	descriptor   *DependencyDescriptor
+	structure    *FrameDependencyStructure
+	activeChains uint32
+	writer       *BitStreamWriter
+	bestTemplate TemplateMatch
+}
+
+func NewDependencyDescriptorWriter(buf []byte, structure *FrameDependencyStructure, activeChains uint32, descriptor *DependencyDescriptor) (*DependencyDescriptorWriter, error) {
+	writer := NewBitStreamWriter(buf)
+	w := &DependencyDescriptorWriter{
+		descriptor:   descriptor,
+		structure:    structure,
+		activeChains: activeChains,
+		writer:       writer,
+	}
+	return w, w.findBestTemplate()
+}
+
+func (w *DependencyDescriptorWriter) ResetBuf(buf []byte) {
+	w.writer = NewBitStreamWriter(buf)
+}
+
+func (w *DependencyDescriptorWriter) Write() error {
+	if err := w.findBestTemplate(); err != nil {
+		return err
+	}
+
+	if err := w.writeMandatoryFields(); err != nil {
+		return err
+	}
+
+	if w.hasExtendedFields() {
+		if err := w.writeExtendedFields(); err != nil {
+			return err
+		}
+
+		if err := w.writeFrameDependencyDefinition(); err != nil {
+			return err
+		}
+	}
+
+	remainingBits := w.writer.RemainingBits()
+	// Zero remaining memory to avoid leaving it uninitialized.
+	if remainingBits%64 != 0 {
+		if err := w.writeBits(0, remainingBits%64); err != nil {
+			return err
+		}
+	}
+
+	for i := 0; i < remainingBits/64; i++ {
+		if err := w.writeBits(0, 64); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (w *DependencyDescriptorWriter) findBestTemplate() error {
+	// Find templates with same spatial and temporal layer of frame dependency.
+	var (
+		firstSameLayer                      *FrameDependencyTemplate
+		firstSameLayerIdx, lastSameLayerIdx int
+	)
+	for i, t := range w.structure.Templates {
+		if w.descriptor.FrameDependencies.SpatialId == t.SpatialId &&
+			w.descriptor.FrameDependencies.TemporalId == t.TemporalId {
+			firstSameLayer = t
+			firstSameLayerIdx = i
+			break
+		}
+	}
+
+	if firstSameLayer == nil {
+		return fmt.Errorf("no template found for spatial layer %d and temporal layer %d", w.descriptor.FrameDependencies.SpatialId, w.descriptor.FrameDependencies.TemporalId)
+	}
+
+	for i, t := range w.structure.Templates[firstSameLayerIdx:] {
+		if w.descriptor.FrameDependencies.SpatialId != t.SpatialId ||
+			w.descriptor.FrameDependencies.TemporalId != t.TemporalId {
+			lastSameLayerIdx = i + firstSameLayerIdx
+		}
+	}
+
+	// Search if there any better template that have small extra size.
+	w.bestTemplate = w.calculateMatch(firstSameLayerIdx, firstSameLayer)
+	for i := firstSameLayerIdx + 1; i <= lastSameLayerIdx; i++ {
+		t := w.structure.Templates[i]
+		match := w.calculateMatch(i, t)
+		if match.ExtraSizeBits < w.bestTemplate.ExtraSizeBits {
+			w.bestTemplate = match
+		}
+	}
+	return nil
+}
+
+func (w *DependencyDescriptorWriter) calculateMatch(idx int, template *FrameDependencyTemplate) TemplateMatch {
+	var result TemplateMatch
+	result.TemplateIdx = idx
+	result.NeedCustomFdiffs = w.descriptor.FrameDependencies.FrameDiffs != nil && !slices.Equal(w.descriptor.FrameDependencies.FrameDiffs, template.FrameDiffs)
+	result.NeedCustomDtis = w.descriptor.FrameDependencies.DecodeTargetIndications != nil && !slices.Equal(w.descriptor.FrameDependencies.DecodeTargetIndications, template.DecodeTargetIndications)
+
+	for i := 0; i < w.structure.NumChains; i++ {
+		if w.activeChains&(1<<i) != 0 && (len(w.descriptor.FrameDependencies.ChainDiffs) <= i || len(template.ChainDiffs) <= i || w.descriptor.FrameDependencies.ChainDiffs[i] != template.ChainDiffs[i]) {
+			result.NeedCustomChains = true
+			break
+		}
+	}
+
+	if result.NeedCustomFdiffs {
+		result.ExtraSizeBits = 2 * (1 + len(w.descriptor.FrameDependencies.FrameDiffs))
+
+		for _, fdiff := range w.descriptor.FrameDependencies.FrameDiffs {
+			if fdiff <= (1 << 4) {
+				result.ExtraSizeBits += 4
+			} else if fdiff <= (1 << 8) {
+				result.ExtraSizeBits += 8
+			} else {
+				result.ExtraSizeBits += 12
+			}
+		}
+	}
+
+	if result.NeedCustomDtis {
+		result.ExtraSizeBits += 2 * len(w.descriptor.FrameDependencies.DecodeTargetIndications)
+	}
+	if result.NeedCustomChains {
+		result.ExtraSizeBits += 8 * w.structure.NumChains
+	}
+
+	return result
+}
+
+func (w *DependencyDescriptorWriter) writeMandatoryFields() error {
+	if err := w.writeBool(w.descriptor.FirstPacketInFrame); err != nil {
+		return err
+	}
+
+	if err := w.writeBool(w.descriptor.LastPacketInFrame); err != nil {
+		return err
+	}
+
+	templateId := (w.bestTemplate.TemplateIdx + w.structure.StructureId) % MaxTemplates
+
+	if err := w.writeBits(uint64(templateId), 6); err != nil {
+		return err
+	}
+
+	return w.writeBits(uint64(w.descriptor.FrameNumber), 16)
+}
+
+func (w *DependencyDescriptorWriter) writeBool(val bool) error {
+	v := uint64(0)
+	if val {
+		v = 1
+	}
+	return w.writer.WriteBits(v, 1)
+}
+
+func (w *DependencyDescriptorWriter) writeBits(val uint64, bitCount int) error {
+	if err := w.writer.WriteBits(val, bitCount); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (w *DependencyDescriptorWriter) hasExtendedFields() bool {
+	return w.bestTemplate.ExtraSizeBits > 0 || w.descriptor.AttachedStructure != nil || w.descriptor.ActiveDecodeTargetsBitmask != nil
+}
+
+func (w *DependencyDescriptorWriter) writeExtendedFields() error {
+	// template_dependency_structure_present_flag
+	if err := w.writeBool(w.descriptor.AttachedStructure != nil); err != nil {
+		return err
+	}
+
+	// active_decode_targets_present_flag
+	activeDecodeTargetsPresentFlag := w.shouldWriteActiveDecodeTargetsBitmask()
+	if err := w.writeBool(activeDecodeTargetsPresentFlag); err != nil {
+		return err
+	}
+
+	// need_custom_dtis
+	if err := w.writeBool(w.bestTemplate.NeedCustomDtis); err != nil {
+		return err
+	}
+
+	// need_custom_fdiffs
+	if err := w.writeBool(w.bestTemplate.NeedCustomFdiffs); err != nil {
+		return err
+	}
+
+	// need_custom_chains
+	if err := w.writeBool(w.bestTemplate.NeedCustomChains); err != nil {
+		return err
+	}
+
+	// template_dependency_structure
+	if w.descriptor.AttachedStructure != nil {
+		if err := w.writeTemplateDependencyStructure(); err != nil {
+			return err
+		}
+	}
+
+	// active_decode_targets_bitmask
+	if activeDecodeTargetsPresentFlag {
+		if err := w.writeBits(uint64(*w.descriptor.ActiveDecodeTargetsBitmask), w.structure.NumDecodeTargets); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (w *DependencyDescriptorWriter) writeTemplateDependencyStructure() error {
+	if w.structure.StructureId < 0 || w.structure.StructureId >= MaxTemplates ||
+		w.structure.NumDecodeTargets <= 0 || w.structure.NumDecodeTargets > MaxDecodeTargets {
+		return fmt.Errorf("invalid arguments, structureId: %d, numDecodeTargets: %d", w.structure.StructureId, w.structure.NumDecodeTargets)
+	}
+
+	if err := w.writeBits(uint64(w.structure.StructureId), 6); err != nil {
+		return err
+	}
+
+	if err := w.writeBits(uint64(w.structure.NumDecodeTargets-1), 5); err != nil {
+		return err
+	}
+
+	if err := w.writeTemplateLayers(); err != nil {
+		return err
+	}
+
+	if err := w.writeTemplateDtis(); err != nil {
+		return err
+	}
+
+	if err := w.writeTemplateFdiffs(); err != nil {
+		return err
+	}
+
+	if err := w.writeTemplateChains(); err != nil {
+		return err
+	}
+
+	hasResolutions := len(w.structure.Resolutions) > 0
+	if err := w.writeBool(hasResolutions); err != nil {
+		return err
+	}
+	return w.writeResolutions()
+}
+
+func (w *DependencyDescriptorWriter) writeTemplateLayers() error {
+	if len(w.structure.Templates) <= 0 || len(w.structure.Templates) > MaxTemplates ||
+		w.structure.Templates[0].SpatialId != 0 || w.structure.Templates[0].TemporalId != 0 {
+		return fmt.Errorf("invalid templates, len %d, templates[0]: spatialId %d, temporalId %d", len(w.structure.Templates), w.structure.Templates[0].SpatialId, w.structure.Templates[0].TemporalId)
+	}
+	for i := 1; i < len(w.structure.Templates); i++ {
+		nextLayerIdc := getNextLayerIdc(w.structure.Templates[i-1], w.structure.Templates[i])
+		if nextLayerIdc >= 3 {
+			return fmt.Errorf("invalid next_layer_idc %d", nextLayerIdc)
+		}
+		if err := w.writeBits(uint64(nextLayerIdc), 2); err != nil {
+			return err
+		}
+	}
+	return w.writeBits(uint64(noMoreLayer), 2)
+}
+
+func getNextLayerIdc(prevTemplate, nextTemplate *FrameDependencyTemplate) nextLayerIdcType {
+	if nextTemplate.SpatialId == prevTemplate.SpatialId && nextTemplate.TemporalId == prevTemplate.TemporalId {
+		return sameLayer
+	} else if nextTemplate.SpatialId == prevTemplate.SpatialId && nextTemplate.TemporalId == prevTemplate.TemporalId+1 {
+		return nextTemporalLayer
+	} else if nextTemplate.SpatialId == prevTemplate.SpatialId+1 && nextTemplate.TemporalId == 0 {
+		return nextSpatialLayer
+	}
+
+	return invalidLayer
+}
+
+func (w *DependencyDescriptorWriter) writeTemplateDtis() error {
+	for _, t := range w.structure.Templates {
+		for _, dti := range t.DecodeTargetIndications {
+			if err := w.writeBits(uint64(dti), 2); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (w *DependencyDescriptorWriter) writeTemplateFdiffs() error {
+	for _, t := range w.structure.Templates {
+		for _, fdiff := range t.FrameDiffs {
+			if err := w.writeBits(uint64(1<<4)|uint64(fdiff-1), 1+4); err != nil {
+				return err
+			}
+		}
+		// no more fdiffs for this template
+		if err := w.writeBits(uint64(0), 1); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (w *DependencyDescriptorWriter) writeTemplateChains() error {
+	if err := w.writeNonSymmetric(uint32(w.structure.NumChains), uint32(w.structure.NumDecodeTargets+1)); err != nil {
+		return err
+	}
+
+	if w.structure.NumChains == 0 {
+		return nil
+	}
+
+	for _, protectedBy := range w.structure.DecodeTargetProtectedByChain {
+		if err := w.writeNonSymmetric(uint32(protectedBy), uint32(w.structure.NumChains)); err != nil {
+			return err
+		}
+	}
+
+	for _, t := range w.structure.Templates {
+		for _, chainDiff := range t.ChainDiffs {
+			if err := w.writeBits(uint64(chainDiff), 4); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (w *DependencyDescriptorWriter) writeNonSymmetric(value, numValues uint32) error {
+	return w.writer.WriteNonSymmetric(value, numValues)
+}
+
+func (w *DependencyDescriptorWriter) writeResolutions() error {
+	for _, res := range w.structure.Resolutions {
+		if err := w.writeBits(uint64(res.Width)-1, 16); err != nil {
+			return err
+		}
+		if err := w.writeBits(uint64(res.Height)-1, 16); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (w *DependencyDescriptorWriter) writeFrameDependencyDefinition() error {
+	if w.bestTemplate.NeedCustomDtis {
+		if err := w.writeFrameDtis(); err != nil {
+			return err
+		}
+	}
+
+	if w.bestTemplate.NeedCustomFdiffs {
+		if err := w.writeFrameFdiffs(); err != nil {
+			return err
+		}
+	}
+
+	if w.bestTemplate.NeedCustomChains {
+		if err := w.writeFrameChains(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (w *DependencyDescriptorWriter) writeFrameDtis() error {
+	for _, dti := range w.descriptor.FrameDependencies.DecodeTargetIndications {
+		if err := w.writeBits(uint64(dti), 2); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (w *DependencyDescriptorWriter) writeFrameFdiffs() error {
+	for _, fdiff := range w.descriptor.FrameDependencies.FrameDiffs {
+		if fdiff <= (1 << 4) {
+			if err := w.writeBits(uint64(1<<4)|uint64(fdiff-1), 2+4); err != nil {
+				return err
+			}
+		} else if fdiff <= (1 << 8) {
+			if err := w.writeBits(uint64(2<<8)|uint64(fdiff-1), 2+8); err != nil {
+				return err
+			}
+		} else { // fdiff <= (1<<12)
+			if err := w.writeBits(uint64(3<<12)|uint64(fdiff-1), 2+12); err != nil {
+				return err
+			}
+		}
+	}
+	// no more fdiffs
+	return w.writeBits(uint64(0), 2)
+}
+
+func (w *DependencyDescriptorWriter) writeFrameChains() error {
+	for i := 0; i < w.structure.NumChains; i++ {
+		chainDiff := 0
+		if w.activeChains&(1<<i) != 0 {
+			chainDiff = w.descriptor.FrameDependencies.ChainDiffs[i]
+		}
+		if err := w.writeBits(uint64(chainDiff), 8); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+const mandatoryFieldSize = 1 + 1 + 6 + 16
+
+func (w *DependencyDescriptorWriter) ValueSizeBits() int {
+	valueSizeBits := mandatoryFieldSize + w.bestTemplate.ExtraSizeBits
+	if w.hasExtendedFields() {
+		valueSizeBits += 5
+		if w.descriptor.AttachedStructure != nil {
+			valueSizeBits += w.structureSizeBits()
+		}
+		if w.shouldWriteActiveDecodeTargetsBitmask() {
+			valueSizeBits += w.structure.NumDecodeTargets
+		}
+	}
+
+	return valueSizeBits
+}
+
+func (w *DependencyDescriptorWriter) shouldWriteActiveDecodeTargetsBitmask() bool {
+	if w.descriptor.ActiveDecodeTargetsBitmask == nil {
+		return false
+	}
+
+	allDecodeTargetsBitmask := (uint64(1) << w.structure.NumDecodeTargets) - 1
+	if w.descriptor.AttachedStructure != nil && uint64(*w.descriptor.ActiveDecodeTargetsBitmask) == allDecodeTargetsBitmask {
+		return false
+	}
+
+	return true
+}
+
+func (w *DependencyDescriptorWriter) structureSizeBits() int {
+	// template_id offset (6 bits) and number of decode targets (5 bits)
+	bits := 11
+	// template layers
+	bits += 2 * len(w.structure.Templates)
+	// dtis
+	bits += 2 * len(w.structure.Templates) * w.structure.NumDecodeTargets
+	// fdiffs. each templates uses 1 + 5 * sizeof(fdiff) bits.
+	bits += len(w.structure.Templates)
+	for _, t := range w.structure.Templates {
+		bits += 5 * len(t.FrameDiffs)
+	}
+	bits += SizeNonSymmetricBits(uint32(w.structure.NumChains), uint32(w.structure.NumDecodeTargets+1))
+	if w.structure.NumChains > 0 {
+		for _, protectedBy := range w.structure.DecodeTargetProtectedByChain {
+			bits += SizeNonSymmetricBits(uint32(protectedBy), uint32(w.structure.NumChains))
+		}
+		bits += 4 * len(w.structure.Templates) * w.structure.NumChains
+	}
+
+	// resolutions
+	bits += 1 + 32*len(w.structure.Resolutions)
+
+	return bits
+}

pkg/sfu/rtpextension/playoutdelay/playoutdelay.go

@@ -0,0 +1,72 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package playoutdelay
+
+import (
+	"encoding/binary"
+	"errors"
+)
+
+const (
+	PlayoutDelayURI        = "http://www.webrtc.org/experiments/rtp-hdrext/playout-delay"
+	MaxPlayoutDelayDefault = 10000            // 10s, equal to chrome's default max playout delay
+	PlayoutDelayMaxValue   = 10 * (1<<12 - 1) // max value for playout delay can be represented
+
+	playoutDelayExtensionSize = 3
+)
+
+var (
+	errPlayoutDelayOverflow = errors.New("playout delay overflow")
+	errTooSmall             = errors.New("buffer too small")
+)
+
+//  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// |  ID   | len=2 |       MIN delay       |       MAX delay       |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// The wired MIN/MAX delay is in 10ms unit
+
+type PlayOutDelay struct {
+	Min, Max uint16 // delay in ms
+}
+
+func PlayoutDelayFromValue(min, max uint16) PlayOutDelay {
+	if min > PlayoutDelayMaxValue {
+		min = PlayoutDelayMaxValue
+	}
+	if max > PlayoutDelayMaxValue {
+		max = PlayoutDelayMaxValue
+	}
+	return PlayOutDelay{Min: min, Max: max}
+}
+
+func (p PlayOutDelay) Marshal() ([]byte, error) {
+	min, max := p.Min/10, p.Max/10
+	if min >= 1<<12 || max >= 1<<12 {
+		return nil, errPlayoutDelayOverflow
+	}
+
+	return []byte{byte(min >> 4), byte(min<<4) | byte(max>>8), byte(max)}, nil
+}
+
+func (p *PlayOutDelay) Unmarshal(rawData []byte) error {
+	if len(rawData) < playoutDelayExtensionSize {
+		return errTooSmall
+	}
+
+	p.Min = (binary.BigEndian.Uint16(rawData) >> 4) * 10
+	p.Max = (binary.BigEndian.Uint16(rawData[1:]) & 0x0FFF) * 10
+	return nil
+}

pkg/sfu/rtpextension/playoutdelay/playoutdelay_test.go

@@ -0,0 +1,57 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package playoutdelay
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestPlayoutDelay(t *testing.T) {
+	p1 := PlayOutDelay{Min: 100, Max: 200}
+	b, err := p1.Marshal()
+	require.NoError(t, err)
+	require.Len(t, b, playoutDelayExtensionSize)
+	var p2 PlayOutDelay
+	err = p2.Unmarshal(b)
+	require.NoError(t, err)
+	require.Equal(t, p1, p2)
+
+	// overflow
+	p3 := PlayOutDelay{Min: 100, Max: (1 << 12) * 10}
+	_, err = p3.Marshal()
+	require.ErrorIs(t, err, errPlayoutDelayOverflow)
+
+	// too small
+	p4 := PlayOutDelay{}
+	err = p4.Unmarshal([]byte{0x00, 0x00})
+	require.ErrorIs(t, err, errTooSmall)
+
+	// from value
+	p5 := PlayoutDelayFromValue(1<<12*10, 1<<12*10+10)
+	_, err = p5.Marshal()
+	require.NoError(t, err)
+	require.Equal(t, uint16((1<<12)-1)*10, p5.Min)
+	require.Equal(t, uint16((1<<12)-1)*10, p5.Max)
+
+	p6 := PlayOutDelay{Min: 100, Max: PlayoutDelayMaxValue}
+	bytes, err := p6.Marshal()
+	require.NoError(t, err)
+	p6Unmarshal := PlayOutDelay{}
+	err = p6Unmarshal.Unmarshal(bytes)
+	require.NoError(t, err)
+	require.Equal(t, p6, p6Unmarshal)
+}

pkg/sfu/rtpmunger.go

@@ -0,0 +1,353 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sfu
+
+import (
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/sfu/utils"
+)
+
+// RTPMunger
+type SequenceNumberOrdering int
+
+const (
+	SequenceNumberOrderingContiguous SequenceNumberOrdering = iota
+	SequenceNumberOrderingOutOfOrder
+	SequenceNumberOrderingGap
+	SequenceNumberOrderingDuplicate
+)
+
+const (
+	RtxGateWindow = 2000
+)
+
+type TranslationParamsRTP struct {
+	snOrdering        SequenceNumberOrdering
+	extSequenceNumber uint64
+	extTimestamp      uint64
+}
+
+type SnTs struct {
+	extSequenceNumber uint64
+	extTimestamp      uint64
+}
+
+// ----------------------------------------------------------------------
+
+type RTPMunger struct {
+	logger logger.Logger
+
+	extHighestIncomingSN uint64
+	snRangeMap           *utils.RangeMap[uint64, uint64]
+
+	extLastSN       uint64
+	extSecondLastSN uint64
+	snOffset        uint64
+
+	extLastTS       uint64
+	extSecondLastTS uint64
+	tsOffset        uint64
+
+	lastMarker       bool
+	secondLastMarker bool
+
+	extRtxGateSn      uint64
+	isInRtxGateRegion bool
+}
+
+func NewRTPMunger(logger logger.Logger) *RTPMunger {
+	return &RTPMunger{
+		logger:     logger,
+		snRangeMap: utils.NewRangeMap[uint64, uint64](100),
+	}
+}
+
+func (r *RTPMunger) DebugInfo() map[string]any {
+	return map[string]any{
+		"ExtHighestIncomingSN": r.extHighestIncomingSN,
+		"ExtLastSN":            r.extLastSN,
+		"ExtSecondLastSN":      r.extSecondLastSN,
+		"SNOffset":             r.snOffset,
+		"ExtLastTS":            r.extLastTS,
+		"ExtSecondLastTS":      r.extSecondLastTS,
+		"TSOffset":             r.tsOffset,
+		"LastMarker":           r.lastMarker,
+		"SecondLastMarker":     r.secondLastMarker,
+	}
+}
+
+func (r *RTPMunger) GetState() *livekit.RTPMungerState {
+	return &livekit.RTPMungerState{
+		ExtLastSequenceNumber:       r.extLastSN,
+		ExtSecondLastSequenceNumber: r.extSecondLastSN,
+		ExtLastTimestamp:            r.extLastTS,
+		ExtSecondLastTimestamp:      r.extSecondLastTS,
+		LastMarker:                  r.lastMarker,
+		SecondLastMarker:            r.secondLastMarker,
+	}
+}
+
+func (r *RTPMunger) GetExtLastTimestamp() uint64 {
+	return r.extLastTS
+}
+
+func (r *RTPMunger) GetTSOffset() uint64 {
+	return r.tsOffset
+}
+
+func (r *RTPMunger) SeedState(state *livekit.RTPMungerState) {
+	r.extLastSN = state.ExtLastSequenceNumber
+	r.extSecondLastSN = state.ExtSecondLastSequenceNumber
+	r.extLastTS = state.ExtLastTimestamp
+	r.extSecondLastTS = state.ExtSecondLastTimestamp
+	r.lastMarker = state.LastMarker
+	r.secondLastMarker = state.SecondLastMarker
+}
+
+func (r *RTPMunger) SetLastSnTs(extPkt *buffer.ExtPacket) {
+	r.extHighestIncomingSN = extPkt.ExtSequenceNumber - 1
+
+	r.extLastSN = extPkt.ExtSequenceNumber
+	r.extSecondLastSN = r.extLastSN - 1
+	r.snRangeMap.ClearAndResetValue(extPkt.ExtSequenceNumber, 0)
+	r.updateSnOffset()
+
+	r.extLastTS = extPkt.ExtTimestamp
+	r.extSecondLastTS = extPkt.ExtTimestamp
+	r.tsOffset = 0
+}
+
+func (r *RTPMunger) UpdateSnTsOffsets(extPkt *buffer.ExtPacket, snAdjust uint64, tsAdjust uint64) {
+	r.extHighestIncomingSN = extPkt.ExtSequenceNumber - 1
+
+	r.snRangeMap.ClearAndResetValue(extPkt.ExtSequenceNumber, extPkt.ExtSequenceNumber-r.extLastSN-snAdjust)
+	r.updateSnOffset()
+
+	r.tsOffset = extPkt.ExtTimestamp - r.extLastTS - tsAdjust
+}
+
+func (r *RTPMunger) PacketDropped(extPkt *buffer.ExtPacket) {
+	if r.extHighestIncomingSN != extPkt.ExtSequenceNumber {
+		return
+	}
+
+	snOffset, err := r.snRangeMap.GetValue(extPkt.ExtSequenceNumber)
+	if err == nil {
+		outSN := extPkt.ExtSequenceNumber - snOffset
+		if outSN != r.extLastSN {
+			r.logger.Warnw("last outgoing sequence number mismatch", nil, "expected", r.extLastSN, "got", outSN)
+		}
+	}
+	if r.extLastSN == r.extSecondLastSN {
+		r.logger.Warnw("cannot roll back on drop", nil, "extLastSN", r.extLastSN, "secondLastSN", r.extSecondLastSN)
+	}
+
+	if err := r.snRangeMap.ExcludeRange(r.extHighestIncomingSN, r.extHighestIncomingSN+1); err != nil {
+		r.logger.Errorw("could not exclude range", err, "sn", r.extHighestIncomingSN)
+	}
+
+	r.extLastSN = r.extSecondLastSN
+	r.updateSnOffset()
+
+	r.extLastTS = r.extSecondLastTS
+	r.lastMarker = r.secondLastMarker
+}
+
+func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (TranslationParamsRTP, error) {
+	diff := int64(extPkt.ExtSequenceNumber - r.extHighestIncomingSN)
+	if (diff == 1 && len(extPkt.Packet.Payload) != 0) || diff > 1 {
+		// in-order - either contiguous packet with payload OR packet following a gap, may or may not have payload
+		r.extHighestIncomingSN = extPkt.ExtSequenceNumber
+
+		ordering := SequenceNumberOrderingContiguous
+		if diff > 1 {
+			ordering = SequenceNumberOrderingGap
+		}
+
+		extMungedSN := extPkt.ExtSequenceNumber - r.snOffset
+		extMungedTS := extPkt.ExtTimestamp - r.tsOffset
+
+		r.extSecondLastSN = r.extLastSN
+		r.extLastSN = extMungedSN
+		r.extSecondLastTS = r.extLastTS
+		r.extLastTS = extMungedTS
+		r.secondLastMarker = r.lastMarker
+		r.lastMarker = marker
+
+		if extPkt.IsKeyFrame {
+			r.extRtxGateSn = extMungedSN
+			r.isInRtxGateRegion = true
+		}
+
+		if r.isInRtxGateRegion && (extMungedSN-r.extRtxGateSn) > RtxGateWindow {
+			r.isInRtxGateRegion = false
+		}
+
+		return TranslationParamsRTP{
+			snOrdering:        ordering,
+			extSequenceNumber: extMungedSN,
+			extTimestamp:      extMungedTS,
+		}, nil
+	}
+
+	if diff < 0 {
+		// out-of-order, look up sequence number offset cache
+		snOffset, err := r.snRangeMap.GetValue(extPkt.ExtSequenceNumber)
+		if err != nil {
+			return TranslationParamsRTP{
+				snOrdering: SequenceNumberOrderingOutOfOrder,
+			}, errOutOfOrderSequenceNumberCacheMiss
+		}
+
+		extSequenceNumber := extPkt.ExtSequenceNumber - snOffset
+		if extSequenceNumber >= r.extLastSN {
+			// should not happen, just being paranoid
+			r.logger.Errorw(
+				"unexpected packet ordering", nil,
+				"extIncomingSN", extPkt.ExtSequenceNumber,
+				"extHighestIncomingSN", r.extHighestIncomingSN,
+				"extLastSN", r.extLastSN,
+				"snOffsetIncoming", snOffset,
+				"snOffsetHighest", r.snOffset,
+			)
+			return TranslationParamsRTP{
+				snOrdering: SequenceNumberOrderingOutOfOrder,
+			}, errOutOfOrderSequenceNumberCacheMiss
+		}
+
+		return TranslationParamsRTP{
+			snOrdering:        SequenceNumberOrderingOutOfOrder,
+			extSequenceNumber: extSequenceNumber,
+			extTimestamp:      extPkt.ExtTimestamp - r.tsOffset,
+		}, nil
+	}
+
+	// if padding only packet, can be dropped and sequence number adjusted, if contiguous
+	if diff == 1 {
+		r.extHighestIncomingSN = extPkt.ExtSequenceNumber
+
+		if err := r.snRangeMap.ExcludeRange(r.extHighestIncomingSN, r.extHighestIncomingSN+1); err != nil {
+			r.logger.Errorw("could not exclude range", err, "sn", r.extHighestIncomingSN)
+		}
+
+		r.updateSnOffset()
+
+		return TranslationParamsRTP{
+			snOrdering: SequenceNumberOrderingContiguous,
+		}, errPaddingOnlyPacket
+	}
+
+	// can get duplicate packet due to FEC
+	return TranslationParamsRTP{
+		snOrdering: SequenceNumberOrderingDuplicate,
+	}, errDuplicatePacket
+}
+
+func (r *RTPMunger) FilterRTX(nacks []uint16) []uint16 {
+	if !r.isInRtxGateRegion {
+		return nacks
+	}
+
+	filtered := make([]uint16, 0, len(nacks))
+	for _, sn := range nacks {
+		if (sn - uint16(r.extRtxGateSn)) < (1 << 15) {
+			filtered = append(filtered, sn)
+		}
+	}
+
+	return filtered
+}
+
+func (r *RTPMunger) UpdateAndGetPaddingSnTs(
+	num int,
+	clockRate uint32,
+	frameRate uint32,
+	forceMarker bool,
+	extRtpTimestamp uint64,
+) ([]SnTs, error) {
+	if num == 0 {
+		return nil, nil
+	}
+
+	useLastTSForFirst := false
+	tsOffset := 0
+	if !r.lastMarker {
+		if !forceMarker {
+			return nil, errPaddingNotOnFrameBoundary
+		}
+
+		// if forcing frame end, use timestamp of latest received frame for the first one
+		useLastTSForFirst = true
+		tsOffset = 1
+	}
+
+	extLastSN := r.extLastSN
+	extLastTS := r.extLastTS
+	vals := make([]SnTs, num)
+	for i := range num {
+		extLastSN++
+		vals[i].extSequenceNumber = extLastSN
+
+		if frameRate != 0 {
+			if useLastTSForFirst && i == 0 {
+				vals[i].extTimestamp = r.extLastTS
+			} else {
+				ets := extRtpTimestamp + uint64(((uint32(i+1-tsOffset)*clockRate)+frameRate-1)/frameRate)
+				if int64(ets-extLastTS) <= 0 {
+					ets = extLastTS + 1
+				}
+				extLastTS = ets
+				vals[i].extTimestamp = ets
+			}
+		} else {
+			vals[i].extTimestamp = r.extLastTS
+		}
+	}
+
+	r.extSecondLastSN = extLastSN - 1
+	r.extLastSN = extLastSN
+	r.snRangeMap.DecValue(r.extHighestIncomingSN, uint64(num))
+	r.updateSnOffset()
+
+	if len(vals) == 1 {
+		r.extSecondLastTS = r.extLastTS
+	} else {
+		r.extSecondLastTS = vals[len(vals)-2].extTimestamp
+	}
+	r.tsOffset -= extLastTS - r.extLastTS
+	r.extLastTS = extLastTS
+
+	r.secondLastMarker = r.lastMarker
+	if forceMarker {
+		r.lastMarker = true
+	}
+
+	return vals, nil
+}
+
+func (r *RTPMunger) IsOnFrameBoundary() bool {
+	return r.lastMarker
+}
+
+func (r *RTPMunger) updateSnOffset() {
+	snOffset, err := r.snRangeMap.GetValue(r.extHighestIncomingSN + 1)
+	if err != nil {
+		r.logger.Errorw("could not get sequence number offset", err)
+	}
+	r.snOffset = snOffset
+}

pkg/sfu/rtpmunger_test.go

@@ -0,0 +1,587 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sfu
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/protocol/logger"
+
+	"github.com/livekit/livekit-server/pkg/sfu/testutils"
+)
+
+func newRTPMunger() *RTPMunger {
+	return NewRTPMunger(logger.GetLogger())
+}
+
+func TestSetLastSnTs(t *testing.T) {
+	r := newRTPMunger()
+
+	params := &testutils.TestExtPacketParams{
+		SequenceNumber: 23333,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+	}
+	extPkt, err := testutils.GetTestExtPacket(params)
+	require.NoError(t, err)
+	require.NotNil(t, extPkt)
+
+	r.SetLastSnTs(extPkt)
+	require.Equal(t, uint64(23332), r.extHighestIncomingSN)
+	require.Equal(t, uint64(23333), r.extLastSN)
+	require.Equal(t, uint64(0xabcdef), r.extLastTS)
+	_, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	require.Error(t, err)
+	snOffset, err := r.snRangeMap.GetValue(r.extLastSN)
+	require.NoError(t, err)
+	require.Equal(t, uint64(0), snOffset)
+	require.Equal(t, uint64(0), r.snOffset)
+	require.Equal(t, uint64(0), r.tsOffset)
+}
+
+func TestUpdateSnTsOffsets(t *testing.T) {
+	r := newRTPMunger()
+
+	params := &testutils.TestExtPacketParams{
+		SequenceNumber: 23333,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+	}
+	extPkt, _ := testutils.GetTestExtPacket(params)
+	r.SetLastSnTs(extPkt)
+
+	params = &testutils.TestExtPacketParams{
+		SequenceNumber: 33333,
+		Timestamp:      0xabcdef,
+		SSRC:           0x87654321,
+	}
+	extPkt, _ = testutils.GetTestExtPacket(params)
+	r.UpdateSnTsOffsets(extPkt, 1, 1)
+	require.Equal(t, uint64(33332), r.extHighestIncomingSN)
+	require.Equal(t, uint64(23333), r.extLastSN)
+	require.Equal(t, uint64(0xabcdef), r.extLastTS)
+	_, err := r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	require.Error(t, err)
+	_, err = r.snRangeMap.GetValue(r.extLastSN)
+	require.Error(t, err)
+	require.Equal(t, uint64(9999), r.snOffset)
+	require.Equal(t, uint64(0xffff_ffff_ffff_ffff), r.tsOffset)
+}
+
+func TestPacketDropped(t *testing.T) {
+	r := newRTPMunger()
+
+	params := &testutils.TestExtPacketParams{
+		SequenceNumber: 23333,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+		PayloadSize:    10,
+	}
+	extPkt, _ := testutils.GetTestExtPacket(params)
+	r.SetLastSnTs(extPkt)
+	require.Equal(t, uint64(23332), r.extHighestIncomingSN)
+	require.Equal(t, uint64(23333), r.extLastSN)
+	require.Equal(t, uint64(0xabcdef), r.extLastTS)
+	_, err := r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	require.Error(t, err)
+	snOffset, err := r.snRangeMap.GetValue(r.extLastSN)
+	require.NoError(t, err)
+	require.Equal(t, uint64(0), snOffset)
+	require.Equal(t, uint64(0), r.tsOffset)
+
+	r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) // update sequence number offset
+
+	// drop a non-head packet, should cause no change in internals
+	params = &testutils.TestExtPacketParams{
+		SequenceNumber: 33333,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+	}
+	extPkt, _ = testutils.GetTestExtPacket(params)
+	r.PacketDropped(extPkt)
+	require.Equal(t, uint64(23333), r.extHighestIncomingSN)
+	require.Equal(t, uint64(23333), r.extLastSN)
+	snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	require.NoError(t, err)
+	require.Equal(t, uint64(0), snOffset)
+
+	// drop a head packet and check offset increases
+	params = &testutils.TestExtPacketParams{
+		SequenceNumber: 44444,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+		PayloadSize:    20,
+	}
+	extPkt, _ = testutils.GetTestExtPacket(params)
+
+	r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) // update sequence number offset
+	require.Equal(t, uint64(44444), r.extLastSN)
+
+	r.PacketDropped(extPkt)
+	require.Equal(t, uint64(23333), r.extLastSN)
+	_, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	require.Error(t, err)
+	snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN + 1)
+	require.NoError(t, err)
+	require.Equal(t, uint64(1), snOffset)
+
+	params = &testutils.TestExtPacketParams{
+		SequenceNumber: 44445,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+		PayloadSize:    20,
+	}
+	extPkt, _ = testutils.GetTestExtPacket(params)
+
+	r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) // update sequence number offset
+	require.Equal(t, r.extLastSN, uint64(44444))
+	snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	require.NoError(t, err)
+	require.Equal(t, uint64(1), snOffset)
+}
+
+func TestOutOfOrderSequenceNumber(t *testing.T) {
+	r := newRTPMunger()
+
+	params := &testutils.TestExtPacketParams{
+		SequenceNumber: 23333,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+		PayloadSize:    10,
+	}
+	extPkt, _ := testutils.GetTestExtPacket(params)
+	r.SetLastSnTs(extPkt)
+	r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+
+	// should not be able to add a missing sequence number to the cache that is before start
+	err := r.snRangeMap.ExcludeRange(23332, 23333)
+	require.Error(t, err)
+
+	// out-of-order sequence number before start should miss
+	params = &testutils.TestExtPacketParams{
+		SequenceNumber: 23331,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+		PayloadSize:    10,
+	}
+	extPkt, _ = testutils.GetTestExtPacket(params)
+
+	_, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+	require.Error(t, err)
+
+	var tp TranslationParamsRTP
+	//  add a missing sequence number to the cache
+	err = r.snRangeMap.ExcludeRange(23334, 23335)
+	require.NoError(t, err)
+
+	params = &testutils.TestExtPacketParams{
+		SequenceNumber: 23336,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+		PayloadSize:    10,
+	}
+	extPkt, _ = testutils.GetTestExtPacket(params)
+	r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+
+	// out-of-order sequence number should be munged from cache
+	params = &testutils.TestExtPacketParams{
+		SequenceNumber: 23335,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+		PayloadSize:    10,
+	}
+	extPkt, _ = testutils.GetTestExtPacket(params)
+
+	tpExpected := TranslationParamsRTP{
+		snOrdering:        SequenceNumberOrderingOutOfOrder,
+		extSequenceNumber: 23334,
+		extTimestamp:      0xabcdef,
+	}
+
+	tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+	require.NoError(t, err)
+	require.Equal(t, tpExpected, tp)
+
+	params = &testutils.TestExtPacketParams{
+		SequenceNumber: 23332,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+		PayloadSize:    10,
+	}
+	extPkt, _ = testutils.GetTestExtPacket(params)
+
+	tpExpected = TranslationParamsRTP{
+		snOrdering: SequenceNumberOrderingOutOfOrder,
+	}
+
+	tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+	require.Error(t, err, errOutOfOrderSequenceNumberCacheMiss)
+	require.Equal(t, tpExpected, tp)
+}
+
+func TestDuplicateSequenceNumber(t *testing.T) {
+	r := newRTPMunger()
+
+	params := &testutils.TestExtPacketParams{
+		SequenceNumber: 23333,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+	}
+	extPkt, _ := testutils.GetTestExtPacket(params)
+	r.SetLastSnTs(extPkt)
+
+	// send first packet through
+	r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+
+	// send it again - duplicate packet
+	tpExpected := TranslationParamsRTP{
+		snOrdering: SequenceNumberOrderingDuplicate,
+	}
+
+	tp, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+	require.ErrorIs(t, err, errDuplicatePacket)
+	require.Equal(t, tpExpected, tp)
+}
+
+func TestPaddingOnlyPacket(t *testing.T) {
+	r := newRTPMunger()
+
+	params := &testutils.TestExtPacketParams{
+		SequenceNumber: 23333,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+	}
+	extPkt, _ := testutils.GetTestExtPacket(params)
+	r.SetLastSnTs(extPkt)
+
+	// contiguous padding only packet should report an error
+	tpExpected := TranslationParamsRTP{
+		snOrdering: SequenceNumberOrderingContiguous,
+	}
+
+	tp, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+	require.Error(t, err)
+	require.ErrorIs(t, err, errPaddingOnlyPacket)
+	require.Equal(t, tpExpected, tp)
+	require.Equal(t, uint64(23333), r.extHighestIncomingSN)
+	require.Equal(t, uint64(23333), r.extLastSN)
+	_, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	require.Error(t, err)
+
+	var snOffset uint64
+	// padding only packet with a gap should not report an error
+	params = &testutils.TestExtPacketParams{
+		SequenceNumber: 23335,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+	}
+	extPkt, _ = testutils.GetTestExtPacket(params)
+
+	tpExpected = TranslationParamsRTP{
+		snOrdering:        SequenceNumberOrderingGap,
+		extSequenceNumber: 23334,
+		extTimestamp:      0xabcdef,
+	}
+
+	tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+	require.NoError(t, err)
+	require.Equal(t, tpExpected, tp)
+	require.Equal(t, uint64(23335), r.extHighestIncomingSN)
+	require.Equal(t, uint64(23334), r.extLastSN)
+	snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	require.NoError(t, err)
+	require.Equal(t, uint64(1), snOffset)
+}
+
+func TestGapInSequenceNumber(t *testing.T) {
+	r := newRTPMunger()
+
+	params := &testutils.TestExtPacketParams{
+		SequenceNumber: 65533,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+		PayloadSize:    33,
+	}
+	extPkt, _ := testutils.GetTestExtPacket(params)
+	r.SetLastSnTs(extPkt)
+
+	_, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+	require.NoError(t, err)
+
+	// three lost packets
+	params = &testutils.TestExtPacketParams{
+		SequenceNumber: 1,
+		SNCycles:       1,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+		PayloadSize:    33,
+	}
+	extPkt, _ = testutils.GetTestExtPacket(params)
+
+	tpExpected := TranslationParamsRTP{
+		snOrdering:        SequenceNumberOrderingGap,
+		extSequenceNumber: 65536 + 1,
+		extTimestamp:      0xabcdef,
+	}
+
+	tp, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+	require.NoError(t, err)
+	require.Equal(t, tpExpected, tp)
+	require.Equal(t, uint64(65536+1), r.extHighestIncomingSN)
+	require.Equal(t, uint64(65536+1), r.extLastSN)
+	snOffset, err := r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	require.NoError(t, err)
+	require.Equal(t, uint64(0), snOffset)
+
+	// ensure missing sequence numbers have correct cached offset
+	for i := uint64(65534); i != 65536+1; i++ {
+		offset, err := r.snRangeMap.GetValue(i)
+		require.NoError(t, err)
+		require.Equal(t, uint64(0), offset)
+	}
+
+	// a padding only packet should be dropped
+	params = &testutils.TestExtPacketParams{
+		SequenceNumber: 2,
+		SNCycles:       1,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+	}
+	extPkt, _ = testutils.GetTestExtPacket(params)
+
+	tpExpected = TranslationParamsRTP{
+		snOrdering: SequenceNumberOrderingContiguous,
+	}
+
+	tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+	require.ErrorIs(t, err, errPaddingOnlyPacket)
+	require.Equal(t, tpExpected, tp)
+	require.Equal(t, uint64(65536+2), r.extHighestIncomingSN)
+	require.Equal(t, uint64(65536+1), r.extLastSN)
+	_, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	require.Error(t, err)
+
+	// a packet with a gap should be adjusting for dropped padding packet
+	params = &testutils.TestExtPacketParams{
+		SequenceNumber: 4,
+		SNCycles:       1,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+		PayloadSize:    22,
+	}
+	extPkt, _ = testutils.GetTestExtPacket(params)
+
+	tpExpected = TranslationParamsRTP{
+		snOrdering:        SequenceNumberOrderingGap,
+		extSequenceNumber: 65536 + 3,
+		extTimestamp:      0xabcdef,
+	}
+
+	tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+	require.NoError(t, err)
+	require.Equal(t, tpExpected, tp)
+	require.Equal(t, uint64(65536+4), r.extHighestIncomingSN)
+	require.Equal(t, uint64(65536+3), r.extLastSN)
+	snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	require.NoError(t, err)
+	require.Equal(t, uint64(1), snOffset)
+
+	// ensure missing sequence number has correct cached offset
+	offset, err := r.snRangeMap.GetValue(65536 + 3)
+	require.NoError(t, err)
+	require.Equal(t, uint64(1), offset)
+
+	// another contiguous padding only packet should be dropped
+	params = &testutils.TestExtPacketParams{
+		SequenceNumber: 5,
+		SNCycles:       1,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+	}
+	extPkt, _ = testutils.GetTestExtPacket(params)
+
+	tpExpected = TranslationParamsRTP{
+		snOrdering: SequenceNumberOrderingContiguous,
+	}
+
+	tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+	require.ErrorIs(t, err, errPaddingOnlyPacket)
+	require.Equal(t, tpExpected, tp)
+	require.Equal(t, uint64(65536+5), r.extHighestIncomingSN)
+	require.Equal(t, uint64(65536+3), r.extLastSN)
+	_, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	require.Error(t, err)
+
+	// a packet with a gap should be adjusting for dropped packets
+	params = &testutils.TestExtPacketParams{
+		SequenceNumber: 7,
+		SNCycles:       1,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+		PayloadSize:    22,
+	}
+	extPkt, _ = testutils.GetTestExtPacket(params)
+
+	tpExpected = TranslationParamsRTP{
+		snOrdering:        SequenceNumberOrderingGap,
+		extSequenceNumber: 65536 + 5,
+		extTimestamp:      0xabcdef,
+	}
+
+	tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+	require.NoError(t, err)
+	require.Equal(t, tpExpected, tp)
+	require.Equal(t, uint64(65536+7), r.extHighestIncomingSN)
+	require.Equal(t, uint64(65536+5), r.extLastSN)
+	snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	require.NoError(t, err)
+	require.Equal(t, uint64(2), snOffset)
+
+	// ensure missing sequence number has correct cached offset
+	offset, err = r.snRangeMap.GetValue(65536 + 3)
+	require.NoError(t, err)
+	require.Equal(t, uint64(1), offset)
+
+	offset, err = r.snRangeMap.GetValue(65536 + 6)
+	require.NoError(t, err)
+	require.Equal(t, uint64(2), offset)
+
+	// check the missing packets
+	params = &testutils.TestExtPacketParams{
+		SequenceNumber: 6,
+		SNCycles:       1,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+	}
+	extPkt, _ = testutils.GetTestExtPacket(params)
+
+	tpExpected = TranslationParamsRTP{
+		snOrdering:        SequenceNumberOrderingOutOfOrder,
+		extSequenceNumber: 65536 + 4,
+		extTimestamp:      0xabcdef,
+	}
+
+	tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+	require.NoError(t, err)
+	require.Equal(t, tpExpected, tp)
+	require.Equal(t, uint64(65536+7), r.extHighestIncomingSN)
+	require.Equal(t, uint64(65536+5), r.extLastSN)
+	snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	require.NoError(t, err)
+	require.Equal(t, uint64(2), snOffset)
+
+	params = &testutils.TestExtPacketParams{
+		SequenceNumber: 3,
+		SNCycles:       1,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+	}
+	extPkt, _ = testutils.GetTestExtPacket(params)
+
+	tpExpected = TranslationParamsRTP{
+		snOrdering:        SequenceNumberOrderingOutOfOrder,
+		extSequenceNumber: 65536 + 2,
+		extTimestamp:      0xabcdef,
+	}
+
+	tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+	require.NoError(t, err)
+	require.Equal(t, tpExpected, tp)
+	require.Equal(t, uint64(65536+7), r.extHighestIncomingSN)
+	require.Equal(t, uint64(65536+5), r.extLastSN)
+	snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	require.NoError(t, err)
+	require.Equal(t, uint64(2), snOffset)
+}
+
+func TestUpdateAndGetPaddingSnTs(t *testing.T) {
+	r := newRTPMunger()
+
+	params := &testutils.TestExtPacketParams{
+		SequenceNumber: 23333,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+		PayloadSize:    20,
+	}
+	extPkt, _ := testutils.GetTestExtPacket(params)
+	r.SetLastSnTs(extPkt)
+
+	// getting padding without forcing marker should fail
+	_, err := r.UpdateAndGetPaddingSnTs(10, 10, 5, false, 0)
+	require.Error(t, err)
+	require.ErrorIs(t, err, errPaddingNotOnFrameBoundary)
+
+	// forcing a marker should not error out.
+	// And timestamp on first padding should be the same as the last one.
+	numPadding := 10
+	clockRate := uint64(10)
+	frameRate := uint64(5)
+	var sntsExpected = make([]SnTs, numPadding)
+	for i := range numPadding {
+		sntsExpected[i] = SnTs{
+			extSequenceNumber: uint64(params.SequenceNumber) + uint64(i) + 1,
+			extTimestamp:      uint64(params.Timestamp) + ((uint64(i)*clockRate)+frameRate-1)/frameRate,
+		}
+	}
+	snts, err := r.UpdateAndGetPaddingSnTs(numPadding, uint32(clockRate), uint32(frameRate), true, extPkt.ExtTimestamp)
+	require.NoError(t, err)
+	require.Equal(t, sntsExpected, snts)
+
+	// now that there is a marker, timestamp should jump on first padding when asked again
+	for i := range numPadding {
+		sntsExpected[i] = SnTs{
+			extSequenceNumber: uint64(params.SequenceNumber) + uint64(len(snts)) + uint64(i) + 1,
+			extTimestamp:      snts[len(snts)-1].extTimestamp + ((uint64(i+1)*clockRate)+frameRate-1)/frameRate,
+		}
+	}
+	snts, err = r.UpdateAndGetPaddingSnTs(numPadding, uint32(clockRate), uint32(frameRate), false, snts[len(snts)-1].extTimestamp)
+	require.NoError(t, err)
+	require.Equal(t, sntsExpected, snts)
+}
+
+func TestIsOnFrameBoundary(t *testing.T) {
+	r := newRTPMunger()
+
+	params := &testutils.TestExtPacketParams{
+		SequenceNumber: 23333,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+		PayloadSize:    20,
+	}
+	extPkt, _ := testutils.GetTestExtPacket(params)
+	r.SetLastSnTs(extPkt)
+
+	// send it through
+	_, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+	require.NoError(t, err)
+	require.False(t, r.IsOnFrameBoundary())
+
+	// packet with RTP marker
+	params = &testutils.TestExtPacketParams{
+		Marker:         true,
+		SequenceNumber: 23334,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+		PayloadSize:    20,
+	}
+	extPkt, _ = testutils.GetTestExtPacket(params)
+
+	// send it through
+	_, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+	require.NoError(t, err)
+	require.True(t, r.IsOnFrameBoundary())
+}

pkg/sfu/rtpstats/rtpstats_base_lite.go

@@ -0,0 +1,557 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtpstats
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils/mono"
+	"go.uber.org/zap/zapcore"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+const (
+	cGapHistogramNumBins = 101
+	cNumSequenceNumbers  = 65536
+	cFirstSnapshotID     = 1
+)
+
+// -------------------------------------------------------
+
+type RTPDeltaInfoLite struct {
+	StartTime         time.Time
+	EndTime           time.Time
+	Packets           uint32
+	Bytes             uint64
+	PacketsLost       uint32
+	PacketsOutOfOrder uint32
+	Nacks             uint32
+}
+
+func (r *RTPDeltaInfoLite) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if r == nil {
+		return nil
+	}
+
+	e.AddTime("StartTime", r.StartTime)
+	e.AddTime("EndTime", r.EndTime)
+	e.AddUint32("Packets", r.Packets)
+	e.AddUint64("Bytes", r.Bytes)
+	e.AddUint32("PacketsLost", r.PacketsLost)
+	e.AddUint32("PacketsOutOfOrder", r.PacketsOutOfOrder)
+	e.AddUint32("Nacks", r.Nacks)
+	return nil
+}
+
+// -------------------------------------------------------
+
+type snapshotLite struct {
+	isValid bool
+
+	startTime int64
+
+	extStartSN uint64
+	bytes      uint64
+
+	packetsOutOfOrder uint64
+
+	packetsLost uint64
+
+	nacks uint32
+}
+
+func (s *snapshotLite) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if s == nil {
+		return nil
+	}
+
+	e.AddBool("isValid", s.isValid)
+	e.AddTime("startTime", time.Unix(0, s.startTime))
+	e.AddUint64("extStartSN", s.extStartSN)
+	e.AddUint64("bytes", s.bytes)
+	e.AddUint64("packetsOutOfOrder", s.packetsOutOfOrder)
+	e.AddUint64("packetsLost", s.packetsLost)
+	e.AddUint32("nacks", s.nacks)
+	return nil
+}
+
+// ------------------------------------------------------------------
+
+type RTPStatsParams struct {
+	IsRTX bool
+}
+
+type rtpStatsBaseLite struct {
+	params    RTPStatsParams
+	clockRate uint32
+	logger    logger.Logger
+
+	lock sync.RWMutex
+
+	initialized bool
+
+	startTime int64
+	endTime   int64
+
+	bytes uint64
+
+	packetsOutOfOrder uint64
+
+	packetsLost uint64
+
+	gapHistogram [cGapHistogramNumBins]uint32
+
+	nacks        uint32
+	nackAcks     uint32
+	nackMisses   uint32
+	nackRepeated uint32
+
+	plis    uint32
+	lastPli int64
+
+	nextSnapshotLiteID uint32
+	snapshotLites      []snapshotLite
+}
+
+func newRTPStatsBaseLite(params RTPStatsParams) *rtpStatsBaseLite {
+	return &rtpStatsBaseLite{
+		params:             params,
+		logger:             logger.GetLogger(),
+		nextSnapshotLiteID: cFirstSnapshotID,
+		snapshotLites:      make([]snapshotLite, 2),
+	}
+}
+
+func (r *rtpStatsBaseLite) seed(from *rtpStatsBaseLite) bool {
+	if from == nil || !from.initialized || r.initialized {
+		return false
+	}
+
+	r.initialized = from.initialized
+
+	r.startTime = from.startTime
+	// do not clone endTime as a non-zero endTime indicates an ended object
+
+	r.bytes = from.bytes
+
+	r.packetsOutOfOrder = from.packetsOutOfOrder
+
+	r.packetsLost = from.packetsLost
+
+	r.gapHistogram = from.gapHistogram
+
+	r.nacks = from.nacks
+	r.nackAcks = from.nackAcks
+	r.nackMisses = from.nackMisses
+	r.nackRepeated = from.nackRepeated
+
+	r.plis = from.plis
+	r.lastPli = from.lastPli
+
+	r.nextSnapshotLiteID = from.nextSnapshotLiteID
+	r.snapshotLites = make([]snapshotLite, cap(from.snapshotLites))
+	copy(r.snapshotLites, from.snapshotLites)
+	return true
+}
+
+func (r *rtpStatsBaseLite) SetClockRate(clockRate uint32) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	r.setClockRateLocked(clockRate)
+}
+
+func (r *rtpStatsBaseLite) setClockRateLocked(clockRate uint32) {
+	r.clockRate = clockRate
+}
+
+func (r *rtpStatsBaseLite) SetLogger(logger logger.Logger) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	r.logger = logger
+}
+
+func (r *rtpStatsBaseLite) Stop() {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	r.endTime = mono.UnixNano()
+}
+
+func (r *rtpStatsBaseLite) newSnapshotLiteID(extStartSN uint64) uint32 {
+	id := r.nextSnapshotLiteID
+	r.nextSnapshotLiteID++
+
+	if cap(r.snapshotLites) < int(r.nextSnapshotLiteID-cFirstSnapshotID) {
+		snapshotLites := make([]snapshotLite, r.nextSnapshotLiteID-cFirstSnapshotID)
+		copy(snapshotLites, r.snapshotLites)
+		r.snapshotLites = snapshotLites
+	}
+
+	if r.initialized {
+		r.snapshotLites[id-cFirstSnapshotID] = initSnapshotLite(mono.UnixNano(), extStartSN)
+	}
+	return id
+}
+
+func (r *rtpStatsBaseLite) IsActive() bool {
+	r.lock.RLock()
+	defer r.lock.RUnlock()
+
+	return r.initialized && r.endTime == 0
+}
+
+func (r *rtpStatsBaseLite) UpdateNack(nackCount uint32) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	if r.endTime != 0 {
+		return
+	}
+
+	r.nacks += nackCount
+}
+
+func (r *rtpStatsBaseLite) UpdateNackProcessed(nackAckCount uint32, nackMissCount uint32, nackRepeatedCount uint32) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	if r.endTime != 0 {
+		return
+	}
+
+	r.nackAcks += nackAckCount
+	r.nackMisses += nackMissCount
+	r.nackRepeated += nackRepeatedCount
+}
+
+func (r *rtpStatsBaseLite) CheckAndUpdatePli(throttle int64, force bool) bool {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	if r.endTime != 0 || (!force && mono.UnixNano()-r.lastPli < throttle) {
+		return false
+	}
+	r.updatePliLocked(1)
+	r.updatePliTimeLocked()
+	return true
+}
+
+func (r *rtpStatsBaseLite) UpdatePliAndTime(pliCount uint32) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	if r.endTime != 0 {
+		return
+	}
+
+	r.updatePliLocked(pliCount)
+	r.updatePliTimeLocked()
+}
+
+func (r *rtpStatsBaseLite) UpdatePli(pliCount uint32) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	if r.endTime != 0 {
+		return
+	}
+
+	r.updatePliLocked(pliCount)
+}
+
+func (r *rtpStatsBaseLite) updatePliLocked(pliCount uint32) {
+	r.plis += pliCount
+}
+
+func (r *rtpStatsBaseLite) UpdatePliTime() {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	if r.endTime != 0 {
+		return
+	}
+
+	r.updatePliTimeLocked()
+}
+
+func (r *rtpStatsBaseLite) updatePliTimeLocked() {
+	r.lastPli = mono.UnixNano()
+}
+
+func (r *rtpStatsBaseLite) LastPli() int64 {
+	r.lock.RLock()
+	defer r.lock.RUnlock()
+
+	return r.lastPli
+}
+
+func (r *rtpStatsBaseLite) getPacketsSeen(extStartSN, extHighestSN uint64) uint64 {
+	packetsExpected := getPacketsExpected(extStartSN, extHighestSN)
+	if r.packetsLost > packetsExpected {
+		// should not happen
+		return 0
+	}
+
+	return packetsExpected - r.packetsLost
+}
+
+func (r *rtpStatsBaseLite) deltaInfoLite(
+	snapshotLiteID uint32,
+	extStartSN uint64,
+	extHighestSN uint64,
+) (deltaInfoLite *RTPDeltaInfoLite, loggingFields []any, err error) {
+	then, now, ok := r.getAndResetSnapshotLite(snapshotLiteID, extStartSN, extHighestSN)
+	if !ok {
+		return
+	}
+
+	startTime := then.startTime
+	endTime := now.startTime
+
+	packetsExpected := uint32(now.extStartSN - then.extStartSN)
+	if then.extStartSN > extHighestSN {
+		packetsExpected = 0
+	}
+	if packetsExpected > cNumSequenceNumbers {
+		loggingFields = []any{
+			"snapshotLiteID", snapshotLiteID,
+			"snapshotLiteNow", now,
+			"snapshotLiteThen", then,
+			"packetsExpected", packetsExpected,
+			"duration", time.Duration(endTime - startTime),
+		}
+		err = errors.New("too many packets expected in delta lite")
+		return
+	}
+	if packetsExpected == 0 {
+		deltaInfoLite = &RTPDeltaInfoLite{
+			StartTime: time.Unix(0, startTime),
+			EndTime:   time.Unix(0, endTime),
+		}
+		return
+	}
+
+	packetsLost := uint32(now.packetsLost - then.packetsLost)
+	if int32(packetsLost) < 0 {
+		packetsLost = 0
+	}
+	if packetsLost > packetsExpected {
+		loggingFields = []any{
+			"snapshotLiteID", snapshotLiteID,
+			"snapshotLiteNow", now,
+			"snapshotLiteThen", then,
+			"packetsExpected", packetsExpected,
+			"packetsLost", packetsLost,
+			"duration", time.Duration(endTime - startTime),
+		}
+		err = errors.New("unexpected number of packets lost in delta lite")
+	}
+
+	deltaInfoLite = &RTPDeltaInfoLite{
+		StartTime:         time.Unix(0, startTime),
+		EndTime:           time.Unix(0, endTime),
+		Packets:           packetsExpected,
+		Bytes:             now.bytes - then.bytes,
+		PacketsLost:       packetsLost,
+		PacketsOutOfOrder: uint32(now.packetsOutOfOrder - then.packetsOutOfOrder),
+		Nacks:             now.nacks - then.nacks,
+	}
+	return
+}
+
+func (r *rtpStatsBaseLite) marshalLogObject(e zapcore.ObjectEncoder, packetsExpected, packetsSeenMinusPadding uint64) (float64, error) {
+	if r == nil || !r.initialized || r.clockRate == 0 {
+		return 0, errors.New("not initialized")
+	}
+
+	endTime := r.endTime
+	if endTime == 0 {
+		endTime = mono.UnixNano()
+	}
+	elapsed := time.Duration(endTime - r.startTime)
+	if elapsed == 0 {
+		return 0, errors.New("no time elapsed")
+	}
+	elapsedSeconds := elapsed.Seconds()
+
+	e.AddTime("startTime", time.Unix(0, r.startTime))
+	e.AddTime("endTime", time.Unix(0, r.endTime))
+	e.AddDuration("elapsed", elapsed)
+
+	e.AddUint64("packetsExpected", packetsExpected)
+	e.AddFloat64("packetsExpectedRate", float64(packetsExpected)/elapsedSeconds)
+	e.AddUint64("packetsSeenPrimary", packetsSeenMinusPadding)
+	e.AddFloat64("packetsSeenPrimaryRate", float64(packetsSeenMinusPadding)/elapsedSeconds)
+	e.AddUint64("bytes", r.bytes)
+	e.AddFloat64("bitrate", float64(r.bytes)*8.0/elapsedSeconds)
+
+	e.AddUint64("packetsOutOfOrder", r.packetsOutOfOrder)
+
+	e.AddUint64("packetsLost", r.packetsLost)
+	e.AddFloat64("packetsLostRate", float64(r.packetsLost)/elapsedSeconds)
+	if packetsExpected != 0 {
+		e.AddFloat32("packetLostPercentage", float32(r.packetsLost)/float32(packetsExpected)*100.0)
+	}
+
+	var sb strings.Builder
+	for burst, count := range r.gapHistogram {
+		if count == 0 {
+			continue
+		}
+		if sb.Len() > 0 {
+			sb.WriteString(", ")
+		}
+		fmt.Fprintf(&sb, "%d:%d", burst+1, count)
+	}
+	if sb.Len() > 0 {
+		e.AddString("gapHistogram", "["+sb.String()+"]")
+	}
+
+	e.AddUint32("nacks", r.nacks)
+	e.AddUint32("nackAcks", r.nackAcks)
+	e.AddUint32("nackMisses", r.nackMisses)
+	e.AddUint32("nackRepeated", r.nackRepeated)
+
+	e.AddUint32("plis", r.plis)
+	e.AddTime("lastPli", time.Unix(0, r.lastPli))
+	return elapsedSeconds, nil
+}
+
+func (r *rtpStatsBaseLite) toProto(packetsExpected, packetsSeenMinusPadding, packetsLost uint64) *livekit.RTPStats {
+	if r.startTime == 0 || r.clockRate == 0 {
+		return nil
+	}
+
+	endTime := r.endTime
+	if endTime == 0 {
+		endTime = mono.UnixNano()
+	}
+	elapsed := time.Duration(endTime - r.startTime).Seconds()
+	if elapsed == 0.0 {
+		return nil
+	}
+
+	packetRate := float64(packetsSeenMinusPadding) / elapsed
+	bitrate := float64(r.bytes) * 8.0 / elapsed
+
+	packetLostRate := float64(packetsLost) / elapsed
+	packetLostPercentage := float32(0)
+	if packetsExpected != 0 {
+		packetLostPercentage = float32(packetsLost) / float32(packetsExpected) * 100.0
+	}
+
+	p := &livekit.RTPStats{
+		StartTime:            timestamppb.New(time.Unix(0, r.startTime)),
+		EndTime:              timestamppb.New(time.Unix(0, endTime)),
+		Duration:             elapsed,
+		Packets:              uint32(packetsSeenMinusPadding),
+		PacketRate:           packetRate,
+		Bytes:                r.bytes,
+		Bitrate:              bitrate,
+		PacketsLost:          uint32(packetsLost),
+		PacketLossRate:       packetLostRate,
+		PacketLossPercentage: packetLostPercentage,
+		PacketsOutOfOrder:    uint32(r.packetsOutOfOrder),
+		Nacks:                r.nacks,
+		NackAcks:             r.nackAcks,
+		NackMisses:           r.nackMisses,
+		NackRepeated:         r.nackRepeated,
+		Plis:                 r.plis,
+		LastPli:              timestamppb.New(time.Unix(0, r.lastPli)),
+	}
+
+	for i := range len(r.gapHistogram) {
+		if r.gapHistogram[i] == 0 {
+			continue
+		}
+		if p.GapHistogram == nil {
+			p.GapHistogram = make(map[int32]uint32, len(r.gapHistogram))
+		}
+		p.GapHistogram[int32(i+1)] = r.gapHistogram[i]
+	}
+
+	return p
+}
+
+func (r *rtpStatsBaseLite) getAndResetSnapshotLite(snapshotLiteID uint32, extStartSN uint64, extHighestSN uint64) (snapshotLite, snapshotLite, bool) {
+	if !r.initialized {
+		return snapshotLite{}, snapshotLite{}, false
+	}
+
+	idx := snapshotLiteID - cFirstSnapshotID
+	then := r.snapshotLites[idx]
+	if !then.isValid {
+		then = initSnapshotLite(r.startTime, extStartSN)
+		r.snapshotLites[idx] = then
+	}
+
+	// snapshot now
+	now := r.getSnapshotLite(mono.UnixNano(), extHighestSN+1)
+	r.snapshotLites[idx] = now
+	return then, now, true
+}
+
+func (r *rtpStatsBaseLite) updateGapHistogram(gap int) {
+	if gap < 2 {
+		return
+	}
+
+	missing := gap - 1
+	if missing > len(r.gapHistogram) {
+		r.gapHistogram[len(r.gapHistogram)-1]++
+	} else {
+		r.gapHistogram[missing-1]++
+	}
+}
+
+func (r *rtpStatsBaseLite) getSnapshotLite(startTime int64, extStartSN uint64) snapshotLite {
+	return snapshotLite{
+		isValid:           true,
+		startTime:         startTime,
+		extStartSN:        extStartSN,
+		bytes:             r.bytes,
+		packetsOutOfOrder: r.packetsOutOfOrder,
+		packetsLost:       r.packetsLost,
+		nacks:             r.nacks,
+	}
+}
+
+// ----------------------------------
+
+func initSnapshotLite(startTime int64, extStartSN uint64) snapshotLite {
+	return snapshotLite{
+		isValid:    true,
+		startTime:  startTime,
+		extStartSN: extStartSN,
+	}
+}
+
+func getPacketsExpected(extStartSN, extHighestSN uint64) uint64 {
+	return extHighestSN - extStartSN + 1
+}
+
+func shouldLog(count int) bool {
+	return count < 20 || count%200 == 0
+}
+
+// ----------------------------------

pkg/sfu/rtpstats/rtpstats_receiver_lite.go

@@ -0,0 +1,179 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtpstats
+
+import (
+	"go.uber.org/zap/zapcore"
+
+	"github.com/livekit/mediatransportutil/pkg/utils"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/utils/mono"
+)
+
+type RTPFlowStateLite struct {
+	IsNotHandled bool
+
+	LossStartInclusive uint64
+	LossEndExclusive   uint64
+
+	ExtSequenceNumber uint64
+}
+
+func (r *RTPFlowStateLite) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if r == nil {
+		return nil
+	}
+
+	e.AddBool("IsNotHandled", r.IsNotHandled)
+	e.AddUint64("LossStartInclusive", r.LossStartInclusive)
+	e.AddUint64("LossEndExclusive", r.LossEndExclusive)
+	e.AddUint64("ExtSequenceNumber", r.ExtSequenceNumber)
+	return nil
+}
+
+// ---------------------------------------------------------------------
+
+type RTPStatsReceiverLite struct {
+	*rtpStatsBaseLite
+
+	sequenceNumber *utils.WrapAround[uint16, uint64]
+}
+
+func NewRTPStatsReceiverLite(params RTPStatsParams) *RTPStatsReceiverLite {
+	return &RTPStatsReceiverLite{
+		rtpStatsBaseLite: newRTPStatsBaseLite(params),
+		sequenceNumber:   utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
+	}
+}
+
+func (r *RTPStatsReceiverLite) NewSnapshotLiteId() uint32 {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	return r.newSnapshotLiteID(r.sequenceNumber.GetExtendedHighest())
+}
+
+func (r *RTPStatsReceiverLite) Update(packetTime int64, packetSize int, sequenceNumber uint16) (flowStateLite RTPFlowStateLite) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	if r.endTime != 0 {
+		flowStateLite.IsNotHandled = true
+		return
+	}
+
+	var resSN utils.WrapAroundUpdateResult[uint64]
+	if !r.initialized {
+		r.initialized = true
+
+		r.startTime = mono.UnixNano()
+
+		resSN = r.sequenceNumber.Update(sequenceNumber)
+
+		// initialize lite snapshots if any
+		for i := uint32(0); i < r.nextSnapshotLiteID-cFirstSnapshotID; i++ {
+			r.snapshotLites[i] = initSnapshotLite(r.startTime, r.sequenceNumber.GetExtendedStart())
+		}
+
+		r.logger.Debugw(
+			"rtp receiver lite stream start",
+			"rtpStats", lockedRTPStatsReceiverLiteLogEncoder{r},
+		)
+	} else {
+		resSN = r.sequenceNumber.Update(sequenceNumber)
+		if resSN.IsUnhandled {
+			flowStateLite.IsNotHandled = true
+			return
+		}
+	}
+
+	gapSN := int64(resSN.ExtendedVal - resSN.PreExtendedHighest)
+	if gapSN <= 0 { // duplicate OR out-of-order
+		r.packetsOutOfOrder++ // counting duplicate as out-of-order
+		if gapSN != 0 && r.packetsLost > 0 {
+			r.packetsLost--
+		}
+	} else { // in-order
+		r.updateGapHistogram(int(gapSN))
+		r.packetsLost += uint64(gapSN - 1)
+
+		flowStateLite.LossStartInclusive = resSN.PreExtendedHighest + 1
+		flowStateLite.LossEndExclusive = resSN.ExtendedVal
+	}
+	flowStateLite.ExtSequenceNumber = resSN.ExtendedVal
+	r.bytes += uint64(packetSize)
+	return
+}
+
+func (r *RTPStatsReceiverLite) DeltaInfoLite(snapshotLiteID uint32) *RTPDeltaInfoLite {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	deltaInfoLite, loggingFields, err := r.deltaInfoLite(
+		snapshotLiteID,
+		r.sequenceNumber.GetExtendedStart(),
+		r.sequenceNumber.GetExtendedHighest(),
+	)
+	if err != nil {
+		r.logger.Infow(err.Error(), append(loggingFields, "rtpStats", lockedRTPStatsReceiverLiteLogEncoder{r})...)
+	}
+
+	return deltaInfoLite
+}
+
+func (r *RTPStatsReceiverLite) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if r == nil {
+		return nil
+	}
+
+	r.lock.RLock()
+	defer r.lock.RUnlock()
+
+	return lockedRTPStatsReceiverLiteLogEncoder{r}.MarshalLogObject(e)
+}
+
+func (r *RTPStatsReceiverLite) ToProto() *livekit.RTPStats {
+	r.lock.RLock()
+	defer r.lock.RUnlock()
+
+	return r.rtpStatsBaseLite.toProto(r.sequenceNumber.GetExtendedStart(), r.sequenceNumber.GetExtendedHighest(), r.packetsLost)
+}
+
+// ----------------------------------
+
+type lockedRTPStatsReceiverLiteLogEncoder struct {
+	*RTPStatsReceiverLite
+}
+
+func (r lockedRTPStatsReceiverLiteLogEncoder) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if r.RTPStatsReceiverLite == nil {
+		return nil
+	}
+
+	extStartSN, extHighestSN := r.sequenceNumber.GetExtendedStart(), r.sequenceNumber.GetExtendedHighest()
+	if _, err := r.rtpStatsBaseLite.marshalLogObject(
+		e,
+		getPacketsExpected(extStartSN, extHighestSN),
+		getPacketsExpected(extStartSN, extHighestSN),
+	); err != nil {
+		return err
+	}
+
+	e.AddUint64("extStartSN", r.sequenceNumber.GetExtendedStart())
+	e.AddUint64("extHighestSN", r.sequenceNumber.GetExtendedHighest())
+	return nil
+}
+
+// ----------------------------------

pkg/sfu/rtpstats/rtpstats_sender_lite.go

@@ -0,0 +1,121 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtpstats
+
+import (
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/utils/mono"
+	"go.uber.org/zap/zapcore"
+)
+
+type RTPStatsSenderLite struct {
+	*rtpStatsBaseLite
+
+	extStartSN   uint64
+	extHighestSN uint64
+}
+
+func NewRTPStatsSenderLite(params RTPStatsParams) *RTPStatsSenderLite {
+	return &RTPStatsSenderLite{
+		rtpStatsBaseLite: newRTPStatsBaseLite(params),
+	}
+}
+
+func (r *RTPStatsSenderLite) Update(packetTime int64, packetSize int, extSequenceNumber uint64) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	if r.endTime != 0 {
+		return
+	}
+
+	if !r.initialized {
+		r.initialized = true
+
+		r.startTime = mono.UnixNano()
+
+		r.extStartSN = extSequenceNumber
+		r.extHighestSN = extSequenceNumber - 1
+
+		r.logger.Debugw(
+			"rtp sender lite stream start",
+			"rtpStats", lockedRTPStatsSenderLiteLogEncoder{r},
+		)
+	}
+
+	gapSN := int64(extSequenceNumber - r.extHighestSN)
+	if gapSN <= 0 { // duplicate OR out-of-order
+		r.packetsOutOfOrder++ // counting duplicate as out-of-order
+		r.packetsLost--
+	} else { // in-order
+		r.updateGapHistogram(int(gapSN))
+		r.packetsLost += uint64(gapSN - 1)
+
+		r.extHighestSN = extSequenceNumber
+	}
+
+	r.bytes += uint64(packetSize)
+}
+
+func (r *RTPStatsSenderLite) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if r == nil {
+		return nil
+	}
+
+	r.lock.RLock()
+	defer r.lock.RUnlock()
+
+	return lockedRTPStatsSenderLiteLogEncoder{r}.MarshalLogObject(e)
+}
+
+func (r *RTPStatsSenderLite) ToProto() *livekit.RTPStats {
+	r.lock.RLock()
+	defer r.lock.RUnlock()
+
+	return r.rtpStatsBaseLite.toProto(r.extStartSN, r.extHighestSN, r.packetsLost)
+}
+
+func (r *RTPStatsSenderLite) ExtHighestSequenceNumber() uint64 {
+	r.lock.RLock()
+	defer r.lock.RUnlock()
+
+	return r.extHighestSN
+}
+
+// -------------------------------------------------------------------
+
+type lockedRTPStatsSenderLiteLogEncoder struct {
+	*RTPStatsSenderLite
+}
+
+func (r lockedRTPStatsSenderLiteLogEncoder) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if r.RTPStatsSenderLite == nil {
+		return nil
+	}
+
+	if _, err := r.rtpStatsBaseLite.marshalLogObject(
+		e,
+		getPacketsExpected(r.extStartSN, r.extHighestSN),
+		getPacketsExpected(r.extStartSN, r.extHighestSN),
+	); err != nil {
+		return err
+	}
+
+	e.AddUint64("extStartSN", r.extStartSN)
+	e.AddUint64("extHighestSN", r.extHighestSN)
+	return nil
+}
+
+// -------------------------------------------------------------------

pkg/sfu/rtpstats/rtpstats_test.go

@@ -0,0 +1,286 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtpstats
+
+import (
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/pion/rtp"
+	"github.com/stretchr/testify/require"
+)
+
+func getPacket(sn uint16, ts uint32, payloadSize int) *rtp.Packet {
+	return &rtp.Packet{
+		Header: rtp.Header{
+			SequenceNumber: sn,
+			Timestamp:      ts,
+		},
+		Payload: make([]byte, payloadSize),
+	}
+}
+
+func Test_RTPStatsReceiver_Update(t *testing.T) {
+	clockRate := uint32(90000)
+	r := NewRTPStatsReceiver(RTPStatsParams{})
+	r.SetClockRate(clockRate)
+
+	sequenceNumber := uint16(rand.Float64() * float64(1<<16))
+	timestamp := uint32(rand.Float64() * float64(1<<32))
+	packet := getPacket(sequenceNumber, timestamp, 1000)
+	flowState := r.Update(
+		time.Now().UnixNano(),
+		packet.Header.SequenceNumber,
+		packet.Header.Timestamp,
+		packet.Header.Marker,
+		packet.Header.MarshalSize(),
+		len(packet.Payload),
+		0,
+	)
+	require.True(t, r.initialized)
+	require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
+	require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
+	require.Equal(t, timestamp, r.timestamp.GetHighest())
+	require.Equal(t, timestamp, uint32(r.timestamp.GetExtendedHighest()))
+
+	// in-order, no loss
+	sequenceNumber++
+	timestamp += 3000
+	packet = getPacket(sequenceNumber, timestamp, 1000)
+	flowState = r.Update(
+		time.Now().UnixNano(),
+		packet.Header.SequenceNumber,
+		packet.Header.Timestamp,
+		packet.Header.Marker,
+		packet.Header.MarshalSize(),
+		len(packet.Payload),
+		0,
+	)
+	require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
+	require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
+	require.Equal(t, timestamp, r.timestamp.GetHighest())
+	require.Equal(t, timestamp, uint32(r.timestamp.GetExtendedHighest()))
+
+	// out-of-order, would cause a restart which is disallowed
+	packet = getPacket(sequenceNumber-10, timestamp-30000, 1000)
+	flowState = r.Update(
+		time.Now().UnixNano(),
+		packet.Header.SequenceNumber,
+		packet.Header.Timestamp,
+		packet.Header.Marker,
+		packet.Header.MarshalSize(),
+		len(packet.Payload),
+		0,
+	)
+	require.Equal(t, RTPFlowUnhandledReasonPreStartTimestamp, flowState.UnhandledReason)
+	require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
+	require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
+	require.Equal(t, timestamp, r.timestamp.GetHighest())
+	require.Equal(t, timestamp, uint32(r.timestamp.GetExtendedHighest()))
+	require.Equal(t, uint64(0), r.packetsOutOfOrder)
+	require.Equal(t, uint64(0), r.packetsDuplicate)
+
+	// duplicate of the above out-of-order packet, but would not be handled as it causes a restart
+	packet = getPacket(sequenceNumber-10, timestamp-30000, 1000)
+	flowState = r.Update(
+		time.Now().UnixNano(),
+		packet.Header.SequenceNumber,
+		packet.Header.Timestamp,
+		packet.Header.Marker,
+		packet.Header.MarshalSize(),
+		len(packet.Payload),
+		0,
+	)
+	require.Equal(t, RTPFlowUnhandledReasonPreStartTimestamp, flowState.UnhandledReason)
+	require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
+	require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
+	require.Equal(t, timestamp, r.timestamp.GetHighest())
+	require.Equal(t, timestamp, uint32(r.timestamp.GetExtendedHighest()))
+	require.Equal(t, uint64(0), r.packetsOutOfOrder)
+	require.Equal(t, uint64(0), r.packetsDuplicate)
+
+	// loss
+	sequenceNumber += 10
+	timestamp += 30000
+	packet = getPacket(sequenceNumber, timestamp, 1000)
+	flowState = r.Update(
+		time.Now().UnixNano(),
+		packet.Header.SequenceNumber,
+		packet.Header.Timestamp,
+		packet.Header.Marker,
+		packet.Header.MarshalSize(),
+		len(packet.Payload),
+		0,
+	)
+	require.Equal(t, uint64(sequenceNumber-9), flowState.LossStartInclusive)
+	require.Equal(t, uint64(sequenceNumber), flowState.LossEndExclusive)
+	require.Equal(t, uint64(9), r.packetsLost)
+
+	// out-of-order should decrement number of lost packets
+	packet = getPacket(sequenceNumber-6, timestamp-18000, 1000)
+	flowState = r.Update(
+		time.Now().UnixNano(),
+		packet.Header.SequenceNumber,
+		packet.Header.Timestamp,
+		packet.Header.Marker,
+		packet.Header.MarshalSize(),
+		len(packet.Payload),
+		0,
+	)
+	require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
+	require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
+	require.Equal(t, timestamp, r.timestamp.GetHighest())
+	require.Equal(t, timestamp, uint32(r.timestamp.GetExtendedHighest()))
+	require.Equal(t, uint64(1), r.packetsOutOfOrder)
+	require.Equal(t, uint64(0), r.packetsDuplicate)
+	require.Equal(t, uint64(8), r.packetsLost)
+
+	// test sequence number history
+	// with a gap
+	sequenceNumber += 2
+	timestamp += 6000
+	packet = getPacket(sequenceNumber, timestamp, 1000)
+	flowState = r.Update(
+		time.Now().UnixNano(),
+		packet.Header.SequenceNumber,
+		packet.Header.Timestamp,
+		packet.Header.Marker,
+		packet.Header.MarshalSize(),
+		len(packet.Payload),
+		0,
+	)
+	require.Equal(t, uint64(sequenceNumber-1), flowState.LossStartInclusive)
+	require.Equal(t, uint64(sequenceNumber), flowState.LossEndExclusive)
+	require.Equal(t, uint64(9), r.packetsLost)
+	require.False(t, r.history.IsSet(uint64(sequenceNumber)-1))
+
+	// out-of-order
+	sequenceNumber--
+	timestamp -= 3000
+	packet = getPacket(sequenceNumber, timestamp, 999)
+	flowState = r.Update(
+		time.Now().UnixNano(),
+		packet.Header.SequenceNumber,
+		packet.Header.Timestamp,
+		packet.Header.Marker,
+		packet.Header.MarshalSize(),
+		len(packet.Payload),
+		0,
+	)
+	require.Equal(t, uint64(8), r.packetsLost)
+	require.Equal(t, uint64(2), r.packetsOutOfOrder)
+	require.True(t, r.history.IsSet(uint64(sequenceNumber)))
+
+	// padding only
+	sequenceNumber += 2
+	timestamp += 3000
+	packet = getPacket(sequenceNumber, timestamp, 0)
+	flowState = r.Update(
+		time.Now().UnixNano(),
+		packet.Header.SequenceNumber,
+		packet.Header.Timestamp,
+		packet.Header.Marker,
+		packet.Header.MarshalSize(),
+		len(packet.Payload),
+		25,
+	)
+	require.Equal(t, uint64(8), r.packetsLost)
+	require.Equal(t, uint64(2), r.packetsOutOfOrder)
+	require.True(t, r.history.IsSet(uint64(sequenceNumber)))
+	require.True(t, r.history.IsSet(uint64(sequenceNumber)-1))
+	require.True(t, r.history.IsSet(uint64(sequenceNumber)-2))
+
+	// old packet, but simulating increasing sequence number after roll over
+	packet = getPacket(sequenceNumber+400, timestamp-6000, 300)
+	flowState = r.Update(
+		time.Now().UnixNano(),
+		packet.Header.SequenceNumber,
+		packet.Header.Timestamp,
+		packet.Header.Marker,
+		packet.Header.MarshalSize(),
+		len(packet.Payload),
+		0,
+	)
+	require.Equal(t, RTPFlowUnhandledReasonOldSequenceNumber, flowState.UnhandledReason)
+	require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
+	require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
+	require.Equal(t, timestamp, r.timestamp.GetHighest())
+	require.Equal(t, timestamp, uint32(r.timestamp.GetExtendedHighest()))
+
+	r.Stop()
+}
+
+func Test_RTPStatsReceiver_Restart(t *testing.T) {
+	clockRate := uint32(90000)
+	r := NewRTPStatsReceiver(RTPStatsParams{})
+	r.SetClockRate(clockRate)
+
+	// should not restart till there are at least threshold packets
+	require.False(t, r.maybeRestart(10, 20, 1000))
+	require.False(t, r.maybeRestart(11, 20, 1000))
+	require.False(t, r.maybeRestart(13, 20, 1000))
+	require.False(t, r.maybeRestart(14, 20, 1000))
+	// although adding 5th packet should have enough packets for a check,
+	// still should not restart as there is a sequence number gap between 11 and 13
+	require.False(t, r.maybeRestart(15, 20, 1000))
+	require.False(t, r.maybeRestart(16, 19, 1000))
+	// has enough packets, but still cannot restart because timestamps are not increasing
+	require.False(t, r.maybeRestart(17, 21, 1000))
+	require.False(t, r.maybeRestart(18, 21, 1000))
+	require.False(t, r.maybeRestart(19, 21, 1000))
+	// can restart as there are enough packets with proper sequencing
+	require.True(t, r.maybeRestart(20, 21, 1000))
+	require.Equal(t, restartThreshold, r.restartPacketsN)
+
+	r.resetRestart()
+	require.Zero(t, r.restartPacketsN)
+
+	r.Stop()
+}
+
+func Test_RTPStatsSender_getIntervalStats(t *testing.T) {
+	t.Run("packetsNotFoundMetadata should match lost packets", func(t *testing.T) {
+		r := NewRTPStatsSender(RTPStatsParams{}, 1024)
+		stats := r.getIntervalStats(0, 10000, 10000)
+		require.EqualValues(t, 8977, stats.packetsNotFoundMetadata)
+	})
+}
+
+func BenchmarkRTPStatsReceiver_Update(b *testing.B) {
+	const clockRate = 90000
+	const hdrSize = 12
+	const payloadSize = 1200
+	const paddingSize = 0
+	const tsIncrement = 3000 // ~33ms at 90kHz
+
+	r := NewRTPStatsReceiver(RTPStatsParams{})
+	r.SetClockRate(clockRate)
+
+	// initialize before running benchmark loop
+	packetTime := time.Now().UnixNano()
+	sn := uint16(0)
+	ts := uint32(0)
+	r.Update(packetTime, sn, ts, false, hdrSize, payloadSize, paddingSize)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		sn++
+		ts += tsIncrement
+		packetTime += 33 * int64(time.Millisecond) // ~30fps
+		marker := (i+1)%100 == 0                   // marker every 100 packets
+		r.Update(packetTime, sn, ts, marker, hdrSize, payloadSize, paddingSize)
+	}
+}

pkg/sfu/sequencer.go

@@ -0,0 +1,490 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sfu
+
+import (
+	"math"
+	"sync"
+	"time"
+
+	"github.com/livekit/livekit-server/pkg/sfu/utils"
+	"github.com/livekit/protocol/logger"
+	"go.uber.org/zap/zapcore"
+)
+
+const (
+	defaultRtt           = 70
+	ignoreRetransmission = 100 // Ignore packet retransmission after ignoreRetransmission milliseconds
+	maxAck               = 3
+)
+
+type packetMeta struct {
+	// Original extended sequence number from stream.
+	// The original extended sequence number is used to find the original
+	// packet from publisher
+	sourceSeqNo uint64
+	// Modified sequence number after offset.
+	// This sequence number is used for the associated
+	// down track, is modified according the offsets, and
+	// must not be shared
+	targetSeqNo uint16
+	// Modified timestamp for current associated
+	// down track.
+	timestamp uint32
+	// Modified marker
+	marker bool
+	// The last time this packet was nack requested.
+	// Sometimes clients request the same packet more than once, so keep
+	// track of the requested packets helps to avoid writing multiple times
+	// the same packet.
+	// The resolution is 1 ms counting after the sequencer start time.
+	lastNack uint32
+	// number of NACKs this packet has received
+	nacked uint8
+	// Spatial layer of packet
+	layer int8
+	// Information that differs depending on the codec
+	codecBytes       [8]byte
+	numCodecBytesIn  uint8
+	numCodecBytesOut uint8
+	codecBytesSlice  []byte
+	// Dependency Descriptor of packet
+	ddBytes      [8]byte
+	ddBytesSize  uint8
+	ddBytesSlice []byte
+	// abs-capture-time of packet
+	actBytes []byte
+	// number of packet trailer bytes stripped when the packet was forwarded
+	trailerStripped uint8
+}
+
+func (pm packetMeta) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	e.AddUint64("sourceSeqNo", pm.sourceSeqNo)
+	e.AddUint16("targetSeqNo", pm.targetSeqNo)
+	e.AddInt8("layer", pm.layer)
+	e.AddUint8("nacked", pm.nacked)
+	e.AddUint8("numCodecBytesIn", pm.numCodecBytesIn)
+	if len(pm.codecBytesSlice) != 0 {
+		e.AddInt("codecBytesSlice", len(pm.codecBytesSlice))
+	} else {
+		e.AddUint8("numCodecBytesOut", pm.numCodecBytesOut)
+	}
+	if len(pm.ddBytesSlice) != 0 {
+		e.AddInt("ddBytesSlice", len(pm.ddBytesSlice))
+	} else {
+		e.AddUint8("ddBytesSize", pm.ddBytesSize)
+	}
+	if len(pm.actBytes) != 0 {
+		e.AddInt("actBytes", len(pm.actBytes))
+	}
+	if pm.trailerStripped != 0 {
+		e.AddUint8("trailerStripped", pm.trailerStripped)
+	}
+	return nil
+}
+
+type extPacketMeta struct {
+	packetMeta
+	extSequenceNumber uint64
+	extTimestamp      uint64
+}
+
+func (epm extPacketMeta) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	e.AddObject("packetMeta", epm.packetMeta)
+	e.AddUint64("extSequenceNumber", epm.extSequenceNumber)
+	return nil
+}
+
+// Sequencer stores the packet sequence received by the down track
+type sequencer struct {
+	sync.Mutex
+	size         int
+	startTime    int64
+	initialized  bool
+	extStartSN   uint64
+	extHighestSN uint64
+	snOffset     uint64
+	extHighestTS uint64
+	meta         []packetMeta
+	snRangeMap   *utils.RangeMap[uint64, uint64]
+	rtt          uint32
+	logger       logger.Logger
+}
+
+func newSequencer(size int, maybeSparse bool, logger logger.Logger) *sequencer {
+	if size == 0 {
+		return nil
+	}
+
+	s := &sequencer{
+		size:      size,
+		startTime: time.Now().UnixNano(),
+		meta:      make([]packetMeta, size),
+		rtt:       defaultRtt,
+		logger:    logger,
+	}
+
+	if maybeSparse {
+		s.snRangeMap = utils.NewRangeMap[uint64, uint64]((size + 1) / 2) // assume run lengths of at least 2 in between padding bursts
+	}
+	return s
+}
+
+func (s *sequencer) setRTT(rtt uint32) {
+	s.Lock()
+	defer s.Unlock()
+
+	if rtt == 0 {
+		s.rtt = defaultRtt
+	} else {
+		s.rtt = rtt
+	}
+}
+
+// flush discards all recorded packet metadata. It must be called on a stream restart: the
+// metadata maps outgoing sequence numbers to source packets in the receiver's bucket, and a
+// restart resyncs that bucket, so retransmitting against stale metadata would send the wrong
+// packet (or read past the re-read payload). After flush, NACKs for pre-restart packets are
+// ignored until the sequencer is re-initialized by the next push.
+func (s *sequencer) flush() {
+	s.Lock()
+	defer s.Unlock()
+
+	s.initialized = false
+	s.extStartSN = 0
+	s.extHighestSN = 0
+	s.extHighestTS = 0
+	s.snOffset = 0
+	clear(s.meta)
+	if s.snRangeMap != nil {
+		s.snRangeMap = utils.NewRangeMap[uint64, uint64]((s.size + 1) / 2)
+	}
+}
+
+func (s *sequencer) push(
+	packetTime int64,
+	extIncomingSN, extModifiedSN uint64,
+	extModifiedTS uint64,
+	marker bool,
+	layer int8,
+	codecBytes []byte,
+	numCodecBytesIn int,
+	ddBytes []byte,
+	actBytes []byte,
+	trailerStripped int,
+) {
+	s.Lock()
+	defer s.Unlock()
+
+	if !s.initialized {
+		s.initialized = true
+		s.extStartSN = extModifiedSN
+		s.extHighestSN = extModifiedSN
+		s.extHighestTS = extModifiedTS
+		s.updateSNOffset()
+	}
+
+	if extModifiedSN < s.extStartSN {
+		// old packet, should not happen
+		return
+	}
+
+	extHighestSNAdjusted := s.extHighestSN - s.snOffset
+	extModifiedSNAdjusted := extModifiedSN - s.snOffset
+	if extModifiedSN < s.extHighestSN {
+		if s.snRangeMap != nil {
+			snOffset, err := s.snRangeMap.GetValue(extModifiedSN)
+			if err != nil {
+				s.logger.Errorw(
+					"could not get sequence number offset", err,
+					"extStartSN", s.extStartSN,
+					"extHighestSN", s.extHighestSN,
+					"extIncomingSN", extIncomingSN,
+					"extModifiedSN", extModifiedSN,
+					"snOffset", s.snOffset,
+				)
+				return
+			}
+
+			extModifiedSNAdjusted = extModifiedSN - snOffset
+		}
+	}
+
+	if int64(extModifiedSNAdjusted-extHighestSNAdjusted) <= -int64(s.size) {
+		s.logger.Warnw(
+			"old packet, cannot be sequenced", nil,
+			"extHighestSN", s.extHighestSN,
+			"extIncomingSN", extIncomingSN,
+			"extModifiedSN", extModifiedSN,
+		)
+		return
+	}
+
+	// invalidate missing sequence numbers
+	if extModifiedSNAdjusted > extHighestSNAdjusted {
+		numInvalidated := 0
+		for esn := extHighestSNAdjusted + 1; esn != extModifiedSNAdjusted; esn++ {
+			s.invalidateSlot(int(esn % uint64(s.size)))
+			numInvalidated++
+			if numInvalidated >= s.size {
+				break
+			}
+		}
+	}
+
+	slot := extModifiedSNAdjusted % uint64(s.size)
+	s.meta[slot] = packetMeta{
+		sourceSeqNo:     extIncomingSN,
+		targetSeqNo:     uint16(extModifiedSN),
+		timestamp:       uint32(extModifiedTS),
+		marker:          marker,
+		layer:           layer,
+		numCodecBytesIn: uint8(numCodecBytesIn),
+		trailerStripped: uint8(trailerStripped),
+		lastNack:        s.getRefTime(packetTime), // delay retransmissions after the original transmission
+	}
+	pm := &s.meta[slot]
+
+	pm.numCodecBytesOut = uint8(len(codecBytes))
+	if len(codecBytes) > len(pm.codecBytes) {
+		pm.codecBytesSlice = append([]byte{}, codecBytes...)
+	} else {
+		copy(pm.codecBytes[:pm.numCodecBytesOut], codecBytes)
+	}
+
+	pm.ddBytesSize = uint8(len(ddBytes))
+	if len(ddBytes) > len(pm.ddBytes) {
+		pm.ddBytesSlice = append([]byte{}, ddBytes...)
+	} else {
+		copy(pm.ddBytes[:pm.ddBytesSize], ddBytes)
+	}
+
+	pm.actBytes = append([]byte{}, actBytes...)
+
+	if extModifiedSN > s.extHighestSN {
+		s.extHighestSN = extModifiedSN
+	}
+	if extModifiedTS > s.extHighestTS {
+		s.extHighestTS = extModifiedTS
+	}
+}
+
+func (s *sequencer) pushPadding(extStartSNInclusive uint64, extEndSNInclusive uint64) {
+	s.Lock()
+	defer s.Unlock()
+
+	if s.snRangeMap == nil || !s.initialized {
+		return
+	}
+
+	if extStartSNInclusive <= s.extHighestSN {
+		// a higher sequence number has already been recorded with an offset,
+		// adding an exclusion range before the highest means the offset of sequence numbers
+		// after the exclusion range will be affected and all those higher sequence numbers
+		// need to be patched.
+		//
+		// Not recording exclusion range means a few slots (of the size of exclusion range)
+		// are wasted in this cycle. That should be fine as the exclusion ranges should be
+		// a few packets at a time.
+		if extEndSNInclusive >= s.extHighestSN {
+			s.logger.Errorw("cannot exclude overlapping range", nil, "extHighestSN", s.extHighestSN, "startSN", extStartSNInclusive, "endSN", extEndSNInclusive)
+		} else {
+			s.logger.Warnw("cannot exclude old range", nil, "extHighestSN", s.extHighestSN, "startSN", extStartSNInclusive, "endSN", extEndSNInclusive)
+		}
+
+		// if exclusion range is before what has already been sequenced, invalidate exclusion range slots
+		for esn := extStartSNInclusive; esn != extEndSNInclusive+1; esn++ {
+			diff := int64(esn - s.extHighestSN)
+			if diff >= 0 || diff < -int64(s.size) {
+				// too old OR too new (too new should not happen, just be safe)
+				continue
+			}
+
+			snOffset, err := s.snRangeMap.GetValue(esn)
+			if err != nil {
+				s.logger.Errorw("could not get sequence number offset", err, "sn", esn)
+				continue
+			}
+
+			slot := (esn - snOffset) % uint64(s.size)
+			s.invalidateSlot(int(slot))
+		}
+		return
+	}
+
+	if err := s.snRangeMap.ExcludeRange(extStartSNInclusive, extEndSNInclusive+1); err != nil {
+		s.logger.Errorw("could not exclude range", err, "startSN", extStartSNInclusive, "endSN", extEndSNInclusive)
+		return
+	}
+
+	s.extHighestSN = extEndSNInclusive
+	s.updateSNOffset()
+}
+
+func (s *sequencer) getExtPacketMetas(seqNo []uint16) []extPacketMeta {
+	s.Lock()
+	defer s.Unlock()
+
+	if !s.initialized {
+		return nil
+	}
+
+	snOffset := uint64(0)
+	var err error
+	extPacketMetas := make([]extPacketMeta, 0, len(seqNo))
+	refTime := s.getRefTime(time.Now().UnixNano())
+	highestSN := uint16(s.extHighestSN)
+	highestTS := uint32(s.extHighestTS)
+	for _, sn := range seqNo {
+		diff := highestSN - sn
+		if diff > (1 << 15) {
+			// out-of-order from head (should not happen, just be safe)
+			continue
+		}
+
+		// find slot by adjusting for padding only packets that were not recorded in sequencer
+		extSN := uint64(sn) + (s.extHighestSN & 0xFFFF_FFFF_FFFF_0000)
+		if sn > highestSN {
+			extSN -= (1 << 16)
+		}
+
+		if s.snRangeMap != nil {
+			snOffset, err = s.snRangeMap.GetValue(extSN)
+			if err != nil {
+				// could be padding packet which is excluded and will not have value
+				continue
+			}
+		}
+
+		extSNAdjusted := extSN - snOffset
+		extHighestSNAdjusted := s.extHighestSN - s.snOffset
+		if extHighestSNAdjusted-extSNAdjusted >= uint64(s.size) {
+			// too old
+			continue
+		}
+
+		slot := extSNAdjusted % uint64(s.size)
+		meta := &s.meta[slot]
+		if meta.targetSeqNo != sn || s.isInvalidSlot(int(slot)) {
+			// invalid slot access could happen if padding packets exclusion range could not be recorded
+			continue
+		}
+
+		if meta.nacked < maxAck && refTime-meta.lastNack > uint32(math.Min(float64(ignoreRetransmission), float64(2*s.rtt))) {
+			meta.nacked++
+			meta.lastNack = refTime
+
+			extTS := uint64(meta.timestamp) + (s.extHighestTS & 0xFFFF_FFFF_0000_0000)
+			if meta.timestamp > highestTS {
+				extTS -= (1 << 32)
+			}
+			epm := extPacketMeta{
+				packetMeta:        *meta,
+				extSequenceNumber: extSN,
+				extTimestamp:      extTS,
+			}
+			epm.codecBytesSlice = append([]byte{}, meta.codecBytesSlice...)
+			epm.ddBytesSlice = append([]byte{}, meta.ddBytesSlice...)
+			epm.actBytes = append([]byte{}, meta.actBytes...)
+			extPacketMetas = append(extPacketMetas, epm)
+		}
+	}
+
+	return extPacketMetas
+}
+
+func (s *sequencer) lookupExtPacketMeta(extSN uint64) *extPacketMeta {
+	s.Lock()
+	defer s.Unlock()
+
+	if !s.initialized {
+		return nil
+	}
+
+	snOffset := uint64(0)
+	var err error
+	if s.snRangeMap != nil {
+		snOffset, err = s.snRangeMap.GetValue(extSN)
+		if err != nil {
+			return nil
+		}
+	}
+
+	extSNAdjusted := extSN - snOffset
+	extHighestSNAdjusted := s.extHighestSN - s.snOffset
+	if extHighestSNAdjusted-extSNAdjusted >= uint64(s.size) {
+		// too old
+		return nil
+	}
+
+	slot := extSNAdjusted % uint64(s.size)
+	meta := &s.meta[slot]
+	if s.isInvalidSlot(int(slot)) {
+		// invalid slot access could happen if padding packets exclusion range could not be recorded
+		return nil
+	}
+
+	extTS := uint64(meta.timestamp) + (s.extHighestTS & 0xFFFF_FFFF_0000_0000)
+	if meta.timestamp > uint32(s.extHighestTS) {
+		extTS -= (1 << 32)
+	}
+	epm := extPacketMeta{
+		packetMeta:        *meta,
+		extSequenceNumber: extSN,
+		extTimestamp:      extTS,
+	}
+	epm.codecBytesSlice = append([]byte{}, meta.codecBytesSlice...)
+	epm.ddBytesSlice = append([]byte{}, meta.ddBytesSlice...)
+	epm.actBytes = append([]byte{}, meta.actBytes...)
+	return &epm
+}
+
+func (s *sequencer) getRefTime(at int64) uint32 {
+	return uint32((at - s.startTime) / 1e6)
+}
+
+func (s *sequencer) updateSNOffset() {
+	if s.snRangeMap == nil {
+		return
+	}
+
+	snOffset, err := s.snRangeMap.GetValue(s.extHighestSN + 1)
+	if err != nil {
+		s.logger.Errorw("could not update sequence number offset", err, "extHighestSN", s.extHighestSN)
+		return
+	}
+	s.snOffset = snOffset
+}
+
+func (s *sequencer) invalidateSlot(slot int) {
+	if slot >= len(s.meta) {
+		return
+	}
+
+	s.meta[slot] = packetMeta{
+		sourceSeqNo: 0,
+		targetSeqNo: 0,
+		lastNack:    0,
+	}
+}
+
+func (s *sequencer) isInvalidSlot(slot int) bool {
+	if slot >= len(s.meta) {
+		return true
+	}
+
+	meta := &s.meta[slot]
+	return meta.sourceSeqNo == 0 && meta.targetSeqNo == 0 && meta.lastNack == 0
+}

pkg/sfu/sequencer_test.go

@@ -0,0 +1,406 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sfu
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/protocol/logger"
+)
+
+func Test_sequencer(t *testing.T) {
+	seq := newSequencer(500, false, logger.GetLogger())
+	off := uint16(15)
+
+	for i := uint64(1); i < 518; i++ {
+		seq.push(time.Now().UnixNano(), i, i+uint64(off), 123, true, 2, nil, 0, nil, nil, 0)
+	}
+	// send the last two out-of-order
+	seq.push(time.Now().UnixNano(), 519, 519+uint64(off), 123, false, 2, nil, 0, nil, nil, 0)
+	seq.push(time.Now().UnixNano(), 518, 518+uint64(off), 123, true, 2, nil, 0, nil, nil, 0)
+
+	req := []uint16{57, 58, 62, 63, 513, 514, 515, 516, 517}
+	res := seq.getExtPacketMetas(req)
+	// nothing should be returned as not enough time has elapsed since sending packet
+	require.Equal(t, 0, len(res))
+
+	time.Sleep((ignoreRetransmission + 10) * time.Millisecond)
+	res = seq.getExtPacketMetas(req)
+	require.Equal(t, len(req), len(res))
+	for i, val := range res {
+		require.Equal(t, val.targetSeqNo, req[i])
+		require.Equal(t, val.sourceSeqNo, uint64(req[i]-off))
+		require.Equal(t, val.layer, int8(2))
+		require.Equal(t, val.extSequenceNumber, uint64(req[i]))
+		require.Equal(t, val.extTimestamp, uint64(123))
+	}
+	res = seq.getExtPacketMetas(req)
+	require.Equal(t, 0, len(res))
+	time.Sleep((ignoreRetransmission + 10) * time.Millisecond)
+	res = seq.getExtPacketMetas(req)
+	require.Equal(t, len(req), len(res))
+	for i, val := range res {
+		require.Equal(t, val.targetSeqNo, req[i])
+		require.Equal(t, val.sourceSeqNo, uint64(req[i]-off))
+		require.Equal(t, val.layer, int8(2))
+		require.Equal(t, val.extSequenceNumber, uint64(req[i]))
+		require.Equal(t, val.extTimestamp, uint64(123))
+	}
+
+	seq.push(time.Now().UnixNano(), 521, 521+uint64(off), 123, true, 1, nil, 0, nil, nil, 0)
+	m := seq.getExtPacketMetas([]uint16{521 + off})
+	require.Equal(t, 0, len(m))
+	time.Sleep((ignoreRetransmission + 10) * time.Millisecond)
+	m = seq.getExtPacketMetas([]uint16{521 + off})
+	require.Equal(t, 1, len(m))
+
+	seq.push(time.Now().UnixNano(), 505, 505+uint64(off), 123, false, 1, nil, 0, nil, nil, 0)
+	m = seq.getExtPacketMetas([]uint16{505 + off})
+	require.Equal(t, 0, len(m))
+	time.Sleep((ignoreRetransmission + 10) * time.Millisecond)
+	m = seq.getExtPacketMetas([]uint16{505 + off})
+	require.Equal(t, 1, len(m))
+}
+
+func Test_sequencer_flush(t *testing.T) {
+	seq := newSequencer(500, false, logger.GetLogger())
+	off := uint16(15)
+
+	for i := uint64(1); i < 100; i++ {
+		seq.push(time.Now().UnixNano(), i, i+uint64(off), 123, true, 2, nil, 0, nil, nil, 0)
+	}
+	preFlush := []uint16{57 + off, 58 + off}
+
+	// flush discards all recorded metadata on a stream restart
+	seq.flush()
+
+	// even after enough time elapses, a NACK for a pre-flush packet retransmits nothing
+	time.Sleep((ignoreRetransmission + 10) * time.Millisecond)
+	require.Equal(t, 0, len(seq.getExtPacketMetas(preFlush)))
+
+	// the sequencer re-initializes on the next push and works normally for new packets
+	for i := uint64(200); i < 210; i++ {
+		seq.push(time.Now().UnixNano(), i, i+uint64(off), 456, true, 3, nil, 0, nil, nil, 0)
+	}
+	postFlush := []uint16{205 + off}
+	require.Equal(t, 0, len(seq.getExtPacketMetas(postFlush))) // not enough time elapsed yet
+	time.Sleep((ignoreRetransmission + 10) * time.Millisecond)
+	res := seq.getExtPacketMetas(postFlush)
+	require.Equal(t, 1, len(res))
+	require.Equal(t, uint16(205+off), res[0].targetSeqNo)
+	require.Equal(t, uint64(205), res[0].sourceSeqNo)
+	require.Equal(t, int8(3), res[0].layer)
+
+	// pre-flush packets remain non-retransmittable
+	require.Equal(t, 0, len(seq.getExtPacketMetas(preFlush)))
+}
+
+func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) {
+	type args struct {
+		seqNo []uint16
+	}
+	type input struct {
+		seqNo     uint64
+		isPadding bool
+	}
+	type fields struct {
+		inputs              []input
+		offset              uint64
+		markerOdd           bool
+		markerEven          bool
+		codecBytesOdd       []byte
+		numCodecBytesInOdd  int
+		codecBytesEven      []byte
+		numCodecBytesInEven int
+		codecBytesOversized []byte
+		ddBytesOdd          []byte
+		ddBytesEven         []byte
+		ddBytesOversized    []byte
+		actBytesOdd         []byte
+		actBytesEven        []byte
+	}
+
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   []uint16
+	}{
+		{
+			name: "Should get correct seq numbers",
+			fields: fields{
+				inputs: []input{
+					{65526, false},
+					{65524, false},
+					{65525, false},
+					{65529, false},
+					{65530, false},
+					{65531, true},
+					{65533, false},
+					{65532, true},
+					{65534, false},
+				},
+				offset:              5,
+				markerOdd:           true,
+				markerEven:          false,
+				codecBytesOdd:       []byte{1, 2, 3, 4},
+				numCodecBytesInOdd:  3,
+				codecBytesEven:      []byte{5, 6, 7},
+				numCodecBytesInEven: 4,
+				codecBytesOversized: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9},
+				ddBytesOdd:          []byte{8, 9, 10},
+				ddBytesEven:         []byte{11, 12},
+				ddBytesOversized:    []byte{11, 12, 13, 14, 15, 16, 17, 18, 19},
+				actBytesOdd:         []byte{0, 1, 2, 3, 4, 5, 6, 7},
+				actBytesEven:        []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
+			},
+			args: args{
+				seqNo: []uint16{65526 + 5, 65527 + 5, 65530 + 5, 0 /* 65531 input */, 1 /* 65532 input */, 2 /* 65533 input */, 3 /* 65534 input */},
+			},
+			// although 65526 is originally pushed, that would have been reset by 65532 (padding only packet)
+			// because of trying to add an exclusion range before highest sequence number which will fail
+			// and the resulting fix up of the exclusion range slots
+			want: []uint16{65530, 65533, 65534},
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			n := newSequencer(5, true, logger.GetLogger())
+
+			for _, i := range tt.fields.inputs {
+				if i.isPadding {
+					n.pushPadding(i.seqNo+tt.fields.offset, i.seqNo+tt.fields.offset)
+				} else {
+					if i.seqNo%5 == 0 {
+						n.push(
+							time.Now().UnixNano(),
+							i.seqNo,
+							i.seqNo+tt.fields.offset,
+							123,
+							tt.fields.markerOdd,
+							3,
+							tt.fields.codecBytesOversized,
+							len(tt.fields.codecBytesOversized),
+							tt.fields.ddBytesOversized,
+							tt.fields.actBytesOdd,
+							0,
+						)
+					} else {
+						if i.seqNo%2 == 0 {
+							n.push(
+								time.Now().UnixNano(),
+								i.seqNo,
+								i.seqNo+tt.fields.offset,
+								123,
+								tt.fields.markerEven,
+								3,
+								tt.fields.codecBytesEven,
+								tt.fields.numCodecBytesInEven,
+								tt.fields.ddBytesEven,
+								tt.fields.actBytesEven,
+								0,
+							)
+						} else {
+							n.push(
+								time.Now().UnixNano(),
+								i.seqNo,
+								i.seqNo+tt.fields.offset,
+								123,
+								tt.fields.markerOdd,
+								3,
+								tt.fields.codecBytesOdd,
+								tt.fields.numCodecBytesInOdd,
+								tt.fields.ddBytesOdd,
+								tt.fields.actBytesOdd,
+								0,
+							)
+						}
+					}
+				}
+			}
+
+			time.Sleep((ignoreRetransmission + 10) * time.Millisecond)
+			g := n.getExtPacketMetas(tt.args.seqNo)
+			var got []uint16
+			for _, sn := range g {
+				got = append(got, uint16(sn.sourceSeqNo))
+				if sn.sourceSeqNo%5 == 0 {
+					require.Equal(t, tt.fields.markerOdd, sn.marker)
+					require.Equal(t, tt.fields.codecBytesOversized, sn.codecBytesSlice)
+					require.Equal(t, uint8(len(tt.fields.codecBytesOversized)), sn.numCodecBytesIn)
+					require.Equal(t, tt.fields.ddBytesOversized, sn.ddBytesSlice)
+					require.Equal(t, uint8(len(tt.fields.codecBytesOversized)), sn.ddBytesSize)
+					require.Equal(t, tt.fields.actBytesOdd, sn.actBytes)
+				} else {
+					if sn.sourceSeqNo%2 == 0 {
+						require.Equal(t, tt.fields.markerEven, sn.marker)
+						require.Equal(t, tt.fields.codecBytesEven, sn.codecBytes[:sn.numCodecBytesOut])
+						require.Equal(t, uint8(tt.fields.numCodecBytesInEven), sn.numCodecBytesIn)
+						require.Equal(t, tt.fields.ddBytesEven, sn.ddBytes[:sn.ddBytesSize])
+						require.Equal(t, uint8(len(tt.fields.ddBytesEven)), sn.ddBytesSize)
+						require.Equal(t, tt.fields.actBytesEven, sn.actBytes)
+					} else {
+						require.Equal(t, tt.fields.markerOdd, sn.marker)
+						require.Equal(t, tt.fields.codecBytesOdd, sn.codecBytes[:sn.numCodecBytesOut])
+						require.Equal(t, uint8(tt.fields.numCodecBytesInOdd), sn.numCodecBytesIn)
+						require.Equal(t, tt.fields.ddBytesOdd, sn.ddBytes[:sn.ddBytesSize])
+						require.Equal(t, uint8(len(tt.fields.ddBytesOdd)), sn.ddBytesSize)
+						require.Equal(t, tt.fields.actBytesOdd, sn.actBytes)
+					}
+				}
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("getExtPacketMetas() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) {
+	type args struct {
+		seqNo []uint16
+	}
+	type input struct {
+		seqNo     uint64
+		isPadding bool
+	}
+	type fields struct {
+		inputs              []input
+		offset              uint64
+		markerOdd           bool
+		markerEven          bool
+		codecBytesOdd       []byte
+		numCodecBytesInOdd  int
+		codecBytesEven      []byte
+		numCodecBytesInEven int
+		ddBytesOdd          []byte
+		ddBytesEven         []byte
+		actBytesOdd         []byte
+		actBytesEven        []byte
+	}
+
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   []uint16
+	}{
+		{
+			name: "Should get correct seq numbers",
+			fields: fields{
+				inputs: []input{
+					{2, false},
+					{3, false},
+					{4, false},
+					{7, false},
+					{8, false},
+					{9, true},
+					{11, false},
+					{10, true},
+					{12, false},
+					{13, false},
+				},
+				offset:              5,
+				markerOdd:           true,
+				markerEven:          false,
+				codecBytesOdd:       []byte{1, 2, 3, 4},
+				numCodecBytesInOdd:  3,
+				codecBytesEven:      []byte{5, 6, 7},
+				numCodecBytesInEven: 4,
+				ddBytesOdd:          []byte{8, 9, 10},
+				ddBytesEven:         []byte{11, 12},
+				actBytesOdd:         []byte{8, 9, 10},
+				actBytesEven:        []byte{11, 12},
+			},
+			args: args{
+				seqNo: []uint16{4 + 5, 5 + 5, 8 + 5, 9 + 5, 10 + 5, 11 + 5, 12 + 5},
+			},
+			// although 4 and 8 were originally added, they would be too old after a cycle of sequencer buffer
+			want: []uint16{11, 12},
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			n := newSequencer(5, false, logger.GetLogger())
+
+			for _, i := range tt.fields.inputs {
+				if i.isPadding {
+					n.pushPadding(i.seqNo+tt.fields.offset, i.seqNo+tt.fields.offset)
+				} else {
+					if i.seqNo%2 == 0 {
+						n.push(
+							time.Now().UnixNano(),
+							i.seqNo,
+							i.seqNo+tt.fields.offset,
+							123,
+							tt.fields.markerEven,
+							3,
+							tt.fields.codecBytesEven,
+							tt.fields.numCodecBytesInEven,
+							tt.fields.ddBytesEven,
+							tt.fields.actBytesEven,
+							0,
+						)
+					} else {
+						n.push(
+							time.Now().UnixNano(),
+							i.seqNo,
+							i.seqNo+tt.fields.offset,
+							123,
+							tt.fields.markerOdd,
+							3,
+							tt.fields.codecBytesOdd,
+							tt.fields.numCodecBytesInOdd,
+							tt.fields.ddBytesOdd,
+							tt.fields.actBytesOdd,
+							0,
+						)
+					}
+				}
+			}
+
+			time.Sleep((ignoreRetransmission + 10) * time.Millisecond)
+			g := n.getExtPacketMetas(tt.args.seqNo)
+			var got []uint16
+			for _, sn := range g {
+				got = append(got, uint16(sn.sourceSeqNo))
+				if sn.sourceSeqNo%2 == 0 {
+					require.Equal(t, tt.fields.markerEven, sn.marker)
+					require.Equal(t, tt.fields.codecBytesEven, sn.codecBytes[:sn.numCodecBytesOut])
+					require.Equal(t, uint8(tt.fields.numCodecBytesInEven), sn.numCodecBytesIn)
+					require.Equal(t, tt.fields.ddBytesEven, sn.ddBytes[:sn.ddBytesSize])
+					require.Equal(t, uint8(len(tt.fields.ddBytesEven)), sn.ddBytesSize)
+					require.Equal(t, tt.fields.actBytesEven, sn.actBytes)
+				} else {
+					require.Equal(t, tt.fields.markerOdd, sn.marker)
+					require.Equal(t, tt.fields.codecBytesOdd, sn.codecBytes[:sn.numCodecBytesOut])
+					require.Equal(t, uint8(tt.fields.numCodecBytesInOdd), sn.numCodecBytesIn)
+					require.Equal(t, tt.fields.ddBytesOdd, sn.ddBytes[:sn.ddBytesSize])
+					require.Equal(t, uint8(len(tt.fields.ddBytesOdd)), sn.ddBytesSize)
+					require.Equal(t, tt.fields.actBytesOdd, sn.actBytes)
+				}
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("getExtPacketMetas() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/sfu/sfu.go

@@ -0,0 +1,36 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sfu
+
+import (
+	"sync"
+
+	"github.com/pion/rtp"
+)
+
+var (
+	PacketFactory = &sync.Pool{
+		New: func() any {
+			b := make([]byte, 1460)
+			return &b
+		},
+	}
+
+	RTPHeaderFactory = &sync.Pool{
+		New: func() any {
+			return &rtp.Header{}
+		},
+	}
+)

pkg/sfu/sfufakes/fake_down_track_listener.go

@@ -0,0 +1,290 @@
+// Code generated by counterfeiter. DO NOT EDIT.
+package sfufakes
+
+import (
+	"sync"
+	"time"
+
+	"github.com/livekit/livekit-server/pkg/sfu"
+	"github.com/livekit/protocol/livekit"
+	webrtc "github.com/pion/webrtc/v4"
+)
+
+type FakeDownTrackListener struct {
+	OnBindAndConnectedStub        func()
+	onBindAndConnectedMutex       sync.RWMutex
+	onBindAndConnectedArgsForCall []struct {
+	}
+	OnCodecNegotiatedStub        func(webrtc.RTPCodecCapability)
+	onCodecNegotiatedMutex       sync.RWMutex
+	onCodecNegotiatedArgsForCall []struct {
+		arg1 webrtc.RTPCodecCapability
+	}
+	OnDownTrackCloseStub        func(bool)
+	onDownTrackCloseMutex       sync.RWMutex
+	onDownTrackCloseArgsForCall []struct {
+		arg1 bool
+	}
+	OnMaxSubscribedLayerChangedStub        func(int32)
+	onMaxSubscribedLayerChangedMutex       sync.RWMutex
+	onMaxSubscribedLayerChangedArgsForCall []struct {
+		arg1 int32
+	}
+	OnRttUpdateStub        func(uint32)
+	onRttUpdateMutex       sync.RWMutex
+	onRttUpdateArgsForCall []struct {
+		arg1 uint32
+	}
+	OnStatsUpdateStub        func(*livekit.AnalyticsStat)
+	onStatsUpdateMutex       sync.RWMutex
+	onStatsUpdateArgsForCall []struct {
+		arg1 *livekit.AnalyticsStat
+	}
+	OnStreamStartedStub        func(time.Duration)
+	onStreamStartedMutex       sync.RWMutex
+	onStreamStartedArgsForCall []struct {
+		arg1 time.Duration
+	}
+	invocations      map[string][][]interface{}
+	invocationsMutex sync.RWMutex
+}
+
+func (fake *FakeDownTrackListener) OnBindAndConnected() {
+	fake.onBindAndConnectedMutex.Lock()
+	fake.onBindAndConnectedArgsForCall = append(fake.onBindAndConnectedArgsForCall, struct {
+	}{})
+	stub := fake.OnBindAndConnectedStub
+	fake.recordInvocation("OnBindAndConnected", []interface{}{})
+	fake.onBindAndConnectedMutex.Unlock()
+	if stub != nil {
+		fake.OnBindAndConnectedStub()
+	}
+}
+
+func (fake *FakeDownTrackListener) OnBindAndConnectedCallCount() int {
+	fake.onBindAndConnectedMutex.RLock()
+	defer fake.onBindAndConnectedMutex.RUnlock()
+	return len(fake.onBindAndConnectedArgsForCall)
+}
+
+func (fake *FakeDownTrackListener) OnBindAndConnectedCalls(stub func()) {
+	fake.onBindAndConnectedMutex.Lock()
+	defer fake.onBindAndConnectedMutex.Unlock()
+	fake.OnBindAndConnectedStub = stub
+}
+
+func (fake *FakeDownTrackListener) OnCodecNegotiated(arg1 webrtc.RTPCodecCapability) {
+	fake.onCodecNegotiatedMutex.Lock()
+	fake.onCodecNegotiatedArgsForCall = append(fake.onCodecNegotiatedArgsForCall, struct {
+		arg1 webrtc.RTPCodecCapability
+	}{arg1})
+	stub := fake.OnCodecNegotiatedStub
+	fake.recordInvocation("OnCodecNegotiated", []interface{}{arg1})
+	fake.onCodecNegotiatedMutex.Unlock()
+	if stub != nil {
+		fake.OnCodecNegotiatedStub(arg1)
+	}
+}
+
+func (fake *FakeDownTrackListener) OnCodecNegotiatedCallCount() int {
+	fake.onCodecNegotiatedMutex.RLock()
+	defer fake.onCodecNegotiatedMutex.RUnlock()
+	return len(fake.onCodecNegotiatedArgsForCall)
+}
+
+func (fake *FakeDownTrackListener) OnCodecNegotiatedCalls(stub func(webrtc.RTPCodecCapability)) {
+	fake.onCodecNegotiatedMutex.Lock()
+	defer fake.onCodecNegotiatedMutex.Unlock()
+	fake.OnCodecNegotiatedStub = stub
+}
+
+func (fake *FakeDownTrackListener) OnCodecNegotiatedArgsForCall(i int) webrtc.RTPCodecCapability {
+	fake.onCodecNegotiatedMutex.RLock()
+	defer fake.onCodecNegotiatedMutex.RUnlock()
+	argsForCall := fake.onCodecNegotiatedArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeDownTrackListener) OnDownTrackClose(arg1 bool) {
+	fake.onDownTrackCloseMutex.Lock()
+	fake.onDownTrackCloseArgsForCall = append(fake.onDownTrackCloseArgsForCall, struct {
+		arg1 bool
+	}{arg1})
+	stub := fake.OnDownTrackCloseStub
+	fake.recordInvocation("OnDownTrackClose", []interface{}{arg1})
+	fake.onDownTrackCloseMutex.Unlock()
+	if stub != nil {
+		fake.OnDownTrackCloseStub(arg1)
+	}
+}
+
+func (fake *FakeDownTrackListener) OnDownTrackCloseCallCount() int {
+	fake.onDownTrackCloseMutex.RLock()
+	defer fake.onDownTrackCloseMutex.RUnlock()
+	return len(fake.onDownTrackCloseArgsForCall)
+}
+
+func (fake *FakeDownTrackListener) OnDownTrackCloseCalls(stub func(bool)) {
+	fake.onDownTrackCloseMutex.Lock()
+	defer fake.onDownTrackCloseMutex.Unlock()
+	fake.OnDownTrackCloseStub = stub
+}
+
+func (fake *FakeDownTrackListener) OnDownTrackCloseArgsForCall(i int) bool {
+	fake.onDownTrackCloseMutex.RLock()
+	defer fake.onDownTrackCloseMutex.RUnlock()
+	argsForCall := fake.onDownTrackCloseArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeDownTrackListener) OnMaxSubscribedLayerChanged(arg1 int32) {
+	fake.onMaxSubscribedLayerChangedMutex.Lock()
+	fake.onMaxSubscribedLayerChangedArgsForCall = append(fake.onMaxSubscribedLayerChangedArgsForCall, struct {
+		arg1 int32
+	}{arg1})
+	stub := fake.OnMaxSubscribedLayerChangedStub
+	fake.recordInvocation("OnMaxSubscribedLayerChanged", []interface{}{arg1})
+	fake.onMaxSubscribedLayerChangedMutex.Unlock()
+	if stub != nil {
+		fake.OnMaxSubscribedLayerChangedStub(arg1)
+	}
+}
+
+func (fake *FakeDownTrackListener) OnMaxSubscribedLayerChangedCallCount() int {
+	fake.onMaxSubscribedLayerChangedMutex.RLock()
+	defer fake.onMaxSubscribedLayerChangedMutex.RUnlock()
+	return len(fake.onMaxSubscribedLayerChangedArgsForCall)
+}
+
+func (fake *FakeDownTrackListener) OnMaxSubscribedLayerChangedCalls(stub func(int32)) {
+	fake.onMaxSubscribedLayerChangedMutex.Lock()
+	defer fake.onMaxSubscribedLayerChangedMutex.Unlock()
+	fake.OnMaxSubscribedLayerChangedStub = stub
+}
+
+func (fake *FakeDownTrackListener) OnMaxSubscribedLayerChangedArgsForCall(i int) int32 {
+	fake.onMaxSubscribedLayerChangedMutex.RLock()
+	defer fake.onMaxSubscribedLayerChangedMutex.RUnlock()
+	argsForCall := fake.onMaxSubscribedLayerChangedArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeDownTrackListener) OnRttUpdate(arg1 uint32) {
+	fake.onRttUpdateMutex.Lock()
+	fake.onRttUpdateArgsForCall = append(fake.onRttUpdateArgsForCall, struct {
+		arg1 uint32
+	}{arg1})
+	stub := fake.OnRttUpdateStub
+	fake.recordInvocation("OnRttUpdate", []interface{}{arg1})
+	fake.onRttUpdateMutex.Unlock()
+	if stub != nil {
+		fake.OnRttUpdateStub(arg1)
+	}
+}
+
+func (fake *FakeDownTrackListener) OnRttUpdateCallCount() int {
+	fake.onRttUpdateMutex.RLock()
+	defer fake.onRttUpdateMutex.RUnlock()
+	return len(fake.onRttUpdateArgsForCall)
+}
+
+func (fake *FakeDownTrackListener) OnRttUpdateCalls(stub func(uint32)) {
+	fake.onRttUpdateMutex.Lock()
+	defer fake.onRttUpdateMutex.Unlock()
+	fake.OnRttUpdateStub = stub
+}
+
+func (fake *FakeDownTrackListener) OnRttUpdateArgsForCall(i int) uint32 {
+	fake.onRttUpdateMutex.RLock()
+	defer fake.onRttUpdateMutex.RUnlock()
+	argsForCall := fake.onRttUpdateArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeDownTrackListener) OnStatsUpdate(arg1 *livekit.AnalyticsStat) {
+	fake.onStatsUpdateMutex.Lock()
+	fake.onStatsUpdateArgsForCall = append(fake.onStatsUpdateArgsForCall, struct {
+		arg1 *livekit.AnalyticsStat
+	}{arg1})
+	stub := fake.OnStatsUpdateStub
+	fake.recordInvocation("OnStatsUpdate", []interface{}{arg1})
+	fake.onStatsUpdateMutex.Unlock()
+	if stub != nil {
+		fake.OnStatsUpdateStub(arg1)
+	}
+}
+
+func (fake *FakeDownTrackListener) OnStatsUpdateCallCount() int {
+	fake.onStatsUpdateMutex.RLock()
+	defer fake.onStatsUpdateMutex.RUnlock()
+	return len(fake.onStatsUpdateArgsForCall)
+}
+
+func (fake *FakeDownTrackListener) OnStatsUpdateCalls(stub func(*livekit.AnalyticsStat)) {
+	fake.onStatsUpdateMutex.Lock()
+	defer fake.onStatsUpdateMutex.Unlock()
+	fake.OnStatsUpdateStub = stub
+}
+
+func (fake *FakeDownTrackListener) OnStatsUpdateArgsForCall(i int) *livekit.AnalyticsStat {
+	fake.onStatsUpdateMutex.RLock()
+	defer fake.onStatsUpdateMutex.RUnlock()
+	argsForCall := fake.onStatsUpdateArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeDownTrackListener) OnStreamStarted(arg1 time.Duration) {
+	fake.onStreamStartedMutex.Lock()
+	fake.onStreamStartedArgsForCall = append(fake.onStreamStartedArgsForCall, struct {
+		arg1 time.Duration
+	}{arg1})
+	stub := fake.OnStreamStartedStub
+	fake.recordInvocation("OnStreamStarted", []interface{}{arg1})
+	fake.onStreamStartedMutex.Unlock()
+	if stub != nil {
+		fake.OnStreamStartedStub(arg1)
+	}
+}
+
+func (fake *FakeDownTrackListener) OnStreamStartedCallCount() int {
+	fake.onStreamStartedMutex.RLock()
+	defer fake.onStreamStartedMutex.RUnlock()
+	return len(fake.onStreamStartedArgsForCall)
+}
+
+func (fake *FakeDownTrackListener) OnStreamStartedCalls(stub func(time.Duration)) {
+	fake.onStreamStartedMutex.Lock()
+	defer fake.onStreamStartedMutex.Unlock()
+	fake.OnStreamStartedStub = stub
+}
+
+func (fake *FakeDownTrackListener) OnStreamStartedArgsForCall(i int) time.Duration {
+	fake.onStreamStartedMutex.RLock()
+	defer fake.onStreamStartedMutex.RUnlock()
+	argsForCall := fake.onStreamStartedArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeDownTrackListener) Invocations() map[string][][]interface{} {
+	fake.invocationsMutex.RLock()
+	defer fake.invocationsMutex.RUnlock()
+	copiedInvocations := map[string][][]interface{}{}
+	for key, value := range fake.invocations {
+		copiedInvocations[key] = value
+	}
+	return copiedInvocations
+}
+
+func (fake *FakeDownTrackListener) recordInvocation(key string, args []interface{}) {
+	fake.invocationsMutex.Lock()
+	defer fake.invocationsMutex.Unlock()
+	if fake.invocations == nil {
+		fake.invocations = map[string][][]interface{}{}
+	}
+	if fake.invocations[key] == nil {
+		fake.invocations[key] = [][]interface{}{}
+	}
+	fake.invocations[key] = append(fake.invocations[key], args)
+}
+
+var _ sfu.DownTrackListener = new(FakeDownTrackListener)

pkg/sfu/sfufakes/fake_down_track_stream_allocator_listener.go

@@ -0,0 +1,586 @@
+// Code generated by counterfeiter. DO NOT EDIT.
+package sfufakes
+
+import (
+	"sync"
+
+	"github.com/livekit/livekit-server/pkg/sfu"
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/sfu/bwe"
+	"github.com/pion/rtcp"
+)
+
+type FakeDownTrackStreamAllocatorListener struct {
+	BWETypeStub        func() bwe.BWEType
+	bWETypeMutex       sync.RWMutex
+	bWETypeArgsForCall []struct {
+	}
+	bWETypeReturns struct {
+		result1 bwe.BWEType
+	}
+	bWETypeReturnsOnCall map[int]struct {
+		result1 bwe.BWEType
+	}
+	IsBWEEnabledStub        func(*sfu.DownTrack) bool
+	isBWEEnabledMutex       sync.RWMutex
+	isBWEEnabledArgsForCall []struct {
+		arg1 *sfu.DownTrack
+	}
+	isBWEEnabledReturns struct {
+		result1 bool
+	}
+	isBWEEnabledReturnsOnCall map[int]struct {
+		result1 bool
+	}
+	IsSubscribeMutableStub        func(*sfu.DownTrack) bool
+	isSubscribeMutableMutex       sync.RWMutex
+	isSubscribeMutableArgsForCall []struct {
+		arg1 *sfu.DownTrack
+	}
+	isSubscribeMutableReturns struct {
+		result1 bool
+	}
+	isSubscribeMutableReturnsOnCall map[int]struct {
+		result1 bool
+	}
+	OnAvailableLayersChangedStub        func(*sfu.DownTrack)
+	onAvailableLayersChangedMutex       sync.RWMutex
+	onAvailableLayersChangedArgsForCall []struct {
+		arg1 *sfu.DownTrack
+	}
+	OnBitrateAvailabilityChangedStub        func(*sfu.DownTrack)
+	onBitrateAvailabilityChangedMutex       sync.RWMutex
+	onBitrateAvailabilityChangedArgsForCall []struct {
+		arg1 *sfu.DownTrack
+	}
+	OnMaxPublishedSpatialChangedStub        func(*sfu.DownTrack)
+	onMaxPublishedSpatialChangedMutex       sync.RWMutex
+	onMaxPublishedSpatialChangedArgsForCall []struct {
+		arg1 *sfu.DownTrack
+	}
+	OnMaxPublishedTemporalChangedStub        func(*sfu.DownTrack)
+	onMaxPublishedTemporalChangedMutex       sync.RWMutex
+	onMaxPublishedTemporalChangedArgsForCall []struct {
+		arg1 *sfu.DownTrack
+	}
+	OnREMBStub        func(*sfu.DownTrack, *rtcp.ReceiverEstimatedMaximumBitrate)
+	onREMBMutex       sync.RWMutex
+	onREMBArgsForCall []struct {
+		arg1 *sfu.DownTrack
+		arg2 *rtcp.ReceiverEstimatedMaximumBitrate
+	}
+	OnResumeStub        func(*sfu.DownTrack)
+	onResumeMutex       sync.RWMutex
+	onResumeArgsForCall []struct {
+		arg1 *sfu.DownTrack
+	}
+	OnSubscribedLayerChangedStub        func(*sfu.DownTrack, buffer.VideoLayer)
+	onSubscribedLayerChangedMutex       sync.RWMutex
+	onSubscribedLayerChangedArgsForCall []struct {
+		arg1 *sfu.DownTrack
+		arg2 buffer.VideoLayer
+	}
+	OnSubscriptionChangedStub        func(*sfu.DownTrack)
+	onSubscriptionChangedMutex       sync.RWMutex
+	onSubscriptionChangedArgsForCall []struct {
+		arg1 *sfu.DownTrack
+	}
+	OnTransportCCFeedbackStub        func(*sfu.DownTrack, *rtcp.TransportLayerCC)
+	onTransportCCFeedbackMutex       sync.RWMutex
+	onTransportCCFeedbackArgsForCall []struct {
+		arg1 *sfu.DownTrack
+		arg2 *rtcp.TransportLayerCC
+	}
+	invocations      map[string][][]interface{}
+	invocationsMutex sync.RWMutex
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) BWEType() bwe.BWEType {
+	fake.bWETypeMutex.Lock()
+	ret, specificReturn := fake.bWETypeReturnsOnCall[len(fake.bWETypeArgsForCall)]
+	fake.bWETypeArgsForCall = append(fake.bWETypeArgsForCall, struct {
+	}{})
+	stub := fake.BWETypeStub
+	fakeReturns := fake.bWETypeReturns
+	fake.recordInvocation("BWEType", []interface{}{})
+	fake.bWETypeMutex.Unlock()
+	if stub != nil {
+		return stub()
+	}
+	if specificReturn {
+		return ret.result1
+	}
+	return fakeReturns.result1
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) BWETypeCallCount() int {
+	fake.bWETypeMutex.RLock()
+	defer fake.bWETypeMutex.RUnlock()
+	return len(fake.bWETypeArgsForCall)
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) BWETypeCalls(stub func() bwe.BWEType) {
+	fake.bWETypeMutex.Lock()
+	defer fake.bWETypeMutex.Unlock()
+	fake.BWETypeStub = stub
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) BWETypeReturns(result1 bwe.BWEType) {
+	fake.bWETypeMutex.Lock()
+	defer fake.bWETypeMutex.Unlock()
+	fake.BWETypeStub = nil
+	fake.bWETypeReturns = struct {
+		result1 bwe.BWEType
+	}{result1}
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) BWETypeReturnsOnCall(i int, result1 bwe.BWEType) {
+	fake.bWETypeMutex.Lock()
+	defer fake.bWETypeMutex.Unlock()
+	fake.BWETypeStub = nil
+	if fake.bWETypeReturnsOnCall == nil {
+		fake.bWETypeReturnsOnCall = make(map[int]struct {
+			result1 bwe.BWEType
+		})
+	}
+	fake.bWETypeReturnsOnCall[i] = struct {
+		result1 bwe.BWEType
+	}{result1}
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) IsBWEEnabled(arg1 *sfu.DownTrack) bool {
+	fake.isBWEEnabledMutex.Lock()
+	ret, specificReturn := fake.isBWEEnabledReturnsOnCall[len(fake.isBWEEnabledArgsForCall)]
+	fake.isBWEEnabledArgsForCall = append(fake.isBWEEnabledArgsForCall, struct {
+		arg1 *sfu.DownTrack
+	}{arg1})
+	stub := fake.IsBWEEnabledStub
+	fakeReturns := fake.isBWEEnabledReturns
+	fake.recordInvocation("IsBWEEnabled", []interface{}{arg1})
+	fake.isBWEEnabledMutex.Unlock()
+	if stub != nil {
+		return stub(arg1)
+	}
+	if specificReturn {
+		return ret.result1
+	}
+	return fakeReturns.result1
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) IsBWEEnabledCallCount() int {
+	fake.isBWEEnabledMutex.RLock()
+	defer fake.isBWEEnabledMutex.RUnlock()
+	return len(fake.isBWEEnabledArgsForCall)
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) IsBWEEnabledCalls(stub func(*sfu.DownTrack) bool) {
+	fake.isBWEEnabledMutex.Lock()
+	defer fake.isBWEEnabledMutex.Unlock()
+	fake.IsBWEEnabledStub = stub
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) IsBWEEnabledArgsForCall(i int) *sfu.DownTrack {
+	fake.isBWEEnabledMutex.RLock()
+	defer fake.isBWEEnabledMutex.RUnlock()
+	argsForCall := fake.isBWEEnabledArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) IsBWEEnabledReturns(result1 bool) {
+	fake.isBWEEnabledMutex.Lock()
+	defer fake.isBWEEnabledMutex.Unlock()
+	fake.IsBWEEnabledStub = nil
+	fake.isBWEEnabledReturns = struct {
+		result1 bool
+	}{result1}
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) IsBWEEnabledReturnsOnCall(i int, result1 bool) {
+	fake.isBWEEnabledMutex.Lock()
+	defer fake.isBWEEnabledMutex.Unlock()
+	fake.IsBWEEnabledStub = nil
+	if fake.isBWEEnabledReturnsOnCall == nil {
+		fake.isBWEEnabledReturnsOnCall = make(map[int]struct {
+			result1 bool
+		})
+	}
+	fake.isBWEEnabledReturnsOnCall[i] = struct {
+		result1 bool
+	}{result1}
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) IsSubscribeMutable(arg1 *sfu.DownTrack) bool {
+	fake.isSubscribeMutableMutex.Lock()
+	ret, specificReturn := fake.isSubscribeMutableReturnsOnCall[len(fake.isSubscribeMutableArgsForCall)]
+	fake.isSubscribeMutableArgsForCall = append(fake.isSubscribeMutableArgsForCall, struct {
+		arg1 *sfu.DownTrack
+	}{arg1})
+	stub := fake.IsSubscribeMutableStub
+	fakeReturns := fake.isSubscribeMutableReturns
+	fake.recordInvocation("IsSubscribeMutable", []interface{}{arg1})
+	fake.isSubscribeMutableMutex.Unlock()
+	if stub != nil {
+		return stub(arg1)
+	}
+	if specificReturn {
+		return ret.result1
+	}
+	return fakeReturns.result1
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) IsSubscribeMutableCallCount() int {
+	fake.isSubscribeMutableMutex.RLock()
+	defer fake.isSubscribeMutableMutex.RUnlock()
+	return len(fake.isSubscribeMutableArgsForCall)
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) IsSubscribeMutableCalls(stub func(*sfu.DownTrack) bool) {
+	fake.isSubscribeMutableMutex.Lock()
+	defer fake.isSubscribeMutableMutex.Unlock()
+	fake.IsSubscribeMutableStub = stub
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) IsSubscribeMutableArgsForCall(i int) *sfu.DownTrack {
+	fake.isSubscribeMutableMutex.RLock()
+	defer fake.isSubscribeMutableMutex.RUnlock()
+	argsForCall := fake.isSubscribeMutableArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) IsSubscribeMutableReturns(result1 bool) {
+	fake.isSubscribeMutableMutex.Lock()
+	defer fake.isSubscribeMutableMutex.Unlock()
+	fake.IsSubscribeMutableStub = nil
+	fake.isSubscribeMutableReturns = struct {
+		result1 bool
+	}{result1}
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) IsSubscribeMutableReturnsOnCall(i int, result1 bool) {
+	fake.isSubscribeMutableMutex.Lock()
+	defer fake.isSubscribeMutableMutex.Unlock()
+	fake.IsSubscribeMutableStub = nil
+	if fake.isSubscribeMutableReturnsOnCall == nil {
+		fake.isSubscribeMutableReturnsOnCall = make(map[int]struct {
+			result1 bool
+		})
+	}
+	fake.isSubscribeMutableReturnsOnCall[i] = struct {
+		result1 bool
+	}{result1}
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnAvailableLayersChanged(arg1 *sfu.DownTrack) {
+	fake.onAvailableLayersChangedMutex.Lock()
+	fake.onAvailableLayersChangedArgsForCall = append(fake.onAvailableLayersChangedArgsForCall, struct {
+		arg1 *sfu.DownTrack
+	}{arg1})
+	stub := fake.OnAvailableLayersChangedStub
+	fake.recordInvocation("OnAvailableLayersChanged", []interface{}{arg1})
+	fake.onAvailableLayersChangedMutex.Unlock()
+	if stub != nil {
+		fake.OnAvailableLayersChangedStub(arg1)
+	}
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnAvailableLayersChangedCallCount() int {
+	fake.onAvailableLayersChangedMutex.RLock()
+	defer fake.onAvailableLayersChangedMutex.RUnlock()
+	return len(fake.onAvailableLayersChangedArgsForCall)
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnAvailableLayersChangedCalls(stub func(*sfu.DownTrack)) {
+	fake.onAvailableLayersChangedMutex.Lock()
+	defer fake.onAvailableLayersChangedMutex.Unlock()
+	fake.OnAvailableLayersChangedStub = stub
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnAvailableLayersChangedArgsForCall(i int) *sfu.DownTrack {
+	fake.onAvailableLayersChangedMutex.RLock()
+	defer fake.onAvailableLayersChangedMutex.RUnlock()
+	argsForCall := fake.onAvailableLayersChangedArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnBitrateAvailabilityChanged(arg1 *sfu.DownTrack) {
+	fake.onBitrateAvailabilityChangedMutex.Lock()
+	fake.onBitrateAvailabilityChangedArgsForCall = append(fake.onBitrateAvailabilityChangedArgsForCall, struct {
+		arg1 *sfu.DownTrack
+	}{arg1})
+	stub := fake.OnBitrateAvailabilityChangedStub
+	fake.recordInvocation("OnBitrateAvailabilityChanged", []interface{}{arg1})
+	fake.onBitrateAvailabilityChangedMutex.Unlock()
+	if stub != nil {
+		fake.OnBitrateAvailabilityChangedStub(arg1)
+	}
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnBitrateAvailabilityChangedCallCount() int {
+	fake.onBitrateAvailabilityChangedMutex.RLock()
+	defer fake.onBitrateAvailabilityChangedMutex.RUnlock()
+	return len(fake.onBitrateAvailabilityChangedArgsForCall)
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnBitrateAvailabilityChangedCalls(stub func(*sfu.DownTrack)) {
+	fake.onBitrateAvailabilityChangedMutex.Lock()
+	defer fake.onBitrateAvailabilityChangedMutex.Unlock()
+	fake.OnBitrateAvailabilityChangedStub = stub
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnBitrateAvailabilityChangedArgsForCall(i int) *sfu.DownTrack {
+	fake.onBitrateAvailabilityChangedMutex.RLock()
+	defer fake.onBitrateAvailabilityChangedMutex.RUnlock()
+	argsForCall := fake.onBitrateAvailabilityChangedArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnMaxPublishedSpatialChanged(arg1 *sfu.DownTrack) {
+	fake.onMaxPublishedSpatialChangedMutex.Lock()
+	fake.onMaxPublishedSpatialChangedArgsForCall = append(fake.onMaxPublishedSpatialChangedArgsForCall, struct {
+		arg1 *sfu.DownTrack
+	}{arg1})
+	stub := fake.OnMaxPublishedSpatialChangedStub
+	fake.recordInvocation("OnMaxPublishedSpatialChanged", []interface{}{arg1})
+	fake.onMaxPublishedSpatialChangedMutex.Unlock()
+	if stub != nil {
+		fake.OnMaxPublishedSpatialChangedStub(arg1)
+	}
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnMaxPublishedSpatialChangedCallCount() int {
+	fake.onMaxPublishedSpatialChangedMutex.RLock()
+	defer fake.onMaxPublishedSpatialChangedMutex.RUnlock()
+	return len(fake.onMaxPublishedSpatialChangedArgsForCall)
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnMaxPublishedSpatialChangedCalls(stub func(*sfu.DownTrack)) {
+	fake.onMaxPublishedSpatialChangedMutex.Lock()
+	defer fake.onMaxPublishedSpatialChangedMutex.Unlock()
+	fake.OnMaxPublishedSpatialChangedStub = stub
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnMaxPublishedSpatialChangedArgsForCall(i int) *sfu.DownTrack {
+	fake.onMaxPublishedSpatialChangedMutex.RLock()
+	defer fake.onMaxPublishedSpatialChangedMutex.RUnlock()
+	argsForCall := fake.onMaxPublishedSpatialChangedArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnMaxPublishedTemporalChanged(arg1 *sfu.DownTrack) {
+	fake.onMaxPublishedTemporalChangedMutex.Lock()
+	fake.onMaxPublishedTemporalChangedArgsForCall = append(fake.onMaxPublishedTemporalChangedArgsForCall, struct {
+		arg1 *sfu.DownTrack
+	}{arg1})
+	stub := fake.OnMaxPublishedTemporalChangedStub
+	fake.recordInvocation("OnMaxPublishedTemporalChanged", []interface{}{arg1})
+	fake.onMaxPublishedTemporalChangedMutex.Unlock()
+	if stub != nil {
+		fake.OnMaxPublishedTemporalChangedStub(arg1)
+	}
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnMaxPublishedTemporalChangedCallCount() int {
+	fake.onMaxPublishedTemporalChangedMutex.RLock()
+	defer fake.onMaxPublishedTemporalChangedMutex.RUnlock()
+	return len(fake.onMaxPublishedTemporalChangedArgsForCall)
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnMaxPublishedTemporalChangedCalls(stub func(*sfu.DownTrack)) {
+	fake.onMaxPublishedTemporalChangedMutex.Lock()
+	defer fake.onMaxPublishedTemporalChangedMutex.Unlock()
+	fake.OnMaxPublishedTemporalChangedStub = stub
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnMaxPublishedTemporalChangedArgsForCall(i int) *sfu.DownTrack {
+	fake.onMaxPublishedTemporalChangedMutex.RLock()
+	defer fake.onMaxPublishedTemporalChangedMutex.RUnlock()
+	argsForCall := fake.onMaxPublishedTemporalChangedArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnREMB(arg1 *sfu.DownTrack, arg2 *rtcp.ReceiverEstimatedMaximumBitrate) {
+	fake.onREMBMutex.Lock()
+	fake.onREMBArgsForCall = append(fake.onREMBArgsForCall, struct {
+		arg1 *sfu.DownTrack
+		arg2 *rtcp.ReceiverEstimatedMaximumBitrate
+	}{arg1, arg2})
+	stub := fake.OnREMBStub
+	fake.recordInvocation("OnREMB", []interface{}{arg1, arg2})
+	fake.onREMBMutex.Unlock()
+	if stub != nil {
+		fake.OnREMBStub(arg1, arg2)
+	}
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnREMBCallCount() int {
+	fake.onREMBMutex.RLock()
+	defer fake.onREMBMutex.RUnlock()
+	return len(fake.onREMBArgsForCall)
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnREMBCalls(stub func(*sfu.DownTrack, *rtcp.ReceiverEstimatedMaximumBitrate)) {
+	fake.onREMBMutex.Lock()
+	defer fake.onREMBMutex.Unlock()
+	fake.OnREMBStub = stub
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnREMBArgsForCall(i int) (*sfu.DownTrack, *rtcp.ReceiverEstimatedMaximumBitrate) {
+	fake.onREMBMutex.RLock()
+	defer fake.onREMBMutex.RUnlock()
+	argsForCall := fake.onREMBArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnResume(arg1 *sfu.DownTrack) {
+	fake.onResumeMutex.Lock()
+	fake.onResumeArgsForCall = append(fake.onResumeArgsForCall, struct {
+		arg1 *sfu.DownTrack
+	}{arg1})
+	stub := fake.OnResumeStub
+	fake.recordInvocation("OnResume", []interface{}{arg1})
+	fake.onResumeMutex.Unlock()
+	if stub != nil {
+		fake.OnResumeStub(arg1)
+	}
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnResumeCallCount() int {
+	fake.onResumeMutex.RLock()
+	defer fake.onResumeMutex.RUnlock()
+	return len(fake.onResumeArgsForCall)
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnResumeCalls(stub func(*sfu.DownTrack)) {
+	fake.onResumeMutex.Lock()
+	defer fake.onResumeMutex.Unlock()
+	fake.OnResumeStub = stub
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnResumeArgsForCall(i int) *sfu.DownTrack {
+	fake.onResumeMutex.RLock()
+	defer fake.onResumeMutex.RUnlock()
+	argsForCall := fake.onResumeArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnSubscribedLayerChanged(arg1 *sfu.DownTrack, arg2 buffer.VideoLayer) {
+	fake.onSubscribedLayerChangedMutex.Lock()
+	fake.onSubscribedLayerChangedArgsForCall = append(fake.onSubscribedLayerChangedArgsForCall, struct {
+		arg1 *sfu.DownTrack
+		arg2 buffer.VideoLayer
+	}{arg1, arg2})
+	stub := fake.OnSubscribedLayerChangedStub
+	fake.recordInvocation("OnSubscribedLayerChanged", []interface{}{arg1, arg2})
+	fake.onSubscribedLayerChangedMutex.Unlock()
+	if stub != nil {
+		fake.OnSubscribedLayerChangedStub(arg1, arg2)
+	}
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnSubscribedLayerChangedCallCount() int {
+	fake.onSubscribedLayerChangedMutex.RLock()
+	defer fake.onSubscribedLayerChangedMutex.RUnlock()
+	return len(fake.onSubscribedLayerChangedArgsForCall)
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnSubscribedLayerChangedCalls(stub func(*sfu.DownTrack, buffer.VideoLayer)) {
+	fake.onSubscribedLayerChangedMutex.Lock()
+	defer fake.onSubscribedLayerChangedMutex.Unlock()
+	fake.OnSubscribedLayerChangedStub = stub
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnSubscribedLayerChangedArgsForCall(i int) (*sfu.DownTrack, buffer.VideoLayer) {
+	fake.onSubscribedLayerChangedMutex.RLock()
+	defer fake.onSubscribedLayerChangedMutex.RUnlock()
+	argsForCall := fake.onSubscribedLayerChangedArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnSubscriptionChanged(arg1 *sfu.DownTrack) {
+	fake.onSubscriptionChangedMutex.Lock()
+	fake.onSubscriptionChangedArgsForCall = append(fake.onSubscriptionChangedArgsForCall, struct {
+		arg1 *sfu.DownTrack
+	}{arg1})
+	stub := fake.OnSubscriptionChangedStub
+	fake.recordInvocation("OnSubscriptionChanged", []interface{}{arg1})
+	fake.onSubscriptionChangedMutex.Unlock()
+	if stub != nil {
+		fake.OnSubscriptionChangedStub(arg1)
+	}
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnSubscriptionChangedCallCount() int {
+	fake.onSubscriptionChangedMutex.RLock()
+	defer fake.onSubscriptionChangedMutex.RUnlock()
+	return len(fake.onSubscriptionChangedArgsForCall)
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnSubscriptionChangedCalls(stub func(*sfu.DownTrack)) {
+	fake.onSubscriptionChangedMutex.Lock()
+	defer fake.onSubscriptionChangedMutex.Unlock()
+	fake.OnSubscriptionChangedStub = stub
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnSubscriptionChangedArgsForCall(i int) *sfu.DownTrack {
+	fake.onSubscriptionChangedMutex.RLock()
+	defer fake.onSubscriptionChangedMutex.RUnlock()
+	argsForCall := fake.onSubscriptionChangedArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnTransportCCFeedback(arg1 *sfu.DownTrack, arg2 *rtcp.TransportLayerCC) {
+	fake.onTransportCCFeedbackMutex.Lock()
+	fake.onTransportCCFeedbackArgsForCall = append(fake.onTransportCCFeedbackArgsForCall, struct {
+		arg1 *sfu.DownTrack
+		arg2 *rtcp.TransportLayerCC
+	}{arg1, arg2})
+	stub := fake.OnTransportCCFeedbackStub
+	fake.recordInvocation("OnTransportCCFeedback", []interface{}{arg1, arg2})
+	fake.onTransportCCFeedbackMutex.Unlock()
+	if stub != nil {
+		fake.OnTransportCCFeedbackStub(arg1, arg2)
+	}
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnTransportCCFeedbackCallCount() int {
+	fake.onTransportCCFeedbackMutex.RLock()
+	defer fake.onTransportCCFeedbackMutex.RUnlock()
+	return len(fake.onTransportCCFeedbackArgsForCall)
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnTransportCCFeedbackCalls(stub func(*sfu.DownTrack, *rtcp.TransportLayerCC)) {
+	fake.onTransportCCFeedbackMutex.Lock()
+	defer fake.onTransportCCFeedbackMutex.Unlock()
+	fake.OnTransportCCFeedbackStub = stub
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) OnTransportCCFeedbackArgsForCall(i int) (*sfu.DownTrack, *rtcp.TransportLayerCC) {
+	fake.onTransportCCFeedbackMutex.RLock()
+	defer fake.onTransportCCFeedbackMutex.RUnlock()
+	argsForCall := fake.onTransportCCFeedbackArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) Invocations() map[string][][]interface{} {
+	fake.invocationsMutex.RLock()
+	defer fake.invocationsMutex.RUnlock()
+	copiedInvocations := map[string][][]interface{}{}
+	for key, value := range fake.invocations {
+		copiedInvocations[key] = value
+	}
+	return copiedInvocations
+}
+
+func (fake *FakeDownTrackStreamAllocatorListener) recordInvocation(key string, args []interface{}) {
+	fake.invocationsMutex.Lock()
+	defer fake.invocationsMutex.Unlock()
+	if fake.invocations == nil {
+		fake.invocations = map[string][][]interface{}{}
+	}
+	if fake.invocations[key] == nil {
+		fake.invocations[key] = [][]interface{}{}
+	}
+	fake.invocations[key] = append(fake.invocations[key], args)
+}
+
+var _ sfu.DownTrackStreamAllocatorListener = new(FakeDownTrackStreamAllocatorListener)