snapshot of https://github.com/livekit/livekit.git @ 08c5433449c106943231037d0b892895f6e92d3b (2026-09-15)

pkg/sfu/buffer/buffer.go

@@ -0,0 +1,540 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+
+	"github.com/pion/rtcp"
+	"github.com/pion/rtp"
+	"github.com/pion/webrtc/v4"
+
+	sutils "github.com/livekit/livekit-server/pkg/utils"
+	"github.com/livekit/mediatransportutil/pkg/bucket"
+	"github.com/livekit/mediatransportutil/pkg/twcc"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/utils/mono"
+)
+
+const (
+	rtcpReceiverReportDelta = 1e9
+
+	InitPacketBufferSizeVideo = 300
+	InitPacketBufferSizeAudio = 70
+)
+
+var (
+	errInvalidCodec = errors.New("invalid codec")
+)
+
+var _ BufferProvider = (*Buffer)(nil)
+
+type pendingPacket struct {
+	arrivalTime int64
+	packet      []byte
+}
+
+// Buffer contains all packets
+type Buffer struct {
+	*BufferBase
+
+	pPackets     []pendingPacket
+	lastReportAt int64
+	isBound      bool
+
+	twcc      *twcc.Responder
+	twccExtID uint8
+
+	enableAudioLossProxying  bool
+	lastFractionLostToReport uint8 // Last fraction lost from subscribers, should report to publisher; Audio only
+
+	lastPacketRead int
+
+	// callbacks
+	onClose         func()
+	onRtcpFeedback  func([]rtcp.Packet)
+	onFinalRtpStats func(*livekit.RTPStats)
+	onNotifyRTX     func(uint32, uint32, string)
+
+	primaryBufferForRTX *Buffer
+	rtxPktBuf           []byte
+
+	streamInfoProbe       *StreamInfoProbe
+	warnedPendingOverflow bool
+}
+
+// StreamInfoProbe identifies a stream from the mid/rid/rsid header extensions of its
+// packets. It runs on the write path, i. e. as SRTP pushes into this buffer, because
+// nothing reads remote streams through pion's interceptor chain.
+type StreamInfoProbe struct {
+	MidExtID  uint8
+	RidExtID  uint8
+	RsidExtID uint8
+
+	// Tries bounds how many packets are inspected before giving up.
+	Tries int
+
+	// OnFound is called at most once, in a goroutine, as it can re-enter this buffer.
+	OnFound func(ssrc uint32, mid, rid, rsid string)
+}
+
+func NewBuffer(ssrc uint32, maxVideoPkts, maxAudioPkts int) *Buffer {
+	b := &Buffer{}
+	b.BufferBase = NewBufferBase(BufferBaseParams{
+		SSRC:               ssrc,
+		MaxVideoPkts:       maxVideoPkts,
+		MaxAudioPkts:       maxAudioPkts,
+		LoggerComponents:   []string{sutils.ComponentPub, sutils.ComponentSFU},
+		SendPLI:            b.sendPLI,
+		IsReportingEnabled: true,
+	})
+	return b
+}
+
+func (b *Buffer) SetTWCCAndExtID(twcc *twcc.Responder, extID uint8) {
+	b.Lock()
+	defer b.Unlock()
+
+	b.twcc = twcc
+	b.twccExtID = extID
+}
+
+func (b *Buffer) SetAudioLossProxying(enable bool) {
+	b.Lock()
+	defer b.Unlock()
+
+	b.enableAudioLossProxying = enable
+}
+
+func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapability, bitrates int) error {
+	b.Lock()
+	if b.isBound {
+		b.Unlock()
+		return nil
+	}
+
+	if err := b.BufferBase.BindLocked(params, codec, bitrates); err != nil {
+		b.Unlock()
+		return err
+	}
+
+	b.lastReportAt = mono.UnixNano()
+
+	if len(b.pPackets) != 0 {
+		b.logger.Debugw("releasing queued packets on bind", "count", len(b.pPackets))
+	}
+	var rtcpPackets []rtcp.Packet
+	for _, pp := range b.pPackets {
+		rtcpPackets = append(rtcpPackets, b.calc(pp.packet, nil, pp.arrivalTime, true, false)...)
+	}
+	b.pPackets = nil
+
+	b.isBound = true
+	b.Unlock()
+
+	if len(rtcpPackets) != 0 {
+		if cb := b.getOnRtcpFeedback(); cb != nil {
+			cb(rtcpPackets)
+		}
+	}
+
+	return nil
+}
+
+// Write adds an RTP Packet, ordering is not guaranteed, newer packets may arrive later
+func (b *Buffer) Write(pkt []byte) (n int, err error) {
+	var rtpPacket rtp.Packet
+	err = rtpPacket.Unmarshal(pkt)
+	if err != nil {
+		return
+	}
+
+	b.Lock()
+	if b.BufferBase.IsClosed() {
+		b.Unlock()
+		err = io.EOF
+		return
+	}
+
+	now := mono.UnixNano()
+	if b.twcc != nil && b.twccExtID != 0 {
+		if ext := rtpPacket.GetExtension(b.twccExtID); len(ext) >= 2 {
+			b.twcc.Push(rtpPacket.SSRC, binary.BigEndian.Uint16(ext[0:2]), now, rtpPacket.Marker)
+		}
+	}
+
+	// libwebrtc will use 0 ssrc for probing, don't push the packet to pending queue to avoid memory increasing since
+	// the Bind will not be called to consume the pending packets. More details in https://github.com/pion/webrtc/pull/2816
+	if rtpPacket.SSRC == 0 {
+		b.Unlock()
+		return
+	}
+
+	if b.streamInfoProbe != nil {
+		b.probeStreamInfoLocked(&rtpPacket)
+	}
+
+	// handle RTX packet
+	if pb := b.primaryBufferForRTX; pb != nil {
+		b.Unlock()
+
+		// skip padding only packets
+		if rtpPacket.Padding && len(rtpPacket.Payload) == 0 {
+			return
+		}
+
+		pb.writeRTX(&rtpPacket, now)
+		return
+	}
+
+	if !b.isBound {
+		packet := make([]byte, len(pkt))
+		copy(packet, pkt)
+
+		if len(b.pPackets) == 0 {
+			b.logger.Debugw("received first packet")
+		}
+
+		startIdx := 0
+		overflow := len(b.pPackets) - max(b.BufferBase.MaxVideoPkts(), b.BufferBase.MaxAudioPkts())
+		if overflow > 0 {
+			startIdx = overflow
+
+			// a stream that keeps arriving but never binds drops every packet from here
+			// on; for an RTX stream it means the pairing was never established
+			if !b.warnedPendingOverflow {
+				b.warnedPendingOverflow = true
+				b.logger.Warnw(
+					"unbound buffer overflowing, dropping packets", nil,
+					"ssrc", b.BufferBase.SSRC(),
+					"pending", len(b.pPackets),
+				)
+			}
+		}
+		b.pPackets = append(b.pPackets[startIdx:], pendingPacket{
+			packet:      packet,
+			arrivalTime: now,
+		})
+
+		b.BufferBase.NotifyRead()
+		b.Unlock()
+		return
+	}
+
+	rtcpPackets := b.calc(pkt, &rtpPacket, now, false, false)
+	b.Unlock()
+
+	if len(rtcpPackets) != 0 {
+		if cb := b.getOnRtcpFeedback(); cb != nil {
+			cb(rtcpPackets)
+		}
+	}
+	return
+}
+
+// SetStreamInfoProbe installs probe and runs it over packets already queued.
+func (b *Buffer) SetStreamInfoProbe(probe *StreamInfoProbe) {
+	b.Lock()
+	defer b.Unlock()
+
+	b.streamInfoProbe = probe
+
+	for _, pp := range b.pPackets {
+		if b.streamInfoProbe == nil {
+			return
+		}
+
+		var rtpPacket rtp.Packet
+		if err := rtpPacket.Unmarshal(pp.packet); err != nil {
+			continue
+		}
+		b.probeStreamInfoLocked(&rtpPacket)
+	}
+}
+
+// probeStreamInfoLocked inspects one packet, clearing the probe once the stream is
+// identified or the try budget runs out.
+func (b *Buffer) probeStreamInfoLocked(rtpPacket *rtp.Packet) {
+	probe := b.streamInfoProbe
+
+	var mid, rid, rsid string
+	if ext := rtpPacket.GetExtension(probe.MidExtID); ext != nil {
+		mid = string(ext)
+	}
+	if ext := rtpPacket.GetExtension(probe.RidExtID); ext != nil {
+		rid = string(ext)
+	}
+	if ext := rtpPacket.GetExtension(probe.RsidExtID); ext != nil {
+		rsid = string(ext)
+	}
+
+	if mid != "" && (rid != "" || rsid != "") {
+		b.streamInfoProbe = nil
+		b.logger.Debugw("stream found", "ssrc", rtpPacket.SSRC, "mid", mid, "rid", rid, "rsid", rsid)
+		go probe.OnFound(rtpPacket.SSRC, mid, rid, rsid)
+		return
+	}
+
+	// ignore padding only packets for probe count
+	if rtpPacket.Padding && len(rtpPacket.Payload) == 0 {
+		return
+	}
+
+	if probe.Tries--; probe.Tries <= 0 {
+		b.streamInfoProbe = nil
+	}
+}
+
+func (b *Buffer) SetPrimaryBufferForRTX(primaryBuffer *Buffer) {
+	b.Lock()
+	b.primaryBufferForRTX = primaryBuffer
+	pkts := b.pPackets
+	b.pPackets = nil
+	b.Unlock()
+
+	for _, pp := range pkts {
+		var rtpPacket rtp.Packet
+		err := rtpPacket.Unmarshal(pp.packet)
+		if err != nil {
+			continue
+		}
+		if rtpPacket.Padding && len(rtpPacket.Payload) == 0 {
+			continue
+		}
+		primaryBuffer.writeRTX(&rtpPacket, pp.arrivalTime)
+	}
+}
+
+func (b *Buffer) NotifyRTX(ssrc uint32, repairSSRC uint32, rsid string) {
+	if onNotifyRTX := b.getOnNotifyRTX(); onNotifyRTX != nil {
+		onNotifyRTX(ssrc, repairSSRC, rsid)
+	}
+}
+
+func (b *Buffer) writeRTX(rtxPkt *rtp.Packet, arrivalTime int64) {
+	b.Lock()
+	defer b.Unlock()
+	if !b.isBound {
+		return
+	}
+
+	if rtxPkt.PayloadType != b.rtxPayloadType {
+		b.logger.Debugw("unexpected rtx payload type", "expected", b.rtxPayloadType, "actual", rtxPkt.PayloadType)
+		return
+	}
+
+	if b.rtxPktBuf == nil {
+		b.rtxPktBuf = make([]byte, bucket.RTPMaxPktSize)
+	}
+
+	if len(rtxPkt.Payload) < 2 {
+		b.logger.Warnw("rtx payload too short", nil, "size", len(rtxPkt.Payload))
+		return
+	}
+
+	repairedPkt := *rtxPkt
+	repairedPkt.PayloadType = b.payloadType
+	repairedPkt.SequenceNumber = binary.BigEndian.Uint16(rtxPkt.Payload[:2])
+	repairedPkt.SSRC = b.BufferBase.SSRC()
+	repairedPkt.Payload = rtxPkt.Payload[2:]
+	n, err := repairedPkt.MarshalTo(b.rtxPktBuf)
+	if err != nil {
+		b.logger.Errorw("could not marshal repaired packet", err, "ssrc", b.BufferBase.SSRC(), "sn", repairedPkt.SequenceNumber)
+		return
+	}
+
+	b.calc(b.rtxPktBuf[:n], &repairedPkt, arrivalTime, false, true)
+}
+
+func (b *Buffer) Read(buff []byte) (n int, err error) {
+	b.Lock()
+	for {
+		if b.BufferBase.IsClosed() {
+			b.Unlock()
+			return 0, io.EOF
+		}
+
+		if b.pPackets != nil && len(b.pPackets) > b.lastPacketRead {
+			if len(buff) < len(b.pPackets[b.lastPacketRead].packet) {
+				b.Unlock()
+				return 0, bucket.ErrBufferTooSmall
+			}
+
+			n = copy(buff, b.pPackets[b.lastPacketRead].packet)
+			b.lastPacketRead++
+			b.Unlock()
+			return
+		}
+		b.BufferBase.WaitRead()
+	}
+}
+
+func (b *Buffer) Close() error {
+	stats, err := b.BufferBase.CloseWithReason("close")
+	if err != nil {
+		return err
+	}
+
+	if stats != nil {
+		if cb := b.getOnFinalRtpStats(); cb != nil {
+			cb(stats)
+		}
+	}
+
+	if cb := b.getOnClose(); cb != nil {
+		cb()
+	}
+
+	return nil
+}
+
+func (b *Buffer) OnClose(fn func()) {
+	b.Lock()
+	b.onClose = fn
+	b.Unlock()
+}
+
+func (b *Buffer) getOnClose() func() {
+	b.RLock()
+	defer b.RUnlock()
+
+	return b.onClose
+}
+
+func (b *Buffer) sendPLI() {
+	ssrc := b.BufferBase.SSRC()
+	if ssrc == 0 {
+		return
+	}
+
+	b.logger.Debugw("send pli", "mediaSSRC", ssrc)
+	pli := []rtcp.Packet{
+		&rtcp.PictureLossIndication{
+			SenderSSRC: ssrc,
+			MediaSSRC:  ssrc,
+		},
+	}
+
+	if cb := b.getOnRtcpFeedback(); cb != nil {
+		cb(pli)
+	}
+}
+
+func (b *Buffer) calc(rawPkt []byte, rtpPacket *rtp.Packet, arrivalTime int64, isBuffered bool, isRTX bool) []rtcp.Packet {
+	b.BufferBase.HandleIncomingPacketLocked(
+		rawPkt,
+		rtpPacket,
+		arrivalTime,
+		isBuffered,
+		isRTX,
+		nil,
+		0,
+	)
+
+	return b.getRTCPPackets(arrivalTime)
+}
+
+func (b *Buffer) getRTCPPackets(arrivalTime int64) []rtcp.Packet {
+	var pkts []rtcp.Packet
+	if nackPkt := b.getNACKPacket(); nackPkt != nil {
+		pkts = append(pkts, nackPkt)
+	}
+	if receiverReport := b.getRTCPReceiverReport(arrivalTime); receiverReport != nil {
+		pkts = append(pkts, receiverReport)
+	}
+
+	return pkts
+}
+
+func (b *Buffer) getNACKPacket() *rtcp.TransportLayerNack {
+	if nacks := b.BufferBase.GetNACKPairsLocked(); len(nacks) > 0 {
+		ssrc := b.BufferBase.SSRC()
+		return &rtcp.TransportLayerNack{
+			SenderSSRC: ssrc,
+			MediaSSRC:  ssrc,
+			Nacks:      nacks,
+		}
+	}
+	return nil
+}
+
+func (b *Buffer) getRTCPReceiverReport(arrivalTime int64) *rtcp.ReceiverReport {
+	if arrivalTime-b.lastReportAt < rtcpReceiverReportDelta {
+		return nil
+	}
+	b.lastReportAt = arrivalTime
+
+	proxyLoss := b.lastFractionLostToReport
+	if b.codecType == webrtc.RTPCodecTypeAudio && !b.enableAudioLossProxying {
+		proxyLoss = 0
+	}
+
+	if receptionReport := b.BufferBase.GetRtcpReceptionReportLocked(proxyLoss); receptionReport != nil {
+		return &rtcp.ReceiverReport{
+			SSRC:    b.BufferBase.SSRC(),
+			Reports: []rtcp.ReceptionReport{*receptionReport},
+		}
+	}
+
+	return nil
+}
+
+func (b *Buffer) SetLastFractionLostReport(lost uint8) {
+	b.Lock()
+	defer b.Unlock()
+
+	b.lastFractionLostToReport = lost
+}
+
+func (b *Buffer) OnRtcpFeedback(fn func(fb []rtcp.Packet)) {
+	b.Lock()
+	b.onRtcpFeedback = fn
+	b.Unlock()
+}
+
+func (b *Buffer) getOnRtcpFeedback() func(fb []rtcp.Packet) {
+	b.RLock()
+	defer b.RUnlock()
+
+	return b.onRtcpFeedback
+}
+
+func (b *Buffer) OnFinalRtpStats(fn func(*livekit.RTPStats)) {
+	b.Lock()
+	b.onFinalRtpStats = fn
+	b.Unlock()
+}
+
+func (b *Buffer) getOnFinalRtpStats() func(*livekit.RTPStats) {
+	b.RLock()
+	defer b.RUnlock()
+
+	return b.onFinalRtpStats
+}
+
+func (b *Buffer) OnNotifyRTX(fn func(ssrc uint32, repairSSRC uint32, rsid string)) {
+	b.Lock()
+	b.onNotifyRTX = fn
+	b.Unlock()
+}
+
+func (b *Buffer) getOnNotifyRTX() func(ssrc uint32, repairSSRC uint32, rsid string) {
+	b.RLock()
+	defer b.RUnlock()
+
+	return b.onNotifyRTX
+}

pkg/sfu/buffer/buffer_test.go

@@ -0,0 +1,479 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"fmt"
+	"math"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/pion/rtcp"
+	"github.com/pion/rtp"
+	"github.com/pion/webrtc/v4"
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/mediatransportutil/pkg/nack"
+)
+
+var h265Codec = webrtc.RTPCodecParameters{
+	RTPCodecCapability: webrtc.RTPCodecCapability{
+		MimeType:  "video/h265",
+		ClockRate: 90000,
+		RTCPFeedback: []webrtc.RTCPFeedback{{
+			Type: "nack",
+		}},
+	},
+	PayloadType: 116,
+}
+
+var vp8Codec = webrtc.RTPCodecParameters{
+	RTPCodecCapability: webrtc.RTPCodecCapability{
+		MimeType:  "video/vp8",
+		ClockRate: 90000,
+		RTCPFeedback: []webrtc.RTCPFeedback{{
+			Type: "nack",
+		}},
+	},
+	PayloadType: 96,
+}
+
+var opusCodec = webrtc.RTPCodecParameters{
+	RTPCodecCapability: webrtc.RTPCodecCapability{
+		MimeType:  "audio/opus",
+		ClockRate: 48000,
+	},
+	PayloadType: 111,
+}
+
+func TestNack(t *testing.T) {
+	t.Run("nack normal", func(t *testing.T) {
+		buff := NewBuffer(123, 1, 1)
+		buff.codecType = webrtc.RTPCodecTypeVideo
+		require.NotNil(t, buff)
+		var wg sync.WaitGroup
+		// 5 tries
+		wg.Add(5)
+		buff.OnRtcpFeedback(func(fb []rtcp.Packet) {
+			for _, pkt := range fb {
+				switch p := pkt.(type) {
+				case *rtcp.TransportLayerNack:
+					if p.Nacks[0].PacketList()[0] == 1 && p.MediaSSRC == 123 {
+						wg.Done()
+					}
+				}
+			}
+		})
+		buff.Bind(webrtc.RTPParameters{
+			HeaderExtensions: nil,
+			Codecs:           []webrtc.RTPCodecParameters{vp8Codec},
+		}, vp8Codec.RTPCodecCapability, 0)
+		rtt := uint32(20)
+		buff.nacker.SetRTT(rtt)
+		for i := range 15 {
+			if i == 1 {
+				continue
+			}
+			if i < 14 {
+				time.Sleep(time.Duration(float64(rtt)*math.Pow(nack.NackQueueParamsDefault.BackoffFactor, float64(i))+10) * time.Millisecond)
+			} else {
+				time.Sleep(500 * time.Millisecond) // even a long wait should not exceed max retries
+			}
+			pkt := rtp.Packet{
+				Header: rtp.Header{
+					Version:        2,
+					PayloadType:    96,
+					SequenceNumber: uint16(i),
+					Timestamp:      uint32(i),
+					SSRC:           123,
+				},
+				Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
+			}
+			b, err := pkt.Marshal()
+			require.NoError(t, err)
+			_, err = buff.Write(b)
+			require.NoError(t, err)
+		}
+		wg.Wait()
+
+	})
+
+	t.Run("nack with seq wrap", func(t *testing.T) {
+		buff := NewBuffer(123, 1, 1)
+		buff.codecType = webrtc.RTPCodecTypeVideo
+		require.NotNil(t, buff)
+		var wg sync.WaitGroup
+		expects := map[uint16]int{
+			65534: 0,
+			65535: 0,
+			0:     0,
+			1:     0,
+		}
+		wg.Add(5 * len(expects)) // retry 5 times
+		buff.OnRtcpFeedback(func(fb []rtcp.Packet) {
+			for _, pkt := range fb {
+				switch p := pkt.(type) {
+				case *rtcp.TransportLayerNack:
+					if p.MediaSSRC == 123 {
+						for _, v := range p.Nacks {
+							v.Range(func(seq uint16) bool {
+								if _, ok := expects[seq]; ok {
+									wg.Done()
+								} else {
+									require.Fail(t, "unexpected nack seq ", seq)
+								}
+								return true
+							})
+						}
+					}
+				}
+			}
+		})
+		buff.Bind(webrtc.RTPParameters{
+			HeaderExtensions: nil,
+			Codecs:           []webrtc.RTPCodecParameters{vp8Codec},
+		}, vp8Codec.RTPCodecCapability, 0)
+		rtt := uint32(30)
+		buff.nacker.SetRTT(rtt)
+		for i := range 15 {
+			if i > 0 && i < 5 {
+				continue
+			}
+			if i < 14 {
+				time.Sleep(time.Duration(float64(rtt)*math.Pow(nack.NackQueueParamsDefault.BackoffFactor, float64(i))+10) * time.Millisecond)
+			} else {
+				time.Sleep(500 * time.Millisecond) // even a long wait should not exceed max retries
+			}
+			pkt := rtp.Packet{
+				Header: rtp.Header{
+					Version:        2,
+					PayloadType:    96,
+					SequenceNumber: uint16(i + 65533),
+					Timestamp:      uint32(i),
+					SSRC:           123,
+				},
+				Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
+			}
+			b, err := pkt.Marshal()
+			require.NoError(t, err)
+			_, err = buff.Write(b)
+			require.NoError(t, err)
+		}
+		wg.Wait()
+
+	})
+}
+
+func TestNewBuffer(t *testing.T) {
+	tests := []struct {
+		name string
+	}{
+		{
+			name: "Must not be nil and add packets in sequence",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var TestPackets = []*rtp.Packet{
+				{
+					Header: rtp.Header{
+						Version:        2,
+						PayloadType:    96,
+						SequenceNumber: 65533,
+						SSRC:           123,
+					},
+				},
+				{
+					Header: rtp.Header{
+						Version:        2,
+						PayloadType:    96,
+						SequenceNumber: 65534,
+						SSRC:           123,
+					},
+					Payload: []byte{1},
+				},
+				{
+					Header: rtp.Header{
+						Version:        2,
+						PayloadType:    96,
+						SequenceNumber: 2,
+						SSRC:           123,
+					},
+				},
+				{
+					Header: rtp.Header{
+						Version:        2,
+						PayloadType:    96,
+						SequenceNumber: 65535,
+						SSRC:           123,
+					},
+				},
+			}
+			buff := NewBuffer(123, 1, 1)
+			buff.codecType = webrtc.RTPCodecTypeVideo
+			require.NotNil(t, buff)
+			buff.OnRtcpFeedback(func(_ []rtcp.Packet) {})
+			buff.Bind(webrtc.RTPParameters{
+				HeaderExtensions: nil,
+				Codecs:           []webrtc.RTPCodecParameters{vp8Codec},
+			}, vp8Codec.RTPCodecCapability, 0)
+
+			for _, p := range TestPackets {
+				buf, _ := p.Marshal()
+				_, _ = buff.Write(buf)
+			}
+			require.Equal(t, uint16(2), buff.rtpStats.HighestSequenceNumber())
+			require.Equal(t, uint64(65536+2), buff.rtpStats.ExtendedHighestSequenceNumber())
+		})
+	}
+}
+
+func TestFractionLostReport(t *testing.T) {
+	buff := NewBuffer(123, 1, 1)
+	require.NotNil(t, buff)
+
+	var wg sync.WaitGroup
+
+	// with loss proxying
+	wg.Add(1)
+	buff.SetAudioLossProxying(true)
+	buff.SetLastFractionLostReport(55)
+	buff.OnRtcpFeedback(func(fb []rtcp.Packet) {
+		for _, pkt := range fb {
+			switch p := pkt.(type) {
+			case *rtcp.ReceiverReport:
+				for _, v := range p.Reports {
+					require.EqualValues(t, 55, v.FractionLost)
+				}
+				wg.Done()
+			}
+		}
+	})
+	buff.Bind(webrtc.RTPParameters{
+		HeaderExtensions: nil,
+		Codecs:           []webrtc.RTPCodecParameters{opusCodec},
+	}, opusCodec.RTPCodecCapability, 0)
+	for i := range 15 {
+		pkt := rtp.Packet{
+			Header: rtp.Header{
+				Version:        2,
+				PayloadType:    111,
+				SequenceNumber: uint16(i),
+				Timestamp:      uint32(i),
+				SSRC:           123,
+			},
+			Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
+		}
+		b, err := pkt.Marshal()
+		require.NoError(t, err)
+		if i == 1 {
+			time.Sleep(1 * time.Second)
+		}
+		_, err = buff.Write(b)
+		require.NoError(t, err)
+	}
+	wg.Wait()
+
+	wg.Add(1)
+	buff.SetAudioLossProxying(false)
+	buff.OnRtcpFeedback(func(fb []rtcp.Packet) {
+		for _, pkt := range fb {
+			switch p := pkt.(type) {
+			case *rtcp.ReceiverReport:
+				for _, v := range p.Reports {
+					require.EqualValues(t, 0, v.FractionLost)
+				}
+				wg.Done()
+			}
+		}
+	})
+	buff.Bind(webrtc.RTPParameters{
+		HeaderExtensions: nil,
+		Codecs:           []webrtc.RTPCodecParameters{opusCodec},
+	}, opusCodec.RTPCodecCapability, 0)
+	for i := range 15 {
+		pkt := rtp.Packet{
+			Header: rtp.Header{
+				Version:        2,
+				PayloadType:    111,
+				SequenceNumber: uint16(i),
+				Timestamp:      uint32(i),
+				SSRC:           123,
+			},
+			Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
+		}
+		b, err := pkt.Marshal()
+		require.NoError(t, err)
+		if i == 1 {
+			time.Sleep(1 * time.Second)
+		}
+		_, err = buff.Write(b)
+		require.NoError(t, err)
+	}
+	wg.Wait()
+}
+
+func TestCodecChange(t *testing.T) {
+	// codec change before bind
+	buff := NewBuffer(123, 1, 1)
+	require.NotNil(t, buff)
+	changedCodec := make(chan webrtc.RTPCodecParameters, 1)
+	buff.OnCodecChange(func(rp webrtc.RTPCodecParameters) {
+		select {
+		case changedCodec <- rp:
+		default:
+			t.Fatalf("codec change not consumed")
+		}
+	})
+	buff.OnStreamRestart(func(reason string) {
+		require.Equal(t, "codec-change", reason)
+
+		// read once to clear pending restart
+		var buf [1500]byte
+		extPkt, err := buff.ReadExtended(buf[:])
+		require.NoError(t, err)
+		require.Nil(t, extPkt)
+	})
+
+	h265Pkt := rtp.Packet{
+		Header: rtp.Header{
+			Version:        2,
+			PayloadType:    116,
+			SequenceNumber: 1,
+			Timestamp:      1,
+			SSRC:           123,
+		},
+		Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
+	}
+	buf, err := h265Pkt.Marshal()
+	require.NoError(t, err)
+	_, err = buff.Write(buf)
+	require.NoError(t, err)
+
+	select {
+	case <-changedCodec:
+		t.Fatalf("unexpected codec change")
+	case <-time.After(100 * time.Millisecond):
+	}
+
+	// Bind sets up VP8 as expected codec,
+	// packet written to the buffer above before Bind is H.265,
+	// that should trigger a codec change and a stream restart
+	// when the queued packets from Write before Bind get flushed
+	buff.Bind(
+		webrtc.RTPParameters{
+			HeaderExtensions: nil,
+			Codecs:           []webrtc.RTPCodecParameters{vp8Codec, h265Codec},
+		},
+		vp8Codec.RTPCodecCapability,
+		0,
+	)
+
+	select {
+	case c := <-changedCodec:
+		require.Equal(t, h265Codec, c)
+	case <-time.After(1 * time.Second):
+		t.Fatalf("expected codec change")
+	}
+
+	// second codec change - writing VP8 packet after Bind should trigger another codec change
+	vp8Pkt := rtp.Packet{
+		Header: rtp.Header{
+			Version:        2,
+			PayloadType:    96,
+			SequenceNumber: 3,
+			Timestamp:      3,
+			SSRC:           123,
+		},
+		Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
+	}
+	buf, err = vp8Pkt.Marshal()
+	require.NoError(t, err)
+	_, err = buff.Write(buf)
+	require.NoError(t, err)
+
+	select {
+	case c := <-changedCodec:
+		require.Equal(t, vp8Codec, c)
+	case <-time.After(1 * time.Second):
+		t.Fatalf("expected codec change")
+	}
+	fmt.Printf("done second codec change\n") // REMOVE
+
+	// out of order pkts can't cause codec change
+	// rewrite the VP8 packet to start the sequence after a stream restart
+	_, err = buff.Write(buf)
+	require.NoError(t, err)
+
+	h265Pkt.SequenceNumber = 2
+	h265Pkt.Timestamp = 2
+	buf, err = h265Pkt.Marshal()
+	require.NoError(t, err)
+	_, err = buff.Write(buf)
+	require.NoError(t, err)
+	select {
+	case <-changedCodec:
+		t.Fatalf("unexpected codec change")
+	case <-time.After(100 * time.Millisecond):
+	}
+
+	// unknown codec should not cause change even if it is in order
+	h265Pkt.SequenceNumber = 4
+	h265Pkt.Timestamp = 4
+	h265Pkt.PayloadType = 117
+	buf, err = h265Pkt.Marshal()
+	require.NoError(t, err)
+	_, err = buff.Write(buf)
+	require.NoError(t, err)
+	select {
+	case <-changedCodec:
+		t.Fatalf("unexpected codec change")
+	case <-time.After(100 * time.Millisecond):
+	}
+
+	// an in-order packet should change codec again
+	h265Pkt.SequenceNumber = 5
+	h265Pkt.Timestamp = 5
+	h265Pkt.PayloadType = 116
+	buf, err = h265Pkt.Marshal()
+	require.NoError(t, err)
+	_, err = buff.Write(buf)
+	require.NoError(t, err)
+	select {
+	case c := <-changedCodec:
+		require.Equal(t, h265Codec, c)
+	case <-time.After(1 * time.Second):
+		t.Fatalf("expected codec change")
+	}
+}
+
+func BenchmarkMemcpu(b *testing.B) {
+	buf := make([]byte, 1500*1500*10)
+	buf2 := make([]byte, 1500*1500*20)
+
+	for b.Loop() {
+		copy(buf2, buf)
+	}
+}
+
+func BenchmarkExtPacketFactory(b *testing.B) {
+
+	for b.Loop() {
+		extPkt := ExtPacketFactory.Get().(*ExtPacket)
+		*extPkt = ExtPacket{}
+		ExtPacketFactory.Put(extPkt)
+	}
+}

pkg/sfu/buffer/datastats.go

@@ -0,0 +1,102 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"sync"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	"github.com/livekit/protocol/livekit"
+)
+
+type DataStatsParam struct {
+	WindowDuration time.Duration
+}
+
+type DataStats struct {
+	params      DataStatsParam
+	lock        sync.RWMutex
+	totalBytes  int64
+	startTime   time.Time
+	endTime     time.Time
+	windowStart int64
+	windowBytes int64
+}
+
+func NewDataStats(params DataStatsParam) *DataStats {
+	return &DataStats{
+		params:      params,
+		startTime:   time.Now(),
+		windowStart: time.Now().UnixNano(),
+	}
+}
+
+func (s *DataStats) Update(bytes int, time int64) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	s.totalBytes += int64(bytes)
+
+	if s.params.WindowDuration > 0 && time-s.windowStart > s.params.WindowDuration.Nanoseconds() {
+		s.windowBytes = 0
+		s.windowStart = time
+	}
+	s.windowBytes += int64(bytes)
+}
+
+func (s *DataStats) ToProtoActive() *livekit.RTPStats {
+	if s.params.WindowDuration == 0 {
+		return &livekit.RTPStats{}
+	}
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+	now := time.Now().UnixNano()
+	duration := now - s.windowStart
+	if duration > s.params.WindowDuration.Nanoseconds() {
+		return &livekit.RTPStats{}
+	}
+
+	return &livekit.RTPStats{
+		StartTime: timestamppb.New(time.Unix(s.windowStart/1e9, s.windowStart%1e9)),
+		EndTime:   timestamppb.New(time.Unix(0, now)),
+		Duration:  float64(duration / 1e9),
+		Bytes:     uint64(s.windowBytes),
+		Bitrate:   float64(s.windowBytes) * 8 / float64(duration) / 1e9,
+	}
+}
+
+func (s *DataStats) Stop() {
+	s.lock.Lock()
+	s.endTime = time.Now()
+	s.lock.Unlock()
+}
+
+func (s *DataStats) ToProtoAggregateOnly() *livekit.RTPStats {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+
+	end := s.endTime
+	if end.IsZero() {
+		end = time.Now()
+	}
+	return &livekit.RTPStats{
+		StartTime: timestamppb.New(s.startTime),
+		EndTime:   timestamppb.New(end),
+		Duration:  end.Sub(s.startTime).Seconds(),
+		Bytes:     uint64(s.totalBytes),
+		Bitrate:   float64(s.totalBytes) * 8 / end.Sub(s.startTime).Seconds(),
+	}
+}

pkg/sfu/buffer/datastats_test.go

@@ -0,0 +1,53 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/livekit/protocol/livekit"
+)
+
+func TestDataStats(t *testing.T) {
+	stats := NewDataStats(DataStatsParam{WindowDuration: time.Second})
+
+	time.Sleep(time.Millisecond)
+	r := stats.ToProtoAggregateOnly()
+	require.Equal(t, r.StartTime.AsTime().UnixNano(), stats.startTime.UnixNano())
+	require.NotZero(t, r.EndTime)
+	require.NotZero(t, r.Duration)
+	r.StartTime = nil
+	r.EndTime = nil
+	r.Duration = 0
+	require.True(t, proto.Equal(r, &livekit.RTPStats{}))
+
+	stats.Update(100, time.Now().UnixNano())
+	r = stats.ToProtoActive()
+	require.EqualValues(t, 100, r.Bytes)
+	require.NotZero(t, r.Bitrate)
+
+	// wait for window duration
+	time.Sleep(time.Second)
+	r = stats.ToProtoActive()
+	require.True(t, proto.Equal(r, &livekit.RTPStats{}))
+	stats.Stop()
+	r = stats.ToProtoAggregateOnly()
+	require.EqualValues(t, 100, r.Bytes)
+	require.NotZero(t, r.Bitrate)
+}

pkg/sfu/buffer/dependencydescriptorparser.go

@@ -0,0 +1,364 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"fmt"
+	"slices"
+	"sync"
+	"time"
+
+	"github.com/pion/rtp"
+	"go.uber.org/atomic"
+
+	dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
+	"github.com/livekit/mediatransportutil/pkg/codec"
+	"github.com/livekit/mediatransportutil/pkg/utils"
+	"github.com/livekit/protocol/logger"
+)
+
+var (
+	ExtDependencyDescriptorFactory = &sync.Pool{
+		New: func() any {
+			return &ExtDependencyDescriptor{}
+		},
+	}
+)
+
+// --------------------------------------
+
+const (
+	ddRestartThreshold = 30 * time.Second
+
+	// frame integrity check 2 seconds for L3T3 30fps video
+	integrityCheckFrame = 180
+	integrityCheckPkt   = 1024
+)
+
+var (
+	ErrFrameEarlierThanKeyFrame            = fmt.Errorf("frame is earlier than current keyframe")
+	ErrDDStructureAttachedToNonFirstPacket = fmt.Errorf("dependency descriptor structure is attached to non-first packet of a frame")
+	ErrDDExtentionNotFound                 = fmt.Errorf("dependency descriptor extension not found")
+)
+
+type DependencyDescriptorParser struct {
+	structure         *dd.FrameDependencyStructure
+	ddExtID           uint8
+	logger            logger.Logger
+	onMaxLayerChanged func(int32, int32)
+	decodeTargets     []DependencyDescriptorDecodeTarget
+
+	seqWrapAround        *utils.WrapAround[uint16, uint64]
+	frameWrapAround      *utils.WrapAround[uint16, uint64]
+	structureExtFrameNum uint64
+	// drop threshold for frames belonging to a previous dependency structure.
+	// advanced only when the structure id actually changes: structureExtFrameNum
+	// advances on every structure-bearing key frame, so with frequent key frames
+	// repeating the same structure (e. g. screen content), valid late/retransmitted
+	// frames would be dropped as "earlier than current structure".
+	structureChangeExtFrameNum uint64
+	activeDecodeTargetsExtSeq  uint64
+	activeDecodeTargetsMask    uint32
+	frameChecker               *FrameIntegrityChecker
+
+	ddNotFoundCount atomic.Uint32
+
+	// restart detection
+	restartGeneration int
+	enableRestart     bool
+	lastPacketAt      time.Time
+}
+
+func NewDependencyDescriptorParser(ddExtID uint8, logger logger.Logger, onMaxLayerChanged func(int32, int32), enableRestart bool) *DependencyDescriptorParser {
+	return &DependencyDescriptorParser{
+		ddExtID:           ddExtID,
+		logger:            logger,
+		onMaxLayerChanged: onMaxLayerChanged,
+		seqWrapAround:     utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
+		frameWrapAround:   utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
+		frameChecker:      NewFrameIntegrityChecker(integrityCheckFrame, integrityCheckPkt),
+		enableRestart:     enableRestart,
+	}
+}
+
+type ExtDependencyDescriptor struct {
+	Descriptor *dd.DependencyDescriptor
+
+	DecodeTargets              []DependencyDescriptorDecodeTarget
+	StructureUpdated           bool
+	ActiveDecodeTargetsUpdated bool
+	Integrity                  bool
+	ExtFrameNum                uint64
+	// the frame number of the keyframe which the current frame depends on
+	ExtKeyFrameNum uint64
+
+	// increase when the stream restarts, clear and reinitialize all dd state includes
+	// attached structure, frame chain, decode target.
+	RestartGeneration int
+}
+
+func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescriptor, VideoLayer, error) {
+	var videoLayer VideoLayer
+	ddBuf := pkt.GetExtension(r.ddExtID)
+	if ddBuf == nil {
+		ddNotFoundCount := r.ddNotFoundCount.Inc()
+		if ddNotFoundCount%100 == 0 {
+			r.logger.Warnw("dependency descriptor extension is not present", nil, "seq", pkt.SequenceNumber, "count", ddNotFoundCount)
+		}
+		return nil, videoLayer, ErrDDExtentionNotFound
+	}
+
+	var ddVal dd.DependencyDescriptor
+	ext := &dd.DependencyDescriptorExtension{
+		Descriptor: &ddVal,
+		Structure:  r.structure,
+	}
+	_, err := ext.Unmarshal(ddBuf)
+	if err != nil {
+		if err != dd.ErrDDReaderNoStructure && err != dd.ErrDDReaderInvalidTemplateIndex {
+			r.logger.Infow("failed to parse generic dependency descriptor", err, "payload", pkt.PayloadType, "ddbufLen", len(ddBuf))
+		}
+		return nil, videoLayer, err
+	}
+
+	var restart bool
+	if r.enableRestart {
+		if !r.lastPacketAt.IsZero() && time.Since(r.lastPacketAt) > ddRestartThreshold {
+			r.restart()
+			restart = true
+			r.logger.Debugw(
+				"dependency descriptor parser restart stream",
+				"generation", r.restartGeneration,
+				"lastPacketAt", r.lastPacketAt,
+				"sinceLast", time.Since(r.lastPacketAt),
+				"frameWrapAround", r.frameWrapAround,
+			)
+		}
+		r.lastPacketAt = time.Now()
+	}
+
+	extSeq := r.seqWrapAround.Update(pkt.SequenceNumber).ExtendedVal
+
+	if ddVal.FrameDependencies != nil {
+		videoLayer.Spatial, videoLayer.Temporal = int32(ddVal.FrameDependencies.SpatialId), int32(ddVal.FrameDependencies.TemporalId)
+	}
+
+	// assume the packet is in-order when stream restarting
+	unwrapped := r.frameWrapAround.UpdateWithOrderKnown(ddVal.FrameNumber, restart)
+	extFN := unwrapped.ExtendedVal
+
+	if extFN < r.structureChangeExtFrameNum {
+		r.logger.Debugw(
+			"drop frame which is earlier than current structure",
+			"fn", ddVal.FrameNumber,
+			"extFN", extFN,
+			"structureChangeExtFrameNum", r.structureChangeExtFrameNum,
+			"unwrappedFN", unwrapped,
+			"frameWrapAround", r.frameWrapAround,
+		)
+		return nil, videoLayer, ErrFrameEarlierThanKeyFrame
+	}
+
+	r.frameChecker.AddPacket(extSeq, extFN, &ddVal)
+
+	extDD := ExtDependencyDescriptorFactory.Get().(*ExtDependencyDescriptor)
+	*extDD = ExtDependencyDescriptor{
+		Descriptor:        &ddVal,
+		ExtFrameNum:       extFN,
+		Integrity:         r.frameChecker.FrameIntegrity(extFN),
+		RestartGeneration: r.restartGeneration,
+	}
+
+	if ddVal.AttachedStructure != nil {
+		if !ddVal.FirstPacketInFrame {
+			r.logger.Warnw(
+				"attached structure is not the first packet in frame", nil,
+				"sn", pkt.SequenceNumber,
+				"extSeq", extSeq,
+				"fn", ddVal.FrameNumber,
+				"extFN", extFN,
+			)
+			ReleaseExtDependencyDescriptor(extDD)
+			return nil, videoLayer, ErrDDStructureAttachedToNonFirstPacket
+		}
+
+		if extFN < r.structureExtFrameNum {
+			// out-of-order key frame repeating the current structure: accepting it
+			// would regress structureExtFrameNum (ExtKeyFrameNum) and replay a stale
+			// structure update, confusing the downtrack's dependency descriptor
+			// selector.
+			r.logger.Debugw(
+				"drop out-of-order key frame",
+				"extFN", extFN,
+				"structureExtFrameNum", r.structureExtFrameNum,
+			)
+			ReleaseExtDependencyDescriptor(extDD)
+			return nil, videoLayer, ErrFrameEarlierThanKeyFrame
+		}
+
+		if r.structure == nil || ddVal.AttachedStructure.StructureId != r.structure.StructureId {
+			// structure actually changed (or first structure): advance the drop threshold
+			// so that only frames preceding this structure are dropped.
+			r.structureChangeExtFrameNum = extFN
+			r.logger.Debugw(
+				"structure updated",
+				"structureID", ddVal.AttachedStructure.StructureId,
+				"sn", pkt.SequenceNumber,
+				"extSeq", extSeq,
+				"fn", ddVal.FrameNumber,
+				"extFN", extFN,
+				"descriptor", ddVal.String(),
+				"unwrappedFN", unwrapped,
+				"frameWrapAround", r.frameWrapAround,
+			)
+		}
+		r.structure = ddVal.AttachedStructure
+		r.decodeTargets = ProcessFrameDependencyStructure(ddVal.AttachedStructure)
+		if extFN > unwrapped.PreExtendedHighest && extFN-unwrapped.PreExtendedHighest > 1000 {
+			r.logger.Debugw(
+				"large frame number jump on structure updating",
+				"fn", ddVal.FrameNumber,
+				"extFN", extFN,
+				"preExtendedHighest", unwrapped.PreExtendedHighest,
+				"structureExtFrameNum", r.structureExtFrameNum,
+				"unwrappedFN", unwrapped,
+				"frameWrapAround", r.frameWrapAround,
+			)
+		}
+		r.structureExtFrameNum = extFN
+		extDD.StructureUpdated = true
+		extDD.ActiveDecodeTargetsUpdated = true
+		// The dependency descriptor reader will always set ActiveDecodeTargetsBitmask for TemplateDependencyStructure is present,
+		// so don't need to notify max layer change here.
+	}
+
+	if mask := ddVal.ActiveDecodeTargetsBitmask; mask != nil && extSeq > r.activeDecodeTargetsExtSeq {
+		r.activeDecodeTargetsExtSeq = extSeq
+		if *mask != r.activeDecodeTargetsMask {
+			r.activeDecodeTargetsMask = *mask
+			extDD.ActiveDecodeTargetsUpdated = true
+			var maxSpatial, maxTemporal int32
+			for _, dt := range r.decodeTargets {
+				if *mask&(1<<dt.Target) != uint32(dd.DecodeTargetNotPresent) {
+					if maxSpatial < dt.Layer.Spatial {
+						maxSpatial = dt.Layer.Spatial
+					}
+					if maxTemporal < dt.Layer.Temporal {
+						maxTemporal = dt.Layer.Temporal
+					}
+				}
+			}
+			r.logger.Debugw("max layer changed", "maxSpatial", maxSpatial, "maxTemporal", maxTemporal)
+			r.onMaxLayerChanged(maxSpatial, maxTemporal)
+		}
+	}
+
+	extDD.DecodeTargets = r.decodeTargets
+	extDD.ExtKeyFrameNum = r.structureExtFrameNum
+
+	return extDD, videoLayer, nil
+}
+
+func (r *DependencyDescriptorParser) restart() {
+	r.frameChecker = NewFrameIntegrityChecker(integrityCheckFrame, integrityCheckPkt)
+	r.structure = nil
+	r.structureExtFrameNum = 0
+	r.structureChangeExtFrameNum = 0
+	r.activeDecodeTargetsExtSeq = 0
+	r.activeDecodeTargetsMask = 0
+	r.decodeTargets = r.decodeTargets[:0]
+	r.restartGeneration++
+}
+
+// ------------------------------------------------------------------------------
+
+type DependencyDescriptorDecodeTarget struct {
+	Target int
+	Layer  VideoLayer
+}
+
+func (dt *DependencyDescriptorDecodeTarget) String() string {
+	return fmt.Sprintf("DecodeTarget{t: %d, l: %+v}", dt.Target, dt.Layer)
+}
+
+// ------------------------------------------------------------------------------
+
+func ProcessFrameDependencyStructure(structure *dd.FrameDependencyStructure) []DependencyDescriptorDecodeTarget {
+	decodeTargets := make([]DependencyDescriptorDecodeTarget, 0, structure.NumDecodeTargets)
+	for target := 0; target < structure.NumDecodeTargets; target++ {
+		layer := VideoLayer{Spatial: 0, Temporal: 0}
+		for _, t := range structure.Templates {
+			if t.DecodeTargetIndications[target] != dd.DecodeTargetNotPresent {
+				if layer.Spatial < int32(t.SpatialId) {
+					layer.Spatial = int32(t.SpatialId)
+				}
+				if layer.Temporal < int32(t.TemporalId) {
+					layer.Temporal = int32(t.TemporalId)
+				}
+			}
+		}
+		decodeTargets = append(decodeTargets, DependencyDescriptorDecodeTarget{target, layer})
+	}
+
+	// sort decode target layer by spatial and temporal from high to low
+	slices.SortFunc(decodeTargets, func(a, b DependencyDescriptorDecodeTarget) int {
+		if a.Layer.GreaterThan(b.Layer) {
+			return -1
+		}
+		if b.Layer.GreaterThan(a.Layer) {
+			return 1
+		}
+		return 0
+	})
+
+	return decodeTargets
+}
+
+func GetActiveDecodeTargetBitmask(layer VideoLayer, decodeTargets []DependencyDescriptorDecodeTarget) *uint32 {
+	activeBitMask := uint32(0)
+	for _, dt := range decodeTargets {
+		if dt.Layer.Spatial <= layer.Spatial && dt.Layer.Temporal <= layer.Temporal {
+			activeBitMask |= 1 << dt.Target
+		}
+	}
+
+	return &activeBitMask
+}
+
+// ------------------------------------------------------------------------------
+
+func ExtractDependencyDescriptorVideoSize(dd *dd.DependencyDescriptor) []codec.VideoSize {
+	if dd.AttachedStructure == nil {
+		return nil
+	}
+
+	videoSizes := make([]codec.VideoSize, 0, len(dd.AttachedStructure.Resolutions))
+	for _, res := range dd.AttachedStructure.Resolutions {
+		videoSizes = append(videoSizes, codec.VideoSize{Width: uint32(res.Width), Height: uint32(res.Height)})
+	}
+
+	return videoSizes
+}
+
+// ------------------------------------------------------------------------------
+
+func ReleaseExtDependencyDescriptor(extDD *ExtDependencyDescriptor) {
+	if extDD == nil {
+		return
+	}
+
+	*extDD = ExtDependencyDescriptor{}
+	ExtDependencyDescriptorFactory.Put(extDD)
+}

pkg/sfu/buffer/dependencydescriptorparser_test.go

@@ -0,0 +1,188 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"testing"
+
+	"github.com/pion/rtp"
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/protocol/logger"
+
+	dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
+)
+
+const ddTestExtID = uint8(1)
+
+// single spatial layer, single temporal layer, one decode target, one chain
+func newL1T1Structure(structureID int) *dd.FrameDependencyStructure {
+	return &dd.FrameDependencyStructure{
+		StructureId:                  structureID,
+		NumDecodeTargets:             1,
+		NumChains:                    1,
+		DecodeTargetProtectedByChain: []int{0},
+		Templates: []*dd.FrameDependencyTemplate{
+			{ // key frame
+				DecodeTargetIndications: []dd.DecodeTargetIndication{dd.DecodeTargetSwitch},
+				ChainDiffs:              []int{0},
+			},
+			{ // delta frame
+				DecodeTargetIndications: []dd.DecodeTargetIndication{dd.DecodeTargetRequired},
+				FrameDiffs:              []int{1},
+				ChainDiffs:              []int{1},
+			},
+		},
+	}
+}
+
+type ddTestFeeder struct {
+	t      *testing.T
+	parser *DependencyDescriptorParser
+	// structure the writer marshals against, i. e. what the publisher last sent
+	structure *dd.FrameDependencyStructure
+	seq       uint16
+}
+
+func newDDTestFeeder(t *testing.T) *ddTestFeeder {
+	return &ddTestFeeder{
+		t:      t,
+		parser: NewDependencyDescriptorParser(ddTestExtID, logger.GetLogger(), func(int32, int32) {}, false),
+	}
+}
+
+// keyFrame sends a key frame carrying `structure`. Passing the same structure id as
+// the previous key frame models a publisher that repeats an unchanged structure,
+// which is what screen content with frequent key frames does.
+func (f *ddTestFeeder) keyFrame(frameNumber uint16, structure *dd.FrameDependencyStructure) (*ExtDependencyDescriptor, error) {
+	f.structure = structure
+	return f.feed(frameNumber, structure.Templates[0], structure)
+}
+
+func (f *ddTestFeeder) deltaFrame(frameNumber uint16) (*ExtDependencyDescriptor, error) {
+	return f.feed(frameNumber, f.structure.Templates[1], nil)
+}
+
+func (f *ddTestFeeder) feed(
+	frameNumber uint16,
+	template *dd.FrameDependencyTemplate,
+	attachedStructure *dd.FrameDependencyStructure,
+) (*ExtDependencyDescriptor, error) {
+	f.t.Helper()
+
+	ddVal := &dd.DependencyDescriptor{
+		FirstPacketInFrame: true,
+		LastPacketInFrame:  true,
+		FrameNumber:        frameNumber,
+		FrameDependencies:  template,
+		AttachedStructure:  attachedStructure,
+	}
+	buf, err := (&dd.DependencyDescriptorExtension{Descriptor: ddVal, Structure: f.structure}).Marshal()
+	require.NoError(f.t, err)
+
+	f.seq++
+	pkt := &rtp.Packet{Header: rtp.Header{SequenceNumber: f.seq}}
+	require.NoError(f.t, pkt.SetExtension(ddTestExtID, buf))
+
+	extDD, _, err := f.parser.Parse(pkt)
+	return extDD, err
+}
+
+// A key frame that repeats the current structure must not start dropping frames that
+// precede it. Upstream advanced the drop threshold on every structure bearing key
+// frame, so with frequent key frames (screen content) a late or retransmitted frame
+// arriving after one was discarded as "earlier than current structure".
+func TestDependencyDescriptorParserLateFrameAfterRepeatedStructure(t *testing.T) {
+	f := newDDTestFeeder(t)
+	structure := newL1T1Structure(0)
+
+	_, err := f.keyFrame(0, structure)
+	require.NoError(t, err)
+	_, err = f.deltaFrame(1)
+	require.NoError(t, err)
+	_, err = f.deltaFrame(3)
+	require.NoError(t, err)
+
+	// key frame repeating the same structure id
+	_, err = f.keyFrame(4, structure)
+	require.NoError(t, err)
+
+	// frame 2 finally shows up (reordered or retransmitted). The structure has not
+	// changed, so it is still decodable and must be forwarded.
+	extDD, err := f.deltaFrame(2)
+	require.NoError(t, err)
+	require.NotNil(t, extDD)
+	require.EqualValues(t, 2, extDD.ExtFrameNum)
+}
+
+// A key frame carrying a *different* structure does invalidate everything before it:
+// earlier frames reference templates that no longer exist.
+func TestDependencyDescriptorParserLateFrameAfterStructureChange(t *testing.T) {
+	f := newDDTestFeeder(t)
+
+	_, err := f.keyFrame(0, newL1T1Structure(0))
+	require.NoError(t, err)
+	_, err = f.deltaFrame(1)
+	require.NoError(t, err)
+	_, err = f.deltaFrame(3)
+	require.NoError(t, err)
+
+	_, err = f.keyFrame(4, newL1T1Structure(1))
+	require.NoError(t, err)
+
+	_, err = f.deltaFrame(2)
+	require.ErrorIs(t, err, ErrFrameEarlierThanKeyFrame)
+}
+
+// An out-of-order key frame must still be dropped: accepting it would regress
+// structureExtFrameNum (ExtKeyFrameNum) and replay a stale structure update.
+func TestDependencyDescriptorParserOutOfOrderKeyFrame(t *testing.T) {
+	f := newDDTestFeeder(t)
+	structure := newL1T1Structure(0)
+
+	_, err := f.keyFrame(0, structure)
+	require.NoError(t, err)
+	_, err = f.deltaFrame(1)
+	require.NoError(t, err)
+	_, err = f.keyFrame(4, structure)
+	require.NoError(t, err)
+
+	_, err = f.keyFrame(2, structure)
+	require.ErrorIs(t, err, ErrFrameEarlierThanKeyFrame)
+}
+
+// ExtKeyFrameNum keeps tracking every structure bearing key frame, unchanged by the
+// split of the drop threshold into its own field.
+func TestDependencyDescriptorParserExtKeyFrameNum(t *testing.T) {
+	f := newDDTestFeeder(t)
+	structure := newL1T1Structure(0)
+
+	extDD, err := f.keyFrame(0, structure)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, extDD.ExtKeyFrameNum)
+
+	extDD, err = f.deltaFrame(1)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, extDD.ExtKeyFrameNum)
+
+	// repeated structure still advances ExtKeyFrameNum
+	extDD, err = f.keyFrame(4, structure)
+	require.NoError(t, err)
+	require.EqualValues(t, 4, extDD.ExtKeyFrameNum)
+
+	extDD, err = f.deltaFrame(5)
+	require.NoError(t, err)
+	require.EqualValues(t, 4, extDD.ExtKeyFrameNum)
+}

pkg/sfu/buffer/factory.go

@@ -0,0 +1,149 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"io"
+	"sync"
+
+	"github.com/pion/transport/v4/packetio"
+)
+
+type FactoryOfBufferFactory struct {
+	trackingPacketsVideo int
+	trackingPacketsAudio int
+}
+
+func NewFactoryOfBufferFactory(trackingPacketsVideo int, trackingPacketsAudio int) *FactoryOfBufferFactory {
+	return &FactoryOfBufferFactory{
+		trackingPacketsVideo: trackingPacketsVideo,
+		trackingPacketsAudio: trackingPacketsAudio,
+	}
+}
+
+func (f *FactoryOfBufferFactory) CreateBufferFactory() *Factory {
+	return &Factory{
+		trackingPacketsVideo: f.trackingPacketsVideo,
+		trackingPacketsAudio: f.trackingPacketsAudio,
+		rtpBuffers:           make(map[uint32]*Buffer),
+		rtcpReaders:          make(map[uint32]*RTCPReader),
+		rtxPair:              make(map[uint32]uint32),
+	}
+}
+
+type Factory struct {
+	sync.RWMutex
+	trackingPacketsVideo int
+	trackingPacketsAudio int
+	rtpBuffers           map[uint32]*Buffer
+	rtcpReaders          map[uint32]*RTCPReader
+	rtxPair              map[uint32]uint32 // repair -> base
+}
+
+func (f *Factory) GetOrNew(packetType packetio.BufferPacketType, ssrc uint32) io.ReadWriteCloser {
+	f.Lock()
+	defer f.Unlock()
+	switch packetType {
+	case packetio.RTCPBufferPacket:
+		if reader, ok := f.rtcpReaders[ssrc]; ok {
+			return reader
+		}
+		reader := NewRTCPReader(ssrc)
+		f.rtcpReaders[ssrc] = reader
+		reader.OnClose(func() {
+			f.Lock()
+			delete(f.rtcpReaders, ssrc)
+			f.Unlock()
+		})
+		return reader
+	case packetio.RTPBufferPacket:
+		if reader, ok := f.rtpBuffers[ssrc]; ok {
+			return reader
+		}
+		buffer := NewBuffer(ssrc, f.trackingPacketsVideo, f.trackingPacketsAudio)
+		f.rtpBuffers[ssrc] = buffer
+		for repair, base := range f.rtxPair {
+			if repair == ssrc {
+				baseBuffer, ok := f.rtpBuffers[base]
+				if ok {
+					buffer.SetPrimaryBufferForRTX(baseBuffer)
+				}
+				break
+			} else if base == ssrc {
+				repairBuffer, ok := f.rtpBuffers[repair]
+				if ok {
+					repairBuffer.SetPrimaryBufferForRTX(buffer)
+				}
+				break
+			}
+		}
+		buffer.OnClose(func() {
+			f.Lock()
+			delete(f.rtpBuffers, ssrc)
+			delete(f.rtxPair, ssrc)
+			f.Unlock()
+		})
+		return buffer
+	}
+	return nil
+}
+
+func (f *Factory) GetBufferPair(ssrc uint32) (*Buffer, *RTCPReader) {
+	f.RLock()
+	defer f.RUnlock()
+	return f.rtpBuffers[ssrc], f.rtcpReaders[ssrc]
+}
+
+func (f *Factory) GetBuffer(ssrc uint32) *Buffer {
+	f.RLock()
+	defer f.RUnlock()
+	return f.rtpBuffers[ssrc]
+}
+
+func (f *Factory) GetRTCPReader(ssrc uint32) *RTCPReader {
+	f.RLock()
+	defer f.RUnlock()
+	return f.rtcpReaders[ssrc]
+}
+
+// SetStreamInfoProbe installs probe on the buffer of ssrc, reporting whether that
+// buffer exists. False means the stream can never be identified.
+func (f *Factory) SetStreamInfoProbe(ssrc uint32, probe *StreamInfoProbe) bool {
+	f.RLock()
+	buffer := f.rtpBuffers[ssrc]
+	f.RUnlock()
+
+	if buffer == nil {
+		return false
+	}
+
+	buffer.SetStreamInfoProbe(probe)
+	return true
+}
+
+func (f *Factory) SetRTXPair(repair, base uint32, rsid string) {
+	f.Lock()
+	repairBuffer, baseBuffer := f.rtpBuffers[repair], f.rtpBuffers[base]
+	if repairBuffer == nil || baseBuffer == nil {
+		f.rtxPair[repair] = base
+	}
+	f.Unlock()
+	if repairBuffer != nil && baseBuffer != nil {
+		repairBuffer.SetPrimaryBufferForRTX(baseBuffer)
+		if rsid != "" {
+			baseBuffer.NotifyRTX(base, repair, rsid)
+		}
+	}
+}

pkg/sfu/buffer/fps.go

@@ -0,0 +1,750 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"container/list"
+
+	"github.com/pion/rtp/codecs"
+
+	"github.com/livekit/mediatransportutil/pkg/codec"
+	"github.com/livekit/protocol/logger"
+)
+
+var minFramesForCalculation = [...]int{8, 15, 40, 60}
+
+type frameInfo struct {
+	startSeq  uint16
+	endSeq    uint16
+	ts        uint32
+	fn        uint16
+	spatial   int32
+	temporal  int32
+	frameDiff []int
+}
+
+type FrameRateCalculator interface {
+	RecvPacket(ep *ExtPacket) bool
+	GetFrameRate() []float32
+	Completed() bool
+}
+
+// -----------------------------
+
+// FrameRateCalculator based on PictureID in VPx
+type frameRateCalculatorVPx struct {
+	frameRates   [DefaultMaxLayerTemporal + 1]float32
+	clockRate    uint32
+	logger       logger.Logger
+	firstFrames  [DefaultMaxLayerTemporal + 1]*frameInfo
+	secondFrames [DefaultMaxLayerTemporal + 1]*frameInfo
+	fnReceived   [64]*frameInfo
+	baseFrame    *frameInfo
+	completed    bool
+}
+
+func newFrameRateCalculatorVPx(clockRate uint32, logger logger.Logger) *frameRateCalculatorVPx {
+	return &frameRateCalculatorVPx{
+		clockRate: clockRate,
+		logger:    logger,
+	}
+}
+
+func (f *frameRateCalculatorVPx) Completed() bool {
+	return f.completed
+}
+
+func (f *frameRateCalculatorVPx) RecvPacket(ep *ExtPacket, fn uint16) bool {
+	if f.completed {
+		return true
+	}
+
+	if ep.Temporal >= int32(len(f.frameRates)) {
+		f.logger.Warnw("invalid temporal layer", nil, "temporal", ep.Temporal)
+		return false
+	}
+
+	temporal := ep.Temporal
+	if temporal < 0 {
+		temporal = 0
+	}
+
+	if f.baseFrame == nil {
+		f.baseFrame = &frameInfo{ts: ep.Packet.Timestamp, fn: fn}
+		f.fnReceived[0] = f.baseFrame
+		f.firstFrames[temporal] = f.baseFrame
+		return false
+	}
+
+	baseDiff := fn - f.baseFrame.fn
+	if baseDiff == 0 || baseDiff > 0x4000 {
+		return false
+	}
+
+	if baseDiff >= uint16(len(f.fnReceived)) {
+		// frame number is not continuous, reset
+		f.reset()
+
+		return false
+	}
+
+	if f.fnReceived[baseDiff] != nil {
+		return false
+	}
+
+	fi := &frameInfo{
+		ts:       ep.Packet.Timestamp,
+		fn:       fn,
+		temporal: temporal,
+	}
+	f.fnReceived[baseDiff] = fi
+
+	firstFrame := f.firstFrames[temporal]
+	secondFrame := f.secondFrames[temporal]
+	if firstFrame == nil {
+		f.firstFrames[temporal] = fi
+		firstFrame = fi
+	} else {
+		if (secondFrame == nil || secondFrame.fn < fn) && fn != firstFrame.fn && (fn-firstFrame.fn) < 0x4000 {
+			f.secondFrames[temporal] = fi
+		}
+	}
+
+	return f.calc()
+}
+
+func (f *frameRateCalculatorVPx) calc() bool {
+	var rateCounter int
+	for currentTemporal := int32(0); currentTemporal <= DefaultMaxLayerTemporal; currentTemporal++ {
+		if f.frameRates[currentTemporal] > 0 {
+			rateCounter++
+			continue
+		}
+
+		ff := f.firstFrames[currentTemporal]
+		sf := f.secondFrames[currentTemporal]
+		// lower temporal layer has been calculated, but higher layer has not received any frames, it should not exist
+		if rateCounter > 0 && ff == nil {
+			rateCounter++
+			continue
+		}
+		if ff == nil || sf == nil {
+			continue
+		}
+
+		var frameCount int
+		lastTs := ff.ts
+		for j := ff.fn - f.baseFrame.fn + 1; j < sf.fn-f.baseFrame.fn+1; j++ {
+			if f := f.fnReceived[j]; f == nil {
+				break
+			} else if f.temporal <= currentTemporal {
+				frameCount++
+				lastTs = f.ts
+			}
+		}
+		if frameCount >= minFramesForCalculation[currentTemporal] {
+			f.frameRates[currentTemporal] = float32(f.clockRate) / float32(lastTs-ff.ts) * float32(frameCount)
+			rateCounter++
+		}
+	}
+
+	if rateCounter == len(f.frameRates) {
+		f.completed = true
+
+		// normalize frame rates, Microsoft Edge use 3 temporal layers for vp8 but the middle layer has chance to
+		// get a very low frame rate, so we need to normalize the frame rate(use fixed ration 1:2 of highest layer for that layer)
+		if f.frameRates[2] > 0 && f.frameRates[2] > f.frameRates[1]*3 {
+			f.frameRates[1] = f.frameRates[2] / 2
+		}
+		f.reset()
+		return true
+	}
+	return false
+}
+
+func (f *frameRateCalculatorVPx) reset() {
+	for i := range f.firstFrames {
+		f.firstFrames[i] = nil
+		f.secondFrames[i] = nil
+	}
+
+	for i := range f.fnReceived {
+		f.fnReceived[i] = nil
+	}
+	f.baseFrame = nil
+}
+
+func (f *frameRateCalculatorVPx) GetFrameRate() []float32 {
+	return f.frameRates[:]
+}
+
+// -----------------------------
+
+// FrameRateCalculator based on PictureID in VP8
+type FrameRateCalculatorVP8 struct {
+	*frameRateCalculatorVPx
+	logger logger.Logger
+}
+
+func NewFrameRateCalculatorVP8(clockRate uint32, logger logger.Logger) *FrameRateCalculatorVP8 {
+	return &FrameRateCalculatorVP8{
+		frameRateCalculatorVPx: newFrameRateCalculatorVPx(clockRate, logger),
+		logger:                 logger,
+	}
+}
+
+func (f *FrameRateCalculatorVP8) RecvPacket(ep *ExtPacket) bool {
+	if f.frameRateCalculatorVPx.Completed() {
+		return true
+	}
+
+	vp8, ok := ep.Payload.(codec.VP8)
+	if !ok {
+		f.logger.Debugw("no vp8 payload", "sn", ep.Packet.SequenceNumber)
+		return false
+	}
+	if !vp8.I {
+		f.logger.Debugw("no vp8 pictureID", "sn", ep.Packet.SequenceNumber)
+		return false
+	}
+	success := f.frameRateCalculatorVPx.RecvPacket(ep, vp8.PictureID)
+
+	if f.frameRateCalculatorVPx.Completed() {
+		f.logger.Debugw("frame rate calculated", "rate", f.frameRateCalculatorVPx.GetFrameRate())
+	}
+
+	return success
+}
+
+// -----------------------------
+
+// FrameRateCalculator based on PictureID in VP9
+type FrameRateCalculatorVP9 struct {
+	logger    logger.Logger
+	completed bool
+
+	// VP9-TODO - this is assuming three spatial layers. As `completed` marker relies on all layers being finished, have to assume this. FIX.
+	//            Maybe look at number of layers in livekit.TrackInfo and declare completed once advertised layers are measured
+	frameRateCalculatorsVPx [DefaultMaxLayerSpatial + 1]*frameRateCalculatorVPx
+}
+
+func NewFrameRateCalculatorVP9(clockRate uint32, logger logger.Logger) *FrameRateCalculatorVP9 {
+	f := &FrameRateCalculatorVP9{
+		logger: logger,
+	}
+
+	for i := range f.frameRateCalculatorsVPx {
+		f.frameRateCalculatorsVPx[i] = newFrameRateCalculatorVPx(clockRate, logger)
+	}
+
+	return f
+}
+
+func (f *FrameRateCalculatorVP9) Completed() bool {
+	return f.completed
+}
+
+func (f *FrameRateCalculatorVP9) RecvPacket(ep *ExtPacket) bool {
+	if f.completed {
+		return true
+	}
+
+	vp9, ok := ep.Payload.(codecs.VP9Packet)
+	if !ok {
+		f.logger.Debugw("no vp9 payload", "sn", ep.Packet.SequenceNumber)
+		return false
+	}
+	if !vp9.I {
+		f.logger.Debugw("no vp9 pictureID", "sn", ep.Packet.SequenceNumber)
+		return false
+	}
+	if ep.Spatial < 0 || ep.Spatial >= int32(len(f.frameRateCalculatorsVPx)) || f.frameRateCalculatorsVPx[ep.Spatial] == nil {
+		f.logger.Debugw("invalid spatial layer", "sn", ep.Packet.SequenceNumber, "spatial", ep.Spatial)
+		return false
+	}
+
+	success := f.frameRateCalculatorsVPx[ep.Spatial].RecvPacket(ep, vp9.PictureID)
+
+	completed := true
+	for _, frc := range f.frameRateCalculatorsVPx {
+		if !frc.Completed() {
+			completed = false
+			break
+		}
+	}
+
+	if completed {
+		f.completed = true
+
+		var frameRates [DefaultMaxLayerSpatial + 1][]float32
+		for i := range f.frameRateCalculatorsVPx {
+			frameRates[i] = f.frameRateCalculatorsVPx[i].GetFrameRate()
+		}
+		f.logger.Debugw("frame rate calculated", "rate", frameRates)
+	}
+
+	return success
+}
+
+func (f *FrameRateCalculatorVP9) GetFrameRateForSpatial(spatial int32) []float32 {
+	if spatial < 0 || spatial >= int32(len(f.frameRateCalculatorsVPx)) || f.frameRateCalculatorsVPx[spatial] == nil {
+		return nil
+	}
+	return f.frameRateCalculatorsVPx[spatial].GetFrameRate()
+}
+
+func (f *FrameRateCalculatorVP9) GetFrameRateCalculatorForSpatial(spatial int32) *FrameRateCalculatorForVP9Layer {
+	return &FrameRateCalculatorForVP9Layer{
+		FrameRateCalculatorVP9: f,
+		spatial:                spatial,
+	}
+}
+
+// -----------------------------
+
+type FrameRateCalculatorForVP9Layer struct {
+	*FrameRateCalculatorVP9
+	spatial int32
+}
+
+func (f *FrameRateCalculatorForVP9Layer) GetFrameRate() []float32 {
+	return f.FrameRateCalculatorVP9.GetFrameRateForSpatial(f.spatial)
+}
+
+// -----------------------------------------------
+
+// FrameRateCalculator based on Dependency descriptor
+type FrameRateCalculatorDD struct {
+	frameRates   [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]float32
+	clockRate    uint32
+	logger       logger.Logger
+	firstFrames  [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]*frameInfo
+	secondFrames [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]*frameInfo
+	fnReceived   [256]*frameInfo
+	baseFrame    *frameInfo
+	completed    bool
+
+	// frames for each decode target
+	targetFrames [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]list.List
+
+	maxSpatial, maxTemporal int32
+}
+
+func NewFrameRateCalculatorDD(clockRate uint32, logger logger.Logger) *FrameRateCalculatorDD {
+	return &FrameRateCalculatorDD{
+		clockRate:   clockRate,
+		logger:      logger,
+		maxSpatial:  DefaultMaxLayerSpatial,
+		maxTemporal: DefaultMaxLayerTemporal,
+	}
+}
+
+func (f *FrameRateCalculatorDD) Completed() bool {
+	return f.completed
+}
+
+func (f *FrameRateCalculatorDD) SetMaxLayer(spatial, temporal int32) {
+	if spatial <= DefaultMaxLayerSpatial {
+		f.maxSpatial = spatial
+	} else {
+		f.logger.Warnw("max spatial layer out of range", nil, "spatial", spatial)
+	}
+
+	if temporal <= DefaultMaxLayerTemporal {
+		f.maxTemporal = temporal
+	} else {
+		f.logger.Warnw("max temporal layer out of range", nil, "temporal", temporal)
+	}
+}
+
+func (f *FrameRateCalculatorDD) RecvPacket(ep *ExtPacket) bool {
+	if f.completed {
+		return true
+	}
+
+	if ep.DependencyDescriptor == nil {
+		f.logger.Debugw("dependency descriptor is nil")
+		return false
+	}
+
+	spatial := ep.Spatial
+	// non-SVC codec will set spatial to -1
+	if spatial < 0 {
+		spatial = 0
+	}
+	temporal := ep.Temporal
+	if temporal < 0 || temporal > DefaultMaxLayerTemporal || spatial > DefaultMaxLayerSpatial {
+		f.logger.Warnw("invalid spatial or temporal", nil, "spatial", spatial, "temporal", temporal, "sn", ep.Packet.SequenceNumber)
+		return false
+	}
+
+	fn := ep.DependencyDescriptor.Descriptor.FrameNumber
+	if f.baseFrame == nil {
+		f.baseFrame = &frameInfo{ts: ep.Packet.Timestamp, fn: fn}
+		f.fnReceived[0] = f.baseFrame
+		f.firstFrames[spatial][temporal] = f.baseFrame
+		f.secondFrames[spatial][temporal] = f.baseFrame
+		return false
+	}
+
+	baseDiff := fn - f.baseFrame.fn
+	if baseDiff == 0 || baseDiff > 0x8000 {
+		return false
+	}
+
+	if baseDiff >= uint16(len(f.fnReceived)) {
+		// frame number is not continuous, reset
+		f.baseFrame = nil
+		for i := range f.firstFrames {
+			for j := range f.firstFrames[i] {
+				f.firstFrames[i][j] = nil
+				f.secondFrames[i][j] = nil
+				f.targetFrames[i][j].Init()
+			}
+		}
+		for i := range f.fnReceived {
+			f.fnReceived[i] = nil
+		}
+		return false
+	}
+
+	if f.fnReceived[baseDiff] != nil {
+		return false
+	}
+
+	fi := &frameInfo{
+		ts:        ep.Packet.Timestamp,
+		fn:        fn,
+		temporal:  temporal,
+		spatial:   spatial,
+		frameDiff: ep.DependencyDescriptor.Descriptor.FrameDependencies.FrameDiffs,
+	}
+	f.fnReceived[baseDiff] = fi
+
+	if f.firstFrames[spatial][temporal] == nil {
+		f.firstFrames[spatial][temporal] = fi
+		f.secondFrames[spatial][temporal] = fi
+		return false
+	}
+
+	chain := &f.targetFrames[spatial][temporal]
+	if chain.Len() == 0 {
+		chain.PushBack(fn)
+	}
+	for _, fdiff := range ep.DependencyDescriptor.Descriptor.FrameDependencies.FrameDiffs {
+		dependFrame := fn - uint16(fdiff)
+		// frame too old, ignore
+		if dependFrame-f.secondFrames[spatial][temporal].fn > 0x8000 {
+			continue
+		}
+
+	insertFrame:
+		for e := chain.Back(); e != nil; e = e.Prev() {
+			val := e.Value.(uint16)
+			switch {
+			case val == dependFrame:
+				break insertFrame
+			case sn16LT(val, dependFrame):
+				chain.InsertAfter(dependFrame, e)
+				break insertFrame
+			default:
+				if e == chain.Front() {
+					chain.PushFront(dependFrame)
+					break insertFrame
+				}
+			}
+		}
+	}
+	return f.calc()
+}
+
+func (f *FrameRateCalculatorDD) calc() bool {
+	var rateCounter int
+	for currentSpatial := int32(0); currentSpatial <= f.maxSpatial; currentSpatial++ {
+		var currentSpatialRateCounter int
+		for currentTemporal := int32(0); currentTemporal <= f.maxTemporal; currentTemporal++ {
+			if f.frameRates[currentSpatial][currentTemporal] > 0 {
+				rateCounter++
+				currentSpatialRateCounter++
+				continue
+			}
+
+			firstFrame := f.firstFrames[currentSpatial][currentTemporal]
+			// lower temporal layer has been calculated, but higher layer has not received any frames, it should not exist
+			if currentSpatialRateCounter > 0 && firstFrame == nil {
+				currentSpatialRateCounter++
+				rateCounter++
+				continue
+			}
+
+			chain := &f.targetFrames[currentSpatial][currentTemporal]
+
+			// find last decodable frame (no dependency frame is lost)
+			var lastFrame *frameInfo
+			for e := chain.Front(); e != nil; e = e.Next() {
+				diff := e.Value.(uint16) - f.baseFrame.fn
+				if diff >= uint16(len(f.fnReceived)) {
+					continue
+				}
+
+				fi := f.fnReceived[diff]
+				if fi == nil {
+					break
+				} else {
+					lastFrame = fi
+					if firstFrame == nil && fi.spatial == currentSpatial && fi.temporal == currentTemporal {
+						firstFrame = fi
+					}
+				}
+			}
+
+			if lastFrame != nil && lastFrame.fn > f.secondFrames[currentSpatial][currentTemporal].fn {
+				f.secondFrames[currentSpatial][currentTemporal] = lastFrame
+			} else {
+				continue
+			}
+
+			frameCount := 0
+			for i := firstFrame.fn - f.baseFrame.fn; i <= lastFrame.fn-f.baseFrame.fn; i++ {
+				fi := f.fnReceived[i]
+				if fi == nil {
+					continue
+				}
+				if fi.spatial == currentSpatial && fi.temporal <= currentTemporal {
+					frameCount++
+				}
+			}
+
+			if frameCount >= minFramesForCalculation[currentTemporal] && lastFrame.ts > firstFrame.ts {
+				f.frameRates[currentSpatial][currentTemporal] = float32(f.clockRate) / float32(lastFrame.ts-firstFrame.ts) * float32(frameCount)
+				rateCounter++
+			}
+		}
+	}
+
+	if rateCounter == int(f.maxSpatial+1)*int(f.maxTemporal+1) {
+		f.completed = true
+		f.close()
+
+		f.logger.Debugw("frame rate calculated", "rate", f.frameRates)
+		return true
+	}
+	return false
+}
+
+func (f *FrameRateCalculatorDD) GetFrameRateForSpatial(spatial int32) []float32 {
+	if spatial < 0 || spatial >= int32(len(f.frameRates)) {
+		return nil
+	}
+	return f.frameRates[spatial][:]
+}
+
+func (f *FrameRateCalculatorDD) close() {
+	f.baseFrame = nil
+	for i := range f.firstFrames {
+		for j := range f.firstFrames[i] {
+			f.firstFrames[i][j] = nil
+			f.secondFrames[i][j] = nil
+		}
+	}
+
+	for i := range f.fnReceived {
+		f.fnReceived[i] = nil
+	}
+	for i := range f.targetFrames {
+		for j := range f.targetFrames[i] {
+			f.targetFrames[i][j].Init()
+		}
+	}
+}
+
+func (f *FrameRateCalculatorDD) GetFrameRateCalculatorForSpatial(spatial int32) *FrameRateCalculatorForDDLayer {
+	return &FrameRateCalculatorForDDLayer{
+		FrameRateCalculatorDD: f,
+		spatial:               spatial,
+	}
+}
+
+// -----------------------------------------------
+
+type FrameRateCalculatorForDDLayer struct {
+	*FrameRateCalculatorDD
+	spatial int32
+}
+
+func (f *FrameRateCalculatorForDDLayer) GetFrameRate() []float32 {
+	return f.FrameRateCalculatorDD.GetFrameRateForSpatial(f.spatial)
+}
+
+// -----------------------------------------------
+
+type FrameRateCalculatorH26x struct {
+	frameRates [DefaultMaxLayerTemporal + 1]float32
+	clockRate  uint32
+	logger     logger.Logger
+	fnReceived *list.List
+	baseFrame  *frameInfo
+	completed  bool
+}
+
+func NewFrameRateCalculatorH26x(clockRate uint32, logger logger.Logger) *FrameRateCalculatorH26x {
+	return &FrameRateCalculatorH26x{
+		clockRate: clockRate,
+		logger:    logger,
+	}
+}
+
+func (f *FrameRateCalculatorH26x) Completed() bool {
+	return f.completed
+}
+
+func (f *FrameRateCalculatorH26x) RecvPacket(ep *ExtPacket) bool {
+	if f.completed {
+		return true
+	}
+
+	if ep.Temporal >= int32(len(f.frameRates)) {
+		f.logger.Warnw("invalid temporal layer", nil, "temporal", ep.Temporal)
+		return false
+	}
+
+	temporal := ep.Temporal
+	if temporal < 0 {
+		temporal = 0
+	}
+
+	if f.baseFrame == nil {
+		f.baseFrame = &frameInfo{
+			startSeq: ep.Packet.SequenceNumber,
+			endSeq:   ep.Packet.SequenceNumber,
+			ts:       ep.Packet.Timestamp,
+			temporal: temporal,
+		}
+		f.fnReceived = list.New()
+		f.fnReceived.PushBack(f.baseFrame)
+		return false
+	}
+
+	if sn16LTOrEqual(ep.Packet.SequenceNumber, f.baseFrame.startSeq) {
+		return false
+	}
+
+insertFrame:
+	for e := f.fnReceived.Back(); e != nil; e = e.Prev() {
+		frame := e.Value.(*frameInfo)
+		switch {
+		case frame.ts == ep.Packet.Timestamp:
+			if sn16LT(frame.endSeq, ep.Packet.SequenceNumber) {
+				frame.endSeq = ep.Packet.SequenceNumber
+			}
+			if sn16LT(ep.Packet.SequenceNumber, frame.startSeq) {
+				frame.startSeq = ep.Packet.SequenceNumber
+			}
+			break insertFrame
+		case sn32LT(frame.ts, ep.Packet.Timestamp):
+			f.fnReceived.InsertAfter(&frameInfo{
+				startSeq: ep.Packet.SequenceNumber,
+				endSeq:   ep.Packet.SequenceNumber,
+				ts:       ep.Packet.Timestamp,
+				temporal: temporal,
+			}, e)
+			break insertFrame
+		default:
+			if e == f.fnReceived.Front() {
+				f.fnReceived.PushFront(&frameInfo{
+					startSeq: ep.Packet.SequenceNumber,
+					endSeq:   ep.Packet.SequenceNumber,
+					ts:       ep.Packet.Timestamp,
+					temporal: temporal,
+				})
+				break insertFrame
+			}
+		}
+	}
+
+	return f.calc()
+}
+
+func (f *FrameRateCalculatorH26x) calc() bool {
+	frameCounts := make([]int, DefaultMaxLayerTemporal+1)
+	var totalFrameCount int
+	var tsDuration int
+	cur := f.fnReceived.Front()
+	for {
+		next := cur.Next()
+		if next == nil {
+			break
+		}
+		ff := cur.Value.(*frameInfo)
+		nf := next.Value.(*frameInfo)
+		if nf.startSeq-ff.endSeq == 1 {
+			totalFrameCount++
+			tsDuration += int(nf.ts - ff.ts)
+			for i := int(nf.temporal); i < len(frameCounts); i++ {
+				frameCounts[i]++
+			}
+		} else {
+			// reset to find continuous frames
+			totalFrameCount = 0
+			for i := range frameCounts {
+				frameCounts[i] = 0
+			}
+			tsDuration = 0
+		}
+
+		// received enough continuous frames, calculate fps
+		if totalFrameCount >= minFramesForCalculation[DefaultMaxLayerTemporal] {
+			for currentTemporal := int32(0); currentTemporal <= DefaultMaxLayerTemporal; currentTemporal++ {
+				count := frameCounts[currentTemporal]
+				if currentTemporal > 0 && count == frameCounts[currentTemporal-1] {
+					// no frames for this temporal layer
+					f.frameRates[currentTemporal] = 0
+				} else {
+					f.frameRates[currentTemporal] = float32(f.clockRate) / float32(tsDuration) * float32(count)
+				}
+			}
+			f.logger.Debugw("fps changed", "fps", f.GetFrameRate())
+			f.completed = true
+			f.reset()
+			return true
+		}
+
+		cur = next
+	}
+
+	return false
+}
+
+func (f *FrameRateCalculatorH26x) reset() {
+	f.fnReceived.Init()
+	f.baseFrame = nil
+}
+
+func (f *FrameRateCalculatorH26x) GetFrameRate() []float32 {
+	return f.frameRates[:]
+}
+
+// -----------------------------------------------
+func sn16LT(a, b uint16) bool {
+	return a-b > 0x8000
+}
+
+func sn16LTOrEqual(a, b uint16) bool {
+	return a == b || a-b > 0x8000
+}
+
+func sn32LT(a, b uint32) bool {
+	return a-b > 0x80000000
+}

pkg/sfu/buffer/fps_test.go

@@ -0,0 +1,439 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"testing"
+
+	"github.com/pion/rtp"
+	"github.com/stretchr/testify/require"
+
+	dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
+	"github.com/livekit/mediatransportutil/pkg/codec"
+	"github.com/livekit/protocol/logger"
+)
+
+type testFrameInfo struct {
+	header      rtp.Header
+	framenumber uint16
+	spatial     int
+	temporal    int
+	frameDiff   []int
+}
+
+func (f *testFrameInfo) toVP8() *ExtPacket {
+	return &ExtPacket{
+		Packet: &rtp.Packet{Header: f.header},
+		Payload: codec.VP8{
+			I:         true,
+			PictureID: f.framenumber,
+		},
+		VideoLayer: VideoLayer{Spatial: InvalidLayerSpatial, Temporal: int32(f.temporal)},
+	}
+}
+
+func (f *testFrameInfo) toDD() *ExtPacket {
+	return &ExtPacket{
+		Packet: &rtp.Packet{Header: f.header},
+		DependencyDescriptor: &ExtDependencyDescriptor{
+			Descriptor: &dd.DependencyDescriptor{
+				FrameNumber: f.framenumber,
+				FrameDependencies: &dd.FrameDependencyTemplate{
+					FrameDiffs: f.frameDiff,
+				},
+			},
+		},
+		VideoLayer: VideoLayer{Spatial: int32(f.spatial), Temporal: int32(f.temporal)},
+	}
+}
+
+func (f *testFrameInfo) toH26x() *ExtPacket {
+	return &ExtPacket{
+		Packet:     &rtp.Packet{Header: f.header},
+		VideoLayer: VideoLayer{Spatial: InvalidLayerSpatial, Temporal: int32(f.temporal)},
+	}
+}
+
+func createFrames(startFrameNumber uint16, startTs uint32, startSeq uint16, totalFramesPerSpatial int, fps [][]float32, spatialDependency bool) [][]*testFrameInfo {
+	spatials := len(fps)
+	temporals := len(fps[0])
+	frames := make([][]*testFrameInfo, spatials)
+	for s := 0; s < spatials; s++ {
+		frames[s] = make([]*testFrameInfo, 0, totalFramesPerSpatial)
+	}
+	fn := startFrameNumber
+
+	nextTs := make([][]uint32, spatials)
+	tsStep := make([][]uint32, spatials)
+	for i := range spatials {
+		nextTs[i] = make([]uint32, temporals)
+		tsStep[i] = make([]uint32, temporals)
+		for j := 0; j < temporals; j++ {
+			nextTs[i][j] = startTs
+			tsStep[i][j] = uint32(90000 / fps[i][j])
+		}
+	}
+
+	currentTs := make([]uint32, spatials)
+	for i := range spatials {
+		currentTs[i] = startTs
+	}
+	for range totalFramesPerSpatial {
+		for s := range spatials {
+			frame := &testFrameInfo{
+				header:      rtp.Header{Timestamp: currentTs[s], SequenceNumber: startSeq},
+				framenumber: fn,
+				spatial:     s,
+			}
+			for t := 0; t < temporals; t++ {
+				if currentTs[s] >= nextTs[s][t] {
+					frame.temporal = t
+					for nt := t; nt < temporals; nt++ {
+						nextTs[s][nt] += tsStep[s][nt]
+					}
+					break
+				}
+			}
+			currentTs[s] += tsStep[s][temporals-1]
+			frames[s] = append(frames[s], frame)
+			fn++
+			startSeq++
+
+			for fidx := len(frames[s]) - 1; fidx >= 0; fidx-- {
+				cf := frames[s][fidx]
+				if cf.header.Timestamp-frame.header.Timestamp > 0x80000000 {
+					frame.frameDiff = append(frame.frameDiff, int(frame.framenumber-cf.framenumber))
+					break
+				}
+			}
+
+			if spatialDependency && frame.spatial > 0 {
+				for fidx := len(frames[frame.spatial-1]) - 1; fidx >= 0; fidx-- {
+					cf := frames[frame.spatial-1][fidx]
+					if cf.header.Timestamp == frame.header.Timestamp {
+						frame.frameDiff = append(frame.frameDiff, int(frame.framenumber-cf.framenumber))
+						break
+					}
+				}
+			}
+		}
+	}
+
+	return frames
+}
+
+func verifyFps(t *testing.T, expect, got []float32) {
+	require.Equal(t, len(expect), len(got))
+	for i := range expect {
+		require.GreaterOrEqual(t, got[i], expect[i]*0.9, "expect %v, got %v", expect, got)
+		require.LessOrEqual(t, got[i], expect[i]*1.1, "expect %v, got %v", expect, got)
+	}
+}
+
+type testcase struct {
+	startTs           uint32
+	startSeq          uint16
+	startFrameNumber  uint16
+	fps               [][]float32
+	spatialDependency bool
+}
+
+func TestFpsVP8(t *testing.T) {
+	cases := map[string]testcase{
+		"normal": {
+			startTs:          12345678,
+			startFrameNumber: 100,
+			fps:              [][]float32{{5, 10, 15}, {5, 10, 15}, {7.5, 15, 30}},
+		},
+		"frame number and timestamp wrap": {
+			startTs:          (uint32(1) << 31) - 10,
+			startFrameNumber: (uint16(1) << 15) - 10,
+			fps:              [][]float32{{5, 10, 15}, {5, 10, 15}, {7.5, 15, 30}},
+		},
+		"2 temporal layers": {
+			startTs:          12345678,
+			startFrameNumber: 100,
+			fps:              [][]float32{{7.5, 15}, {7.5, 15}, {15, 30}},
+		},
+	}
+
+	for name, c := range cases {
+		testCase := c
+		t.Run(name, func(t *testing.T) {
+			fps := testCase.fps
+			frames := make([][]*testFrameInfo, 0)
+			vp8calcs := make([]*FrameRateCalculatorVP8, len(fps))
+			for i := range vp8calcs {
+				vp8calcs[i] = NewFrameRateCalculatorVP8(90000, logger.GetLogger())
+				frames = append(frames, createFrames(c.startFrameNumber, c.startTs, 10, 200, [][]float32{fps[i]}, false)[0])
+			}
+
+			var frameratesGot bool
+			for s, fs := range frames {
+				for _, f := range fs {
+					if vp8calcs[s].RecvPacket(f.toVP8()) {
+						frameratesGot = true
+						for _, calc := range vp8calcs {
+							if !calc.Completed() {
+								frameratesGot = false
+								break
+							}
+						}
+					}
+				}
+			}
+			require.True(t, frameratesGot)
+			for i, calc := range vp8calcs {
+				fpsExpected := fps[i]
+				fpsGot := calc.GetFrameRate()
+				verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
+			}
+		})
+	}
+	t.Run("packet lost and duplicate", func(t *testing.T) {
+		fps := [][]float32{{7.5, 15}, {7.5, 15}, {15, 30}}
+		frames := make([][]*testFrameInfo, 0)
+		vp8calcs := make([]*FrameRateCalculatorVP8, len(fps))
+		for i := range vp8calcs {
+			vp8calcs[i] = NewFrameRateCalculatorVP8(90000, logger.GetLogger())
+			frames = append(frames, createFrames(100, 12345678, 10, 300, [][]float32{fps[i]}, false)[0])
+			for j := 5; j < 130; j++ {
+				if j%2 == 0 {
+					frames[i][j] = frames[i][j-1]
+				}
+			}
+		}
+
+		var frameratesGot bool
+		for s, fs := range frames {
+			for _, f := range fs {
+				if vp8calcs[s].RecvPacket(f.toVP8()) {
+					frameratesGot = true
+					for _, calc := range vp8calcs {
+						if !calc.Completed() {
+							frameratesGot = false
+							break
+						}
+					}
+				}
+			}
+		}
+		require.True(t, frameratesGot)
+		for i, calc := range vp8calcs {
+			fpsExpected := fps[i]
+			fpsGot := calc.GetFrameRate()
+			verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
+		}
+	})
+}
+
+func TestFpsDD(t *testing.T) {
+	cases := map[string]testcase{
+		"normal": {
+			startTs:           12345678,
+			startFrameNumber:  100,
+			fps:               [][]float32{{5.1, 10.1, 16}, {5.1, 10.1, 16}, {8, 15, 30.1}},
+			spatialDependency: true,
+		},
+		"frame number and timestamp wrap": {
+			startTs:           (uint32(1) << 31) - 10,
+			startFrameNumber:  (uint16(1) << 15) - 10,
+			fps:               [][]float32{{7.5, 15, 30}, {7.5, 15, 30}, {7.5, 15, 30}},
+			spatialDependency: true,
+		},
+		"vp8": {
+			startTs:           12345678,
+			startFrameNumber:  100,
+			fps:               [][]float32{{7.5, 15}, {7.5, 15}, {15, 30}},
+			spatialDependency: false,
+		},
+	}
+
+	for name, c := range cases {
+		testCase := c
+		t.Run(name, func(t *testing.T) {
+			fps := testCase.fps
+			frames := createFrames(c.startFrameNumber, c.startTs, 10, 500, fps, testCase.spatialDependency)
+			ddcalc := NewFrameRateCalculatorDD(90000, logger.GetLogger())
+			ddcalc.SetMaxLayer(int32(len(fps)-1), int32(len(fps[0])-1))
+			ddcalcs := make([]FrameRateCalculator, len(fps))
+			for i := range fps {
+				ddcalcs[i] = ddcalc.GetFrameRateCalculatorForSpatial(int32(i))
+			}
+
+			var frameratesGot bool
+			for s, fs := range frames {
+				for _, f := range fs {
+					if ddcalcs[s].RecvPacket(f.toDD()) {
+						frameratesGot = true
+						for _, calc := range ddcalcs {
+							if !calc.Completed() {
+								frameratesGot = false
+								break
+							}
+						}
+					}
+				}
+			}
+			require.True(t, frameratesGot)
+			for i, calc := range ddcalcs {
+				fpsExpected := fps[i]
+				fpsGot := calc.GetFrameRate()
+				verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
+			}
+		})
+	}
+
+	t.Run("packet lost and duplicate", func(t *testing.T) {
+		fps := [][]float32{{7.5, 15, 30}, {7.5, 15, 30}, {7.5, 15, 30}}
+		frames := createFrames(100, 12345678, 10, 500, fps, true)
+		ddcalc := NewFrameRateCalculatorDD(90000, logger.GetLogger())
+		ddcalc.SetMaxLayer(int32(len(fps)-1), int32(len(fps[0])-1))
+		ddcalcs := make([]FrameRateCalculator, len(fps))
+		for i := range fps {
+			ddcalcs[i] = ddcalc.GetFrameRateCalculatorForSpatial(int32(i))
+			for j := 5; j < 130; j++ {
+				if j%2 == 0 {
+					frames[i][j] = frames[i][j-1]
+				}
+			}
+		}
+
+		var frameratesGot bool
+		for s, fs := range frames {
+			for _, f := range fs {
+				if ddcalcs[s].RecvPacket(f.toDD()) {
+					frameratesGot = true
+					for _, calc := range ddcalcs {
+						if !calc.Completed() {
+							frameratesGot = false
+							break
+						}
+					}
+				}
+			}
+		}
+		require.True(t, frameratesGot)
+		for i, calc := range ddcalcs {
+			fpsExpected := fps[i]
+			fpsGot := calc.GetFrameRate()
+			verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
+		}
+	})
+}
+
+func TestFpsH26x(t *testing.T) {
+	cases := map[string]testcase{
+		"normal": {
+			startTs:          12345678,
+			startSeq:         100,
+			startFrameNumber: 100,
+			fps:              [][]float32{{5, 10, 15}, {5, 10, 15}, {7.5, 15, 30}},
+		},
+		"frame number and timestamp wrap": {
+			startTs:          (uint32(1) << 31) - 10,
+			startSeq:         (uint16(1) << 15) - 10,
+			startFrameNumber: (uint16(1) << 15) - 10,
+			fps:              [][]float32{{5, 10, 15}, {5, 10, 15}, {7.5, 15, 30}},
+		},
+		"2 temporal layers": {
+			startTs:          12345678,
+			startFrameNumber: 100,
+			fps:              [][]float32{{7.5, 15}, {7.5, 15}, {15, 30}},
+		},
+	}
+
+	for name, c := range cases {
+		testCase := c
+		t.Run(name, func(t *testing.T) {
+			fps := testCase.fps
+			frames := make([][]*testFrameInfo, 0)
+			h26xcalcs := make([]*FrameRateCalculatorH26x, len(fps))
+			for i := range h26xcalcs {
+				h26xcalcs[i] = NewFrameRateCalculatorH26x(90000, logger.GetLogger())
+				frames = append(frames, createFrames(c.startFrameNumber, c.startTs, c.startSeq, 200, [][]float32{fps[i]}, false)[0])
+			}
+
+			var frameratesGot bool
+			for s, fs := range frames {
+				for _, f := range fs {
+					if h26xcalcs[s].RecvPacket(f.toH26x()) {
+						frameratesGot = true
+						for _, calc := range h26xcalcs {
+							if !calc.Completed() {
+								frameratesGot = false
+								break
+							}
+						}
+					}
+				}
+			}
+			require.True(t, frameratesGot)
+			for i, calc := range h26xcalcs {
+				fpsExpected := fps[i]
+				fpsGot := calc.GetFrameRate()
+				verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
+			}
+		})
+	}
+
+	t.Run("packet lost and duplicate", func(t *testing.T) {
+		fps := [][]float32{{7.5, 15, 30}, {7.5, 15, 30}, {7.5, 15, 30}}
+		frames := make([][]*testFrameInfo, 0, len(fps))
+		h26xcalcs := make([]FrameRateCalculator, len(fps))
+		for i := range fps {
+			frames = append(frames, createFrames(100, 12345678, 10, 500, [][]float32{fps[i]}, false)[0])
+			h26xcalcs[i] = NewFrameRateCalculatorH26x(90000, logger.GetLogger())
+			for j := 5; j < 130; j++ {
+				if j%2 == 0 {
+					frames[i][j] = frames[i][j-1]
+				}
+			}
+			for j := 130; j < 230; j++ {
+				if j%3 == 0 {
+					frames[i][j] = nil
+				}
+			}
+			for j := 230; j < 330; j++ {
+				if j%2 == 0 {
+					frames[i][j], frames[i][j-1] = frames[i][j-1], frames[i][j]
+				}
+			}
+		}
+		var frameratesGot bool
+		for s, fs := range frames {
+			for _, f := range fs {
+				if f == nil {
+					continue
+				}
+				if h26xcalcs[s].RecvPacket(f.toH26x()) {
+					frameratesGot = true
+					for _, calc := range h26xcalcs {
+						if !calc.Completed() {
+							frameratesGot = false
+							break
+						}
+					}
+				}
+			}
+		}
+		require.True(t, frameratesGot)
+		for i, calc := range h26xcalcs {
+			fpsExpected := fps[i]
+			fpsGot := calc.GetFrameRate()
+			verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
+		}
+	})
+}

pkg/sfu/buffer/frameintegrity.go

@@ -0,0 +1,238 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
+)
+
+type FrameEntity struct {
+	startSeq  *uint64
+	endSeq    *uint64
+	integrity bool
+
+	pktHistory *PacketHistory
+}
+
+func (fe *FrameEntity) AddPacket(extSeq uint64, ddVal *dd.DependencyDescriptor) {
+	// duplicate packet
+	if fe.integrity {
+		return
+	}
+
+	if fe.startSeq == nil && ddVal.FirstPacketInFrame {
+		fe.startSeq = &extSeq
+	}
+	if fe.endSeq == nil && ddVal.LastPacketInFrame {
+		fe.endSeq = &extSeq
+	}
+
+	if fe.startSeq != nil && fe.endSeq != nil {
+		if fe.pktHistory.PacketsConsecutive(*fe.startSeq, *fe.endSeq) {
+			fe.integrity = true
+		}
+	}
+}
+
+func (fe *FrameEntity) Reset() {
+	fe.integrity = false
+	fe.startSeq, fe.endSeq = nil, nil
+}
+
+func (fe *FrameEntity) Integrity() bool {
+	return fe.integrity
+}
+
+// ------------------------------
+
+type PacketHistory struct {
+	base        uint64
+	last        uint64
+	bits        []uint64
+	packetCount int
+	inited      bool
+}
+
+func NewPacketHistory(packetCount int) *PacketHistory {
+	packetCount = (packetCount + 63) / 64 * 64
+	return &PacketHistory{
+		bits:        make([]uint64, packetCount/64),
+		packetCount: packetCount,
+	}
+}
+
+func (ph *PacketHistory) AddPacket(extSeq uint64) {
+	if !ph.inited {
+		ph.inited = true
+		ph.base = uint64(extSeq)
+		// set base to extSeq-100 to avoid out-of-order packets belongs to first frame to be dropped
+		if ph.base > 100 {
+			ph.base -= 100
+		} else {
+			ph.base = 0
+		}
+		ph.last = uint64(extSeq)
+		ph.set(extSeq, true)
+		return
+	}
+
+	if extSeq <= ph.base {
+		// too old
+		return
+	}
+
+	if extSeq <= ph.last {
+		if ph.last-extSeq < uint64(ph.packetCount) {
+			ph.set(extSeq, true)
+		}
+		return
+	}
+
+	// A forward jump only needs at most packetCount slots cleared; anything older aliases
+	// into the same ring and would be overwritten anyway. Cap the loop so a single crafted
+	// sequence number jump (up to ~32k by the extension wrap-around heuristic) cannot force
+	// a large per-packet iteration count.
+	start := ph.last + 1
+	if extSeq-start > uint64(ph.packetCount) {
+		start = extSeq - uint64(ph.packetCount)
+	}
+	for i := start; i < extSeq; i++ {
+		ph.set(i, false)
+	}
+
+	ph.set(extSeq, true)
+	ph.last = extSeq
+}
+
+func (ph *PacketHistory) getPos(seq uint64) (index, offset int) {
+	idx := (seq - ph.base) % uint64(ph.packetCount)
+	return int(idx >> 6), int(idx % 64)
+}
+
+func (ph *PacketHistory) set(seq uint64, received bool) {
+	idx, offset := ph.getPos(seq)
+	if !received {
+		ph.bits[idx] &= ^(1 << offset)
+	} else {
+		ph.bits[idx] |= 1 << (offset)
+	}
+}
+
+func (ph *PacketHistory) PacketsConsecutive(start, end uint64) bool {
+	if start > end {
+		return false
+	}
+
+	if end-start >= uint64(ph.packetCount) {
+		return false
+	}
+
+	startIndex, startOffset := ph.getPos(start)
+	endIndex, endOffset := ph.getPos(end)
+
+	if startIndex == endIndex && end-start <= 64 {
+		testBits := uint64((1<<(endOffset-startOffset+1))-1) << startOffset
+		return ph.bits[startIndex]&testBits == testBits
+	}
+
+	if (ph.bits[startIndex]>>(startOffset))+1 != 1<<(64-startOffset) {
+		return false
+	}
+
+	for i := startIndex + 1; i != endIndex; i++ {
+		if i == len(ph.bits) {
+			i = 0
+			if i == endIndex {
+				break
+			}
+		}
+		if ph.bits[i]+1 != 0 {
+			return false
+		}
+	}
+
+	testBits := uint64((1 << (endOffset + 1)) - 1)
+	return ph.bits[endIndex]&testBits == testBits
+}
+
+// ------------------------------
+
+type FrameIntegrityChecker struct {
+	frameCount int
+	frames     []FrameEntity
+	base       uint64
+	last       uint64
+
+	pktHistory *PacketHistory
+	inited     bool
+}
+
+func NewFrameIntegrityChecker(frameCount, packetCount int) *FrameIntegrityChecker {
+	fc := &FrameIntegrityChecker{
+		frames:     make([]FrameEntity, frameCount),
+		pktHistory: NewPacketHistory(packetCount),
+		frameCount: frameCount,
+	}
+
+	for i := range fc.frames {
+		fc.frames[i].pktHistory = fc.pktHistory
+		fc.frames[i].Reset()
+	}
+	return fc
+}
+
+func (fc *FrameIntegrityChecker) AddPacket(extSeq uint64, extFrameNum uint64, ddVal *dd.DependencyDescriptor) {
+	fc.pktHistory.AddPacket(extSeq)
+
+	if !fc.inited {
+		fc.inited = true
+		fc.base = extFrameNum
+		fc.last = extFrameNum
+	}
+
+	if extFrameNum < fc.base {
+		// frame too old
+		return
+	}
+
+	if extFrameNum <= fc.last {
+		if fc.last-extFrameNum >= uint64(fc.frameCount) {
+			// frame too old
+			return
+		}
+		fc.frames[int(extFrameNum-fc.base)%fc.frameCount].AddPacket(extSeq, ddVal)
+		return
+	}
+
+	// reset missing frames; cap to frameCount so a crafted frame-number jump cannot force a
+	// large loop (older frames alias into the same ring and get overwritten anyway).
+	start := fc.last + 1
+	if extFrameNum-fc.last > uint64(fc.frameCount) {
+		start = extFrameNum - uint64(fc.frameCount) + 1
+	}
+	for i := start; i <= extFrameNum; i++ {
+		fc.frames[int(i-fc.base)%fc.frameCount].Reset()
+	}
+	fc.frames[int(extFrameNum-fc.base)%fc.frameCount].AddPacket(extSeq, ddVal)
+	fc.last = extFrameNum
+}
+
+func (fc *FrameIntegrityChecker) FrameIntegrity(extFrameNum uint64) bool {
+	if extFrameNum < fc.base || extFrameNum > fc.last || fc.last-extFrameNum >= uint64(fc.frameCount) {
+		return false
+	}
+
+	return fc.frames[int(extFrameNum-fc.base)%fc.frameCount].Integrity()
+}

pkg/sfu/buffer/frameintegrity_test.go

@@ -0,0 +1,147 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"math/bits"
+	"math/rand"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
+)
+
+func TestFrameIntegrityChecker(t *testing.T) {
+	fc := NewFrameIntegrityChecker(100, 1000)
+
+	// first frame out of order
+	fc.AddPacket(10, 10, &dd.DependencyDescriptor{})
+	require.False(t, fc.FrameIntegrity(10))
+	fc.AddPacket(9, 10, &dd.DependencyDescriptor{FirstPacketInFrame: true})
+	require.False(t, fc.FrameIntegrity(10))
+	fc.AddPacket(11, 10, &dd.DependencyDescriptor{LastPacketInFrame: true})
+	require.True(t, fc.FrameIntegrity(10))
+
+	// single packet frame
+	fc.AddPacket(100, 100, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true})
+	require.True(t, fc.FrameIntegrity(100))
+	require.False(t, fc.FrameIntegrity(101))
+	require.False(t, fc.FrameIntegrity(99))
+
+	// frame too old than first frame
+	fc.AddPacket(99, 99, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true})
+
+	// multiple packet frame, out of order
+	fc.AddPacket(2001, 2001, &dd.DependencyDescriptor{})
+	require.False(t, fc.FrameIntegrity(2001))
+	require.False(t, fc.FrameIntegrity(1999))
+	// out of frame count(100)
+	require.False(t, fc.FrameIntegrity(100))
+	require.False(t, fc.FrameIntegrity(1900))
+
+	fc.AddPacket(2000, 2001, &dd.DependencyDescriptor{FirstPacketInFrame: true})
+	require.False(t, fc.FrameIntegrity(2001))
+	fc.AddPacket(2002, 2001, &dd.DependencyDescriptor{LastPacketInFrame: true})
+	require.True(t, fc.FrameIntegrity(2001))
+	// duplicate packet
+	fc.AddPacket(2001, 2001, &dd.DependencyDescriptor{})
+	require.True(t, fc.FrameIntegrity(2001))
+
+	// frame too old
+	fc.AddPacket(900, 1900, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true})
+	require.False(t, fc.FrameIntegrity(1900))
+
+	for frame := uint64(2002); frame < 2102; frame++ {
+		// large frame (1000 packets) out of order / retransmitted
+		firstFrame := uint64(3000 + (frame-2002)*1000)
+		lastFrame := uint64(3999 + (frame-2002)*1000)
+		frames := make([]uint64, 0, lastFrame-firstFrame+1)
+		for i := firstFrame; i <= lastFrame; i++ {
+			frames = append(frames, i)
+		}
+		require.False(t, fc.FrameIntegrity(frame))
+		rng := rand.New(rand.NewSource(int64(frame)))
+		rng.Shuffle(len(frames), func(i, j int) { frames[i], frames[j] = frames[j], frames[i] })
+		for i, f := range frames {
+			fc.AddPacket(f, frame, &dd.DependencyDescriptor{
+				FirstPacketInFrame: f == firstFrame,
+				LastPacketInFrame:  f == lastFrame,
+			})
+			require.Equal(t, i == len(frames)-1, fc.FrameIntegrity(frame), i)
+		}
+		require.True(t, fc.FrameIntegrity(frame))
+	}
+}
+
+func countSetBits(ph *PacketHistory) int {
+	n := 0
+	for _, w := range ph.bits {
+		n += bits.OnesCount64(w)
+	}
+	return n
+}
+
+// A forward sequence-number jump much larger than the ring must clear the whole ring,
+// leaving only the newly received sequence number set.
+func TestPacketHistoryLargeForwardJump(t *testing.T) {
+	ph := NewPacketHistory(1000) // rounds up to a multiple of 64
+	require.Equal(t, 1024, ph.packetCount)
+
+	// Fill the entire ring so every slot holds a "received" bit.
+	base := uint64(100000)
+	ph.AddPacket(base)
+	for i := base + 1; i <= base+2000; i++ {
+		ph.AddPacket(i)
+	}
+	require.Equal(t, ph.packetCount, countSetBits(ph))
+	last := base + 2000
+
+	// Forward jump well beyond both the ring and the ~32k extension wrap-around cap. The ring
+	// must end up fully cleared, with only newSeq marked received.
+	newSeq := last + 40000
+	ph.AddPacket(newSeq)
+
+	// If the cap under-cleared, stale bits from the pre-jump fill would survive here.
+	require.Equal(t, 1, countSetBits(ph))
+	require.True(t, ph.PacketsConsecutive(newSeq, newSeq))
+	require.False(t, ph.PacketsConsecutive(newSeq-5, newSeq))
+
+	// The window just below newSeq was cleared and can be refilled normally.
+	for i := newSeq - 5; i < newSeq; i++ {
+		ph.AddPacket(i)
+	}
+	require.True(t, ph.PacketsConsecutive(newSeq-5, newSeq))
+}
+
+// A forward frame-number jump much larger than frameCount must reset the whole frame ring,
+// so no frame that aliases an old slot inherits stale integrity.
+func TestFrameIntegrityCheckerLargeFrameJump(t *testing.T) {
+	fc := NewFrameIntegrityChecker(100, 1000)
+
+	// Populate every ring slot with an integral single-packet frame.
+	for f := uint64(200); f <= 399; f++ {
+		fc.AddPacket(f, f, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true})
+	}
+	require.True(t, fc.FrameIntegrity(399))
+
+	// Jump far beyond frameCount. The capped reset loop must clear the entire frame ring; if it
+	// under-cleared, some aliased slot would still report a stale frame's integrity.
+	newFrame := uint64(399 + 5000)
+	fc.AddPacket(50000, newFrame, &dd.DependencyDescriptor{}) // incomplete frame, no first/last
+	for f := newFrame - uint64(fc.frameCount) + 1; f <= newFrame; f++ {
+		require.False(t, fc.FrameIntegrity(f), "frame %d should not be integral after jump", f)
+	}
+}

pkg/sfu/buffer/rtcpreader.go

@@ -0,0 +1,63 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"io"
+
+	"go.uber.org/atomic"
+)
+
+type RTCPReader struct {
+	ssrc     uint32
+	closed   atomic.Bool
+	onPacket atomic.Value // func([]byte)
+	onClose  func()
+}
+
+func NewRTCPReader(ssrc uint32) *RTCPReader {
+	return &RTCPReader{ssrc: ssrc}
+}
+
+//go:noinline
+func (r *RTCPReader) Write(p []byte) (n int, err error) {
+	if r.closed.Load() {
+		err = io.EOF
+		return
+	}
+	if f, ok := r.onPacket.Load().(func([]byte)); ok && f != nil {
+		f(p)
+	}
+	return
+}
+
+func (r *RTCPReader) OnClose(fn func()) {
+	r.onClose = fn
+}
+
+func (r *RTCPReader) Close() error {
+	if r.closed.Swap(true) {
+		return nil
+	}
+
+	r.onClose()
+	return nil
+}
+
+func (r *RTCPReader) OnPacket(f func([]byte)) {
+	r.onPacket.Store(f)
+}
+
+func (r *RTCPReader) Read(_ []byte) (n int, err error) { return }

pkg/sfu/buffer/streamstats.go

@@ -0,0 +1,24 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import "github.com/livekit/livekit-server/pkg/sfu/rtpstats"
+
+type StreamStatsWithLayers struct {
+	RTPStats *rtpstats.RTPDeltaInfo
+	Layers   map[int32]*rtpstats.RTPDeltaInfo
+
+	RTPStatsRemoteView *rtpstats.RTPDeltaInfo
+}

pkg/sfu/buffer/videolayer.go

@@ -0,0 +1,58 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import "fmt"
+
+const (
+	InvalidLayerSpatial  = int32(-1)
+	InvalidLayerTemporal = int32(-1)
+
+	DefaultMaxLayerSpatial  = int32(2)
+	DefaultMaxLayerTemporal = int32(3)
+)
+
+var (
+	InvalidLayer = VideoLayer{
+		Spatial:  InvalidLayerSpatial,
+		Temporal: InvalidLayerTemporal,
+	}
+
+	DefaultMaxLayer = VideoLayer{
+		Spatial:  DefaultMaxLayerSpatial,
+		Temporal: DefaultMaxLayerTemporal,
+	}
+)
+
+type VideoLayer struct {
+	Spatial  int32
+	Temporal int32
+}
+
+func (v VideoLayer) String() string {
+	return fmt.Sprintf("VideoLayer{s: %d, t: %d}", v.Spatial, v.Temporal)
+}
+
+func (v VideoLayer) GreaterThan(v2 VideoLayer) bool {
+	return v.Spatial > v2.Spatial || (v.Spatial == v2.Spatial && v.Temporal > v2.Temporal)
+}
+
+func (v VideoLayer) SpatialGreaterThanOrEqual(v2 VideoLayer) bool {
+	return v.Spatial >= v2.Spatial
+}
+
+func (v VideoLayer) IsValid() bool {
+	return v.Spatial != InvalidLayerSpatial && v.Temporal != InvalidLayerTemporal
+}

pkg/sfu/buffer/videolayerutils.go

@@ -0,0 +1,520 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"slices"
+
+	"github.com/livekit/protocol/codecs/mime"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+)
+
+const (
+	quarterResolutionQ = "q"
+	halfResolutionH    = "h"
+	fullResolutionF    = "f"
+
+	quarterResolution2 = "2"
+	halfResolution1    = "1"
+	fullResolution0    = "0"
+)
+
+type VideoLayersRid [DefaultMaxLayerSpatial + 1]string
+
+var (
+	videoLayersRidQHF     = VideoLayersRid{quarterResolutionQ, halfResolutionH, fullResolutionF}
+	videoLayersRid210     = VideoLayersRid{quarterResolution2, halfResolution1, fullResolution0}
+	DefaultVideoLayersRid = videoLayersRidQHF
+)
+
+func LayerPresenceFromTrackInfo(mimeType mime.MimeType, trackInfo *livekit.TrackInfo) *[livekit.VideoQuality_HIGH + 1]bool {
+	if trackInfo == nil {
+		return nil
+	}
+
+	layers := GetVideoLayersForMimeType(mimeType, trackInfo)
+	if len(layers) == 0 {
+		return nil
+	}
+
+	var layerPresence [livekit.VideoQuality_HIGH + 1]bool
+	for _, layer := range layers {
+		// WARNING: comparing protobuf enum
+		if layer.Quality <= livekit.VideoQuality_HIGH {
+			layerPresence[layer.Quality] = true
+		} else {
+			logger.Warnw("unexpected quality in track info", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
+		}
+	}
+
+	return &layerPresence
+}
+
+func RidToSpatialLayer(mimeType mime.MimeType, rid string, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) int32 {
+	lp := LayerPresenceFromTrackInfo(mimeType, trackInfo)
+	if lp == nil {
+		switch rid {
+		case quarterResolutionQ:
+			return 0
+		case halfResolutionH:
+			return 1
+		case fullResolutionF:
+			return 2
+		default:
+			return 0
+		}
+	}
+
+	switch rid {
+	case ridSpace[0]:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return 0
+
+		default:
+			// only one quality published, could be any
+			return 0
+		}
+
+	case ridSpace[1]:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return 1
+
+		default:
+			// only one quality published, could be any
+			return 0
+		}
+
+	case ridSpace[2]:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return 2
+
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			logger.Warnw("unexpected rid with only two qualities, low and medium", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo), "rid", ridSpace[2])
+			return 1
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			logger.Warnw("unexpected rid with only two qualities, low and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo), "rid", ridSpace[2])
+			return 1
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			logger.Warnw("unexpected rid with only two qualities, medium and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo), "rid", ridSpace[2])
+			return 1
+
+		default:
+			// only one quality published, could be any
+			return 0
+		}
+
+	default:
+		// no rid, should be single layer
+		return 0
+	}
+}
+
+func SpatialLayerToRid(mimeType mime.MimeType, layer int32, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) string {
+	lp := LayerPresenceFromTrackInfo(mimeType, trackInfo)
+	if lp == nil {
+		switch layer {
+		case 0:
+			return quarterResolutionQ
+		case 1:
+			return halfResolutionH
+		case 2:
+			return fullResolutionF
+		default:
+			return quarterResolutionQ
+		}
+	}
+
+	switch layer {
+	case 0:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return ridSpace[0]
+
+		default:
+			return ridSpace[0]
+		}
+
+	case 1:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return ridSpace[1]
+
+		default:
+			return ridSpace[0]
+		}
+
+	case 2:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return ridSpace[2]
+
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			logger.Warnw("unexpected layer 2 with only two qualities, low and medium", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
+			return ridSpace[1]
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			logger.Warnw("unexpected layer 2 with only two qualities, low and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
+			return ridSpace[1]
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			logger.Warnw("unexpected layer 2 with only two qualities, medium and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
+			return ridSpace[1]
+
+		default:
+			return ridSpace[0]
+		}
+
+	default:
+		return ridSpace[0]
+	}
+}
+
+func VideoQualityToRid(mimeType mime.MimeType, quality livekit.VideoQuality, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) string {
+	return SpatialLayerToRid(mimeType, VideoQualityToSpatialLayer(mimeType, quality, trackInfo), trackInfo, ridSpace)
+}
+
+func SpatialLayerToVideoQuality(mimeType mime.MimeType, layer int32, trackInfo *livekit.TrackInfo) livekit.VideoQuality {
+	lp := LayerPresenceFromTrackInfo(mimeType, trackInfo)
+	if lp == nil {
+		switch layer {
+		case 0:
+			return livekit.VideoQuality_LOW
+		case 1:
+			return livekit.VideoQuality_MEDIUM
+		case 2:
+			return livekit.VideoQuality_HIGH
+		default:
+			return livekit.VideoQuality_OFF
+		}
+	}
+
+	switch layer {
+	case 0:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW]:
+			return livekit.VideoQuality_LOW
+
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_MEDIUM]:
+			return livekit.VideoQuality_MEDIUM
+
+		default:
+			return livekit.VideoQuality_HIGH
+		}
+
+	case 1:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			return livekit.VideoQuality_MEDIUM
+
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return livekit.VideoQuality_HIGH
+
+		default:
+			logger.Errorw("invalid layer", nil, "trackID", trackInfo.Sid, "layer", layer, "trackInfo", logger.Proto(trackInfo))
+			return livekit.VideoQuality_HIGH
+		}
+
+	case 2:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return livekit.VideoQuality_HIGH
+
+		default:
+			logger.Errorw("invalid layer", nil, "trackID", trackInfo.Sid, "layer", layer, "trackInfo", logger.Proto(trackInfo))
+			return livekit.VideoQuality_HIGH
+		}
+	}
+
+	return livekit.VideoQuality_OFF
+}
+
+func VideoQualityToSpatialLayer(mimeType mime.MimeType, quality livekit.VideoQuality, trackInfo *livekit.TrackInfo) int32 {
+	lp := LayerPresenceFromTrackInfo(mimeType, trackInfo)
+	if lp == nil {
+		switch quality {
+		case livekit.VideoQuality_LOW:
+			return 0
+		case livekit.VideoQuality_MEDIUM:
+			return 1
+		case livekit.VideoQuality_HIGH:
+			return 2
+		default:
+			return InvalidLayerSpatial
+		}
+	}
+
+	switch quality {
+	case livekit.VideoQuality_LOW:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		default: // only one quality published, could be any
+			return 0
+		}
+
+	case livekit.VideoQuality_MEDIUM:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			return 1
+
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return 0
+
+		default: // only one quality published, could be any
+			return 0
+		}
+
+	case livekit.VideoQuality_HIGH:
+		switch {
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return 2
+
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
+			fallthrough
+		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
+			fallthrough
+		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
+			return 1
+
+		default: // only one quality published, could be any
+			return 0
+		}
+	}
+
+	return InvalidLayerSpatial
+}
+
+func GetVideoLayerModeForMimeType(mimeType mime.MimeType, ti *livekit.TrackInfo) livekit.VideoLayer_Mode {
+	if ti != nil {
+		for _, codec := range ti.Codecs {
+			if mime.NormalizeMimeType(codec.MimeType) == mimeType {
+				return codec.VideoLayerMode
+			}
+		}
+	}
+
+	return livekit.VideoLayer_MODE_UNUSED
+}
+
+func GetVideoLayersForMimeType(mimeType mime.MimeType, ti *livekit.TrackInfo) []*livekit.VideoLayer {
+	var layers []*livekit.VideoLayer
+	if ti != nil {
+		for _, codec := range ti.Codecs {
+			if mime.NormalizeMimeType(codec.MimeType) == mimeType {
+				layers = codec.Layers
+				break
+			}
+		}
+		if len(layers) == 0 {
+			layers = ti.Layers
+		}
+	}
+	return layers
+}
+
+func GetSpatialLayerForRid(mimeType mime.MimeType, rid string, ti *livekit.TrackInfo) int32 {
+	if ti == nil {
+		return InvalidLayerSpatial
+	}
+
+	if rid == "" {
+		// single layer without RID
+		return 0
+	}
+
+	layers := GetVideoLayersForMimeType(mimeType, ti)
+	for _, layer := range layers {
+		if layer.Rid == rid {
+			return layer.SpatialLayer
+		}
+	}
+
+	if len(layers) != 0 {
+		// RID present in codec, but may not be specified via signalling
+		// (happens with older browsers setting a rid for SVC codecs)
+		hasRid := false
+		for _, layer := range layers {
+			if layer.Rid != "" {
+				hasRid = true
+				break
+			}
+		}
+		if !hasRid {
+			return 0
+		}
+	}
+
+	// SIMULCAST-CODEC-TODO - ideally should return invalid, but there are
+	// VP9 publishers using rid = f, if there are only two layers
+	// in TrackInfo, that will be q;h and f will become invalid.
+	//
+	// Actually, there should be no rids for VP9 in SDP and hence
+	// the above check should take effect. However, as simulcast
+	// codec/back up codec does not update rids from SDP,
+	// the default rids are used when vp9 (primary codec)
+	// is published. Due to that the above check gets bypassed.
+	//
+	// The full proper sequence would be
+	// 1. For primary codec using SVC, there will be no rids.
+	//    The above check should take effect and it should
+	//    return 0 even if some publisher uses a rid like `f`.
+	// 2. When secondary codec is published, rids for the codec
+	//    corresponding to the back up codec mime type should
+	//    be updated in `TrackInfo`. This is a bit tricky
+	//    for a couple of cases
+	//    a. Browsers like Firefox use a different CID every time.
+	//       So, it cannot be matched between `AddTrack` and SDP.
+	//       One option is to look for a published track with
+	//       back up codec and apply it there. But, that becomes
+	//       a challenge if there are multiple published tracks
+	//       with pending back up codec.
+	//    b. The back up codec publish SDP will have the full
+	//       codec list. It should be okay to assume that the
+	//       codec that will be published is the back up codec,
+	//       but just something to be aware of.
+	// 3. Use of this function with proper mime so that proper
+	//    codec section can be looked up in `TrackInfo`.
+	// return InvalidLayerSpatial
+	logger.Infow(
+		"invalid layer for rid, returning default",
+		"trackID", ti.Sid,
+		"rid", rid,
+		"mimeType", mimeType,
+		"trackInfo", logger.Proto(ti),
+	)
+	return 0
+}
+
+func GetSpatialLayerForVideoQuality(mimeType mime.MimeType, quality livekit.VideoQuality, ti *livekit.TrackInfo) int32 {
+	if ti == nil || quality == livekit.VideoQuality_OFF {
+		return InvalidLayerSpatial
+	}
+
+	layers := GetVideoLayersForMimeType(mimeType, ti)
+	for _, layer := range layers {
+		if layer.Quality == quality {
+			return layer.SpatialLayer
+		}
+	}
+
+	if len(layers) == 0 {
+		// single layer
+		return 0
+	}
+
+	// requested quality is higher than available layers, return the highest available layer
+	return VideoQualityToSpatialLayer(mimeType, quality, ti)
+}
+
+func GetVideoQualityForSpatialLayer(mimeType mime.MimeType, spatialLayer int32, ti *livekit.TrackInfo) livekit.VideoQuality {
+	if spatialLayer == InvalidLayerSpatial || ti == nil {
+		return livekit.VideoQuality_OFF
+	}
+
+	layers := GetVideoLayersForMimeType(mimeType, ti)
+	for _, layer := range layers {
+		if layer.SpatialLayer == spatialLayer {
+			return layer.Quality
+		}
+	}
+
+	return livekit.VideoQuality_OFF
+}
+
+func isVideoLayersRidKnown(rids VideoLayersRid, knownRids VideoLayersRid) bool {
+	for _, rid := range rids {
+		if rid == "" {
+			continue
+		}
+
+		if !slices.Contains(knownRids[:], rid) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func NormalizeVideoLayersRid(rids VideoLayersRid) VideoLayersRid {
+	out := rids
+
+	normalize := func(knownRids VideoLayersRid) {
+		idx := 0
+		for _, known := range knownRids {
+			if slices.Contains(rids[:], known) {
+				out[idx] = known
+				idx++
+			}
+		}
+	}
+
+	if isVideoLayersRidKnown(rids, videoLayersRidQHF) {
+		normalize(videoLayersRidQHF)
+	}
+
+	if isVideoLayersRidKnown(rids, videoLayersRid210) {
+		normalize(videoLayersRid210)
+	}
+
+	return out
+}

pkg/sfu/buffer/videolayerutils_test.go

@@ -0,0 +1,892 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package buffer
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/protocol/codecs/mime"
+	"github.com/livekit/protocol/livekit"
+)
+
+func TestRidConversion(t *testing.T) {
+	type RidAndLayer struct {
+		rid   string
+		layer int32
+	}
+	tests := []struct {
+		name       string
+		trackInfo  *livekit.TrackInfo
+		mimeType   mime.MimeType
+		ridToLayer map[string]RidAndLayer
+	}{
+		{
+			"no track info",
+			nil,
+			mime.MimeTypeVP8,
+			map[string]RidAndLayer{
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: halfResolutionH, layer: 1},
+				fullResolutionF:    {rid: fullResolutionF, layer: 2},
+			},
+		},
+		{
+			"no layers",
+			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
+			map[string]RidAndLayer{
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: halfResolutionH, layer: 1},
+				fullResolutionF:    {rid: fullResolutionF, layer: 2},
+			},
+		},
+		{
+			"single layer, low",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]RidAndLayer{
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: quarterResolutionQ, layer: 0},
+				fullResolutionF:    {rid: quarterResolutionQ, layer: 0},
+			},
+		},
+		{
+			"single layer, medium",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]RidAndLayer{
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: quarterResolutionQ, layer: 0},
+				fullResolutionF:    {rid: quarterResolutionQ, layer: 0},
+			},
+		},
+		{
+			"single layer, high",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]RidAndLayer{
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: quarterResolutionQ, layer: 0},
+				fullResolutionF:    {rid: quarterResolutionQ, layer: 0},
+			},
+		},
+		{
+			"two layers, low and medium",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]RidAndLayer{
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: halfResolutionH, layer: 1},
+				fullResolutionF:    {rid: halfResolutionH, layer: 1},
+			},
+		},
+		{
+			"two layers, low and high",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]RidAndLayer{
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: halfResolutionH, layer: 1},
+				fullResolutionF:    {rid: halfResolutionH, layer: 1},
+			},
+		},
+		{
+			"two layers, medium and high",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]RidAndLayer{
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: halfResolutionH, layer: 1},
+				fullResolutionF:    {rid: halfResolutionH, layer: 1},
+			},
+		},
+		{
+			"three layers",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_MEDIUM},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]RidAndLayer{
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: halfResolutionH, layer: 1},
+				fullResolutionF:    {rid: fullResolutionF, layer: 2},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			for testRid, expectedResult := range test.ridToLayer {
+				actualLayer := RidToSpatialLayer(test.mimeType, testRid, test.trackInfo, DefaultVideoLayersRid)
+				require.Equal(t, expectedResult.layer, actualLayer)
+
+				actualRid := SpatialLayerToRid(test.mimeType, actualLayer, test.trackInfo, DefaultVideoLayersRid)
+				require.Equal(t, expectedResult.rid, actualRid)
+			}
+		})
+	}
+}
+
+func TestQualityConversion(t *testing.T) {
+	type QualityAndLayer struct {
+		quality livekit.VideoQuality
+		layer   int32
+	}
+	tests := []struct {
+		name           string
+		trackInfo      *livekit.TrackInfo
+		mimeType       mime.MimeType
+		qualityToLayer map[livekit.VideoQuality]QualityAndLayer
+	}{
+		{
+			"no track info",
+			nil,
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]QualityAndLayer{
+				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_LOW, layer: 0},
+				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
+				livekit.VideoQuality_HIGH:   {quality: livekit.VideoQuality_HIGH, layer: 2},
+			},
+		},
+		{
+			"no layers",
+			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]QualityAndLayer{
+				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_LOW, layer: 0},
+				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
+				livekit.VideoQuality_HIGH:   {quality: livekit.VideoQuality_HIGH, layer: 2},
+			},
+		},
+		{
+			"single layer, low",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]QualityAndLayer{
+				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_LOW, layer: 0},
+				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_LOW, layer: 0},
+				livekit.VideoQuality_HIGH:   {quality: livekit.VideoQuality_LOW, layer: 0},
+			},
+		},
+		{
+			"single layer, medium",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]QualityAndLayer{
+				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_MEDIUM, layer: 0},
+				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
+				livekit.VideoQuality_HIGH:   {quality: livekit.VideoQuality_MEDIUM, layer: 0},
+			},
+		},
+		{
+			"single layer, high",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]QualityAndLayer{
+				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_HIGH, layer: 0},
+				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 0},
+				livekit.VideoQuality_HIGH:   {quality: livekit.VideoQuality_HIGH, layer: 0},
+			},
+		},
+		{
+			"two layers, low and medium",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]QualityAndLayer{
+				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_LOW, layer: 0},
+				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
+				livekit.VideoQuality_HIGH:   {quality: livekit.VideoQuality_MEDIUM, layer: 1},
+			},
+		},
+		{
+			"two layers, low and high",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]QualityAndLayer{
+				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_LOW, layer: 0},
+				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 1},
+				livekit.VideoQuality_HIGH:   {quality: livekit.VideoQuality_HIGH, layer: 1},
+			},
+		},
+		{
+			"two layers, medium and high",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]QualityAndLayer{
+				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_MEDIUM, layer: 0},
+				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
+				livekit.VideoQuality_HIGH:   {quality: livekit.VideoQuality_HIGH, layer: 1},
+			},
+		},
+		{
+			"three layers",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_MEDIUM},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]QualityAndLayer{
+				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_LOW, layer: 0},
+				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
+				livekit.VideoQuality_HIGH:   {quality: livekit.VideoQuality_HIGH, layer: 2},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			for testQuality, expectedResult := range test.qualityToLayer {
+				actualLayer := VideoQualityToSpatialLayer(test.mimeType, testQuality, test.trackInfo)
+				require.Equal(t, expectedResult.layer, actualLayer)
+
+				actualQuality := SpatialLayerToVideoQuality(test.mimeType, actualLayer, test.trackInfo)
+				require.Equal(t, expectedResult.quality, actualQuality)
+			}
+		})
+	}
+}
+
+func TestVideoQualityToRidConversion(t *testing.T) {
+	tests := []struct {
+		name         string
+		trackInfo    *livekit.TrackInfo
+		mimeTye      mime.MimeType
+		qualityToRid map[livekit.VideoQuality]string
+	}{
+		{
+			"no track info",
+			nil,
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]string{
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: halfResolutionH,
+				livekit.VideoQuality_HIGH:   fullResolutionF,
+			},
+		},
+		{
+			"no layers",
+			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]string{
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: halfResolutionH,
+				livekit.VideoQuality_HIGH:   fullResolutionF,
+			},
+		},
+		{
+			"single layer, low",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]string{
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: quarterResolutionQ,
+				livekit.VideoQuality_HIGH:   quarterResolutionQ,
+			},
+		},
+		{
+			"single layer, medium",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]string{
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: quarterResolutionQ,
+				livekit.VideoQuality_HIGH:   quarterResolutionQ,
+			},
+		},
+		{
+			"single layer, high",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]string{
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: quarterResolutionQ,
+				livekit.VideoQuality_HIGH:   quarterResolutionQ,
+			},
+		},
+		{
+			"two layers, low and medium",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]string{
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: halfResolutionH,
+				livekit.VideoQuality_HIGH:   halfResolutionH,
+			},
+		},
+		{
+			"two layers, low and high",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]string{
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: halfResolutionH,
+				livekit.VideoQuality_HIGH:   halfResolutionH,
+			},
+		},
+		{
+			"two layers, medium and high",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]string{
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: quarterResolutionQ,
+				livekit.VideoQuality_HIGH:   halfResolutionH,
+			},
+		},
+		{
+			"three layers",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_MEDIUM},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]string{
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: halfResolutionH,
+				livekit.VideoQuality_HIGH:   fullResolutionF,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			for testQuality, expectedRid := range test.qualityToRid {
+				actualRid := VideoQualityToRid(test.mimeTye, testQuality, test.trackInfo, DefaultVideoLayersRid)
+				require.Equal(t, expectedRid, actualRid)
+			}
+		})
+	}
+}
+
+func TestGetSpatialLayerForRid(t *testing.T) {
+	tests := []struct {
+		name              string
+		trackInfo         *livekit.TrackInfo
+		mimeType          mime.MimeType
+		ridToSpatialLayer map[string]int32
+	}{
+		{
+			"no track info",
+			nil,
+			mime.MimeTypeVP8,
+			map[string]int32{
+				quarterResolutionQ: InvalidLayerSpatial,
+				halfResolutionH:    InvalidLayerSpatial,
+				fullResolutionF:    InvalidLayerSpatial,
+			},
+		},
+		{
+			"no layers",
+			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
+			map[string]int32{
+				// SIMULCAST-CODEC-TODO
+				// quarterResolutionQ: InvalidLayerSpatial,
+				// halfResolutionH:    InvalidLayerSpatial,
+				// fullResolutionF:    InvalidLayerSpatial,
+				quarterResolutionQ: 0,
+				halfResolutionH:    0,
+				fullResolutionF:    0,
+			},
+		},
+		{
+			"no rid",
+			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
+			map[string]int32{
+				"": 0,
+			},
+		},
+		{
+			"single layer",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]int32{
+				quarterResolutionQ: 0,
+				halfResolutionH:    0,
+				fullResolutionF:    0,
+			},
+		},
+		{
+			"layers",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ},
+							{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]int32{
+				quarterResolutionQ: 0,
+				halfResolutionH:    1,
+				// SIMULCAST-CODEC-TODO
+				// fullResolutionF:    InvalidLayerSpatial,
+				fullResolutionF: 0,
+			},
+		},
+		{
+			"layers - no rid",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0},
+							{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]int32{
+				quarterResolutionQ: 0,
+				halfResolutionH:    0,
+				fullResolutionF:    0,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			for testRid, expectedSpatialLayer := range test.ridToSpatialLayer {
+				actualSpatialLayer := GetSpatialLayerForRid(test.mimeType, testRid, test.trackInfo)
+				require.Equal(t, expectedSpatialLayer, actualSpatialLayer)
+			}
+		})
+	}
+}
+
+func TestGetSpatialLayerForVideoQuality(t *testing.T) {
+	tests := []struct {
+		name                       string
+		trackInfo                  *livekit.TrackInfo
+		mimeType                   mime.MimeType
+		videoQualityToSpatialLayer map[livekit.VideoQuality]int32
+	}{
+		{
+			"no track info",
+			nil,
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]int32{
+				livekit.VideoQuality_LOW:    InvalidLayerSpatial,
+				livekit.VideoQuality_MEDIUM: InvalidLayerSpatial,
+				livekit.VideoQuality_HIGH:   InvalidLayerSpatial,
+				livekit.VideoQuality_OFF:    InvalidLayerSpatial,
+			},
+		},
+		{
+			"no layers",
+			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]int32{
+				livekit.VideoQuality_LOW:    0,
+				livekit.VideoQuality_MEDIUM: 0,
+				livekit.VideoQuality_HIGH:   0,
+				livekit.VideoQuality_OFF:    InvalidLayerSpatial,
+			},
+		},
+		{
+			"not all layers",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ},
+							{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]int32{
+				livekit.VideoQuality_LOW:    0,
+				livekit.VideoQuality_MEDIUM: 1,
+				livekit.VideoQuality_HIGH:   1,
+				livekit.VideoQuality_OFF:    InvalidLayerSpatial,
+			},
+		},
+		{
+			"all layers",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ},
+							{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH},
+							{Quality: livekit.VideoQuality_HIGH, SpatialLayer: 2, Rid: fullResolutionF},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]int32{
+				livekit.VideoQuality_LOW:    0,
+				livekit.VideoQuality_MEDIUM: 1,
+				livekit.VideoQuality_HIGH:   2,
+				livekit.VideoQuality_OFF:    InvalidLayerSpatial,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			for testVideoQuality, expectedSpatialLayer := range test.videoQualityToSpatialLayer {
+				actualSpatialLayer := GetSpatialLayerForVideoQuality(test.mimeType, testVideoQuality, test.trackInfo)
+				require.Equal(t, expectedSpatialLayer, actualSpatialLayer)
+			}
+		})
+	}
+}
+
+func TestGetVideoQualityorSpatialLayer(t *testing.T) {
+	tests := []struct {
+		name                       string
+		trackInfo                  *livekit.TrackInfo
+		mimeType                   mime.MimeType
+		spatialLayerToVideoQuality map[int32]livekit.VideoQuality
+	}{
+		{
+			"no track info",
+			nil,
+			mime.MimeTypeVP8,
+			map[int32]livekit.VideoQuality{
+				InvalidLayerSpatial: livekit.VideoQuality_OFF,
+				0:                   livekit.VideoQuality_OFF,
+				1:                   livekit.VideoQuality_OFF,
+				2:                   livekit.VideoQuality_OFF,
+			},
+		},
+		{
+			"no layers",
+			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
+			map[int32]livekit.VideoQuality{
+				InvalidLayerSpatial: livekit.VideoQuality_OFF,
+				0:                   livekit.VideoQuality_OFF,
+				1:                   livekit.VideoQuality_OFF,
+				2:                   livekit.VideoQuality_OFF,
+			},
+		},
+		{
+			"layers",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ},
+							{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[int32]livekit.VideoQuality{
+				InvalidLayerSpatial: livekit.VideoQuality_OFF,
+				0:                   livekit.VideoQuality_LOW,
+				1:                   livekit.VideoQuality_MEDIUM,
+				2:                   livekit.VideoQuality_OFF,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			for testSpatialLayer, expectedVideoQuality := range test.spatialLayerToVideoQuality {
+				actualVideoQuality := GetVideoQualityForSpatialLayer(test.mimeType, testSpatialLayer, test.trackInfo)
+				require.Equal(t, expectedVideoQuality, actualVideoQuality)
+			}
+		})
+	}
+}
+
+func TestNormalizeVideoLayersRid(t *testing.T) {
+	tests := []struct {
+		name       string
+		rids       VideoLayersRid
+		normalized VideoLayersRid
+	}{
+		{
+			"empty",
+			VideoLayersRid{},
+			VideoLayersRid{},
+		},
+		{
+			"unknown pattern",
+			VideoLayersRid{"3", "2", "1"},
+			VideoLayersRid{"3", "2", "1"},
+		},
+		{
+			"qhf",
+			videoLayersRidQHF,
+			videoLayersRidQHF,
+		},
+		{
+			"scrambled qhf",
+			VideoLayersRid{"f", "h", "q"},
+			videoLayersRidQHF,
+		},
+		{
+			"partial qhf",
+			VideoLayersRid{"h", "q"},
+			VideoLayersRid{"q", "h", ""},
+		},
+		{
+			"210",
+			videoLayersRid210,
+			videoLayersRid210,
+		},
+		{
+			"scrambled 210",
+			VideoLayersRid{"2", "0", "1"},
+			videoLayersRid210,
+		},
+		{
+			"partial 210",
+			VideoLayersRid{"1", "2"},
+			VideoLayersRid{"2", "1", ""},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			normalizedRids := NormalizeVideoLayersRid(test.rids)
+			require.Equal(t, test.normalized, normalizedRids)
+		})
+	}
+}