snapshot of https://github.com/livekit/livekit.git @ 08c5433449c106943231037d0b892895f6e92d3b (2026-09-15)

pkg/sfu/videolayerselector/base.go

@@ -0,0 +1,180 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package videolayerselector
+
+import (
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/sfu/videolayerselector/temporallayerselector"
+	"github.com/livekit/protocol/logger"
+)
+
+type Base struct {
+	logger logger.Logger
+
+	tls temporallayerselector.TemporalLayerSelector
+
+	maxLayer     buffer.VideoLayer
+	maxSeenLayer buffer.VideoLayer
+
+	targetLayer         buffer.VideoLayer
+	previousTargetLayer buffer.VideoLayer
+
+	requestSpatial int32
+
+	currentLayer  buffer.VideoLayer
+	previousLayer buffer.VideoLayer
+
+	// when set, on initial acquisition latch directly onto the target (requested) layer instead of
+	// opportunistically latching onto the first lower-layer key frame that arrives (see Simulcast.Select)
+	enableStartAtDesiredQuality bool
+}
+
+func NewBase(logger logger.Logger) *Base {
+	return &Base{
+		logger:              logger,
+		maxLayer:            buffer.InvalidLayer,
+		maxSeenLayer:        buffer.InvalidLayer,
+		targetLayer:         buffer.InvalidLayer, // start off with nothing, let streamallocator/opportunistic forwarder set the target
+		previousTargetLayer: buffer.InvalidLayer,
+		requestSpatial:      buffer.InvalidLayerSpatial,
+		currentLayer:        buffer.InvalidLayer,
+		previousLayer:       buffer.InvalidLayer,
+	}
+}
+
+func (b *Base) getBase() *Base {
+	return b
+}
+
+func (b *Base) getLogger() logger.Logger {
+	return b.logger
+}
+
+func (b *Base) IsOvershootOkay() bool {
+	return false
+}
+
+func (b *Base) SetTemporalLayerSelector(tls temporallayerselector.TemporalLayerSelector) {
+	b.tls = tls
+}
+
+func (b *Base) SetEnableStartAtDesiredQuality(enable bool) {
+	b.enableStartAtDesiredQuality = enable
+}
+
+func (b *Base) SetMax(maxLayer buffer.VideoLayer) {
+	b.maxLayer = maxLayer
+}
+
+func (b *Base) SetMaxSpatial(layer int32) {
+	b.maxLayer.Spatial = layer
+}
+
+func (b *Base) SetMaxTemporal(layer int32) {
+	b.maxLayer.Temporal = layer
+}
+
+func (b *Base) GetMax() buffer.VideoLayer {
+	return b.maxLayer
+}
+
+func (b *Base) SetTarget(targetLayer buffer.VideoLayer) {
+	b.previousTargetLayer = targetLayer
+	b.targetLayer = targetLayer
+}
+
+func (b *Base) GetTarget() buffer.VideoLayer {
+	return b.targetLayer
+}
+
+func (b *Base) SetRequestSpatial(layer int32) {
+	b.requestSpatial = layer
+}
+
+func (b *Base) GetRequestSpatial() int32 {
+	return b.requestSpatial
+}
+
+func (b *Base) CheckSync() (locked bool, layer int32) {
+	layer = b.GetRequestSpatial()
+	locked = layer == b.GetCurrent().Spatial
+	return
+}
+
+func (b *Base) SetMaxSeen(maxSeenLayer buffer.VideoLayer) {
+	b.maxSeenLayer = maxSeenLayer
+}
+
+func (b *Base) SetMaxSeenSpatial(layer int32) {
+	b.maxSeenLayer.Spatial = layer
+}
+
+func (b *Base) SetMaxSeenTemporal(layer int32) {
+	b.maxSeenLayer.Temporal = layer
+}
+
+func (b *Base) GetMaxSeen() buffer.VideoLayer {
+	return b.maxSeenLayer
+}
+
+func (b *Base) SetCurrent(currentLayer buffer.VideoLayer) {
+	b.currentLayer = currentLayer
+}
+
+func (b *Base) GetCurrent() buffer.VideoLayer {
+	return b.currentLayer
+}
+
+func (b *Base) Select(_extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerSelectorResult) {
+	return
+}
+
+func (b *Base) Rollback() {
+	b.logger.Debugw(
+		"rolling back",
+		"previous", b.previousLayer,
+		"current", b.currentLayer,
+		"previousTarget", b.previousTargetLayer,
+		"target", b.targetLayer,
+		"max", b.maxLayer,
+		"req", b.requestSpatial,
+		"maxSeen", b.maxSeenLayer,
+	)
+	b.currentLayer = b.previousLayer
+	b.targetLayer = b.previousTargetLayer
+}
+
+func (b *Base) SelectTemporal(extPkt *buffer.ExtPacket) int32 {
+	if b.tls != nil {
+		this, next := b.tls.Select(extPkt, b.currentLayer.Temporal, b.targetLayer.Temporal)
+		if next != b.currentLayer.Temporal {
+			previousLayer := b.currentLayer
+			b.currentLayer.Temporal = next
+
+			b.logger.Debugw(
+				"updating temporal layer",
+				"previous", previousLayer,
+				"current", b.currentLayer,
+				"target", b.targetLayer,
+				"max", b.maxLayer,
+				"req", b.requestSpatial,
+				"maxSeen", b.maxSeenLayer,
+			)
+		}
+		return this
+	}
+
+	return b.currentLayer.Temporal
+}

pkg/sfu/videolayerselector/decodetarget.go

@@ -0,0 +1,70 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package videolayerselector
+
+import (
+	"fmt"
+
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
+)
+
+type DecodeTarget struct {
+	buffer.DependencyDescriptorDecodeTarget
+	chain  *FrameChain
+	active bool
+}
+
+type FrameDetectionResult struct {
+	TargetValid bool
+	DTI         dd.DecodeTargetIndication
+}
+
+func NewDecodeTarget(target buffer.DependencyDescriptorDecodeTarget, chain *FrameChain) *DecodeTarget {
+	return &DecodeTarget{
+		DependencyDescriptorDecodeTarget: target,
+		chain:                            chain,
+	}
+}
+
+func (dt *DecodeTarget) Valid() bool {
+	return dt.chain == nil || !dt.chain.Broken()
+}
+
+func (dt *DecodeTarget) Active() bool {
+	return dt.active
+}
+
+func (dt *DecodeTarget) UpdateActive(activeBitmask uint32) {
+	active := (activeBitmask & (1 << dt.Target)) != 0
+	dt.active = active
+	if dt.chain != nil {
+		dt.chain.UpdateActive(active)
+	}
+}
+
+func (dt *DecodeTarget) OnFrame(extFrameNum uint64, fd *dd.FrameDependencyTemplate) (FrameDetectionResult, error) {
+	result := FrameDetectionResult{}
+	if len(fd.DecodeTargetIndications) <= dt.Target {
+		return result, fmt.Errorf("mismatch target %d and len(DecodeTargetIndications) %d", dt.Target, len(fd.DecodeTargetIndications))
+	}
+
+	result.DTI = fd.DecodeTargetIndications[dt.Target]
+	// The encoder can choose not to use frame chain in theory, and we need to trace every required frame is decodable in this case.
+	// But we don't observe this in browser and it makes no sense to not use the chain with svc, so only use chain to detect decode target broken now,
+	// and always return decodable if it is not protect by chain.
+	result.TargetValid = dt.Valid()
+	return result, nil
+}

pkg/sfu/videolayerselector/dependencydescriptor.go

@@ -0,0 +1,464 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package videolayerselector
+
+import (
+	"fmt"
+	"runtime/debug"
+	"sync"
+
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	dede "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
+	"github.com/livekit/protocol/logger"
+)
+
+const (
+	decisionCacheMaxElements = 256
+	decisionCacheNackEntries = 80
+)
+
+type DependencyDescriptor struct {
+	*Base
+
+	decisions *SelectorDecisionCache
+
+	previousActiveDecodeTargetsBitmask *uint32
+	activeDecodeTargetsBitmask         *uint32
+	structure                          *dede.FrameDependencyStructure
+	extKeyFrameNum                     uint64
+	keyFrameValid                      bool
+
+	chains []*FrameChain
+
+	decodeTargetsLock sync.RWMutex
+	decodeTargets     []*DecodeTarget
+	fnWrapper         FrameNumberWrapper
+
+	restartGeneration int
+}
+
+func NewDependencyDescriptor(logger logger.Logger) *DependencyDescriptor {
+	return &DependencyDescriptor{
+		Base:      NewBase(logger),
+		decisions: NewSelectorDecisionCache(decisionCacheMaxElements, decisionCacheNackEntries),
+		fnWrapper: FrameNumberWrapper{logger: logger},
+	}
+}
+
+func NewDependencyDescriptorFromOther(vls VideoLayerSelector) *DependencyDescriptor {
+	return &DependencyDescriptor{
+		Base:      vls.getBase(),
+		decisions: NewSelectorDecisionCache(256, 80),
+		fnWrapper: FrameNumberWrapper{logger: vls.getLogger()},
+	}
+}
+
+func (d *DependencyDescriptor) IsOvershootOkay() bool {
+	return false
+}
+
+func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerSelectorResult) {
+	// a packet is always relevant for the svc codec
+	if d.currentLayer.IsValid() {
+		result.IsRelevant = true
+	}
+
+	ddwdt := extPkt.DependencyDescriptor
+	if ddwdt == nil {
+		// packet doesn't have dependency descriptor
+		// d.logger.Debugw(fmt.Sprintf("drop packet, no DD, incoming %v, sn: %d, isKeyFrame: %v", extPkt.VideoLayer, extPkt.Packet.SequenceNumber, extPkt.KeyFrame))
+		return
+	}
+
+	if ddwdt.RestartGeneration > d.restartGeneration {
+		d.logger.Debugw(
+			"stream restarted",
+			"packet", ddwdt.RestartGeneration,
+			"current", d.restartGeneration,
+			"structureKeyFrame", d.extKeyFrameNum,
+			"efn", ddwdt.ExtFrameNum,
+			"lastEfn", d.fnWrapper.LastOrigin(),
+		)
+		d.restart(ddwdt.RestartGeneration)
+	} else if ddwdt.RestartGeneration < d.restartGeneration {
+		// must not happen
+		d.logger.Warnw("packet from old generation", nil, "packet", ddwdt.RestartGeneration, "current", d.restartGeneration)
+	}
+
+	dd := ddwdt.Descriptor
+
+	extFrameNum := ddwdt.ExtFrameNum
+
+	fd := dd.FrameDependencies
+	incomingLayer := buffer.VideoLayer{
+		Spatial:  int32(fd.SpatialId),
+		Temporal: int32(fd.TemporalId),
+	}
+
+	if !d.keyFrameValid && dd.AttachedStructure == nil {
+		// d.logger.Debugw(fmt.Sprintf("drop packet, no attached structure, incoming %v, sn: %d, isKeyFrame: %v", extPkt.VideoLayer, extPkt.Packet.SequenceNumber, extPkt.KeyFrame))
+		return
+	}
+
+	// early return if this frame is already forwarded or dropped
+	sd, err := d.decisions.GetDecision(extFrameNum)
+	if err != nil {
+		// do not mark as dropped as only error is an old frame
+		// d.logger.Debugw(fmt.Sprintf("drop packet on decision error, incoming %v, fn: %d/%d, sn: %d",
+		//	incomingLayer,
+		//	dd.FrameNumber,
+		//	extFrameNum,
+		//	extPkt.Packet.SequenceNumber,
+		// ), "err", err)
+		return
+	}
+	switch sd {
+	case selectorDecisionDropped:
+		// a packet of an alreadty dropped frame, maintain decision
+		// d.logger.Debugw(fmt.Sprintf("drop packet already dropped, incoming %v, fn: %d/%d, sn: %d",
+		//	incomingLayer,
+		//	dd.FrameNumber,
+		//	extFrameNum,
+		//	extPkt.Packet.SequenceNumber,
+		// ))
+		return
+	}
+
+	if ddwdt.StructureUpdated {
+		// d.logger.Debugw("update dependency structure",
+		// 	"structureID", dd.AttachedStructure.StructureId,
+		// 	"structure", dd.AttachedStructure,
+		// 	"decodeTargets", ddwdt.DecodeTargets,
+		// 	"efn", extFrameNum,
+		// 	"sn", extPkt.Packet.SequenceNumber,
+		// 	"isKeyFrame", extPkt.KeyFrame,
+		// 	"currentKeyframe", d.extKeyFrameNum,
+		// )
+
+		d.updateDependencyStructure(dd.AttachedStructure, ddwdt.DecodeTargets, extFrameNum)
+	}
+
+	if ddwdt.ExtKeyFrameNum != d.extKeyFrameNum {
+		// keyframe mismatch, drop and reset chains
+		d.logger.Debugw("drop packet for keyframe mismatch", "incoming", incomingLayer, "efn", extFrameNum, "sn", extPkt.Packet.SequenceNumber, "requiredKeyFrame", ddwdt.ExtKeyFrameNum, "structureKeyFrame", d.extKeyFrameNum)
+		d.decisions.AddDropped(extFrameNum)
+		d.invalidateKeyFrame()
+		return
+	}
+
+	if ddwdt.ActiveDecodeTargetsUpdated {
+		d.updateActiveDecodeTargets(*dd.ActiveDecodeTargetsBitmask)
+	}
+
+	if len(fd.ChainDiffs) != len(d.chains) {
+		d.logger.Debugw("frame chain diff length mismatch", nil,
+			"incoming", incomingLayer,
+			"efn", extFrameNum,
+			"sn", extPkt.Packet.SequenceNumber,
+			"chainDiffs", fd.ChainDiffs,
+			"chains", len(d.chains),
+			"requiredKeyFrame", ddwdt.ExtKeyFrameNum,
+			"structureKeyFrame", d.extKeyFrameNum,
+		)
+		d.decisions.AddDropped(extFrameNum)
+		return
+	}
+
+	for _, chain := range d.chains {
+		chain.OnFrame(extFrameNum, fd)
+	}
+
+	// find decode target closest to targetLayer
+	highestDecodeTarget := buffer.DependencyDescriptorDecodeTarget{
+		Target: -1,
+		Layer:  buffer.InvalidLayer,
+	}
+	var dti dede.DecodeTargetIndication
+	d.decodeTargetsLock.RLock()
+
+	// decodeTargets be sorted from high to low, find the highest decode target that is active and integrity
+	for _, dt := range d.decodeTargets {
+		if !dt.Active() || dt.Layer.Spatial > d.targetLayer.Spatial || dt.Layer.Temporal > d.targetLayer.Temporal {
+			continue
+		}
+
+		frameResult, err := dt.OnFrame(extFrameNum, fd)
+		if err != nil {
+			d.decodeTargetsLock.RUnlock()
+			// dtis error, dependency descriptor might lost
+			d.logger.Warnw(fmt.Sprintf("drop packet for frame detection error,  incoming: %v", incomingLayer), err)
+			d.decisions.AddDropped(extFrameNum)
+			return
+		}
+
+		if frameResult.TargetValid {
+			highestDecodeTarget = dt.DependencyDescriptorDecodeTarget
+			dti = frameResult.DTI
+			break
+		}
+	}
+	d.decodeTargetsLock.RUnlock()
+
+	if highestDecodeTarget.Target < 0 {
+		// no active decode target, do not select
+		// d.logger.Debugw(
+		//	"drop packet for no target found",
+		//	"highestDecodeTarget", highestDecodeTarget,
+		//	"decodeTargets", d.decodeTargets,
+		//	"tagetLayer", d.targetLayer,
+		//	"incoming", incomingLayer,
+		//	"fn", dd.FrameNumber,
+		//	"efn", extFrameNum,
+		//	"sn", extPkt.Packet.SequenceNumber,
+		//	"isKeyFrame", extPkt.KeyFrame,
+		// )
+		d.decisions.AddDropped(extFrameNum)
+		return
+	}
+
+	// DD-TODO : if bandwidth in congest, could drop the 'Discardable' frame
+	if dti == dede.DecodeTargetNotPresent {
+		// d.logger.Debugw(
+		//	"drop packet for decode target not present",
+		//	"highestDecodeTarget", highestDecodeTarget,
+		//	"decodeTargets", d.decodeTargets,
+		//	"tagetLayer", d.targetLayer,
+		//	"incoming", incomingLayer,
+		//	"fn", dd.FrameNumber,
+		//	"efn", extFrameNum,
+		//	"sn", extPkt.Packet.SequenceNumber,
+		//	"isKeyFrame", extPkt.KeyFrame,
+		// )
+		d.decisions.AddDropped(extFrameNum)
+		return
+	}
+
+	// check decodability using reference frames
+	isDecodable := true
+	for _, fdiff := range fd.FrameDiffs {
+		if fdiff == 0 {
+			continue
+		}
+
+		// use relaxed check for frame diff that we have chain intact detection and don't want
+		// to drop packet due to out-of-order packet or recoverable packet loss
+		if sd, _ := d.decisions.GetDecision(extFrameNum - uint64(fdiff)); sd == selectorDecisionDropped {
+			isDecodable = false
+			break
+		}
+	}
+	if !isDecodable {
+		// d.logger.Debugw(
+		//	"drop packet for not decodable",
+		//	"highestDecodeTarget", highestDecodeTarget,
+		//	"decodeTargets", d.decodeTargets,
+		//	"tagetLayer", d.targetLayer,
+		//	"incoming", incomingLayer,
+		//	"fn", dd.FrameNumber,
+		//	"efn", extFrameNum,
+		//	"sn", extPkt.Packet.SequenceNumber,
+		//	"isKeyFrame", extPkt.KeyFrame,
+		// )
+		d.decisions.AddDropped(extFrameNum)
+		return
+	}
+
+	if d.currentLayer != highestDecodeTarget.Layer {
+		result.IsSwitching = true
+		if !d.currentLayer.IsValid() {
+			result.IsResuming = true
+			d.logger.Debugw(
+				"resuming at layer",
+				"current", incomingLayer,
+				"target", d.targetLayer,
+				"max", d.maxLayer,
+				"layer", fd.SpatialId,
+				"req", d.requestSpatial,
+				"maxSeen", d.maxSeenLayer,
+				"feed", extPkt.Packet.SSRC,
+				"fn", dd.FrameNumber,
+				"efn", extFrameNum,
+				"sn", extPkt.Packet.SequenceNumber,
+				"isKeyFrame", extPkt.IsKeyFrame,
+			)
+		}
+
+		d.previousLayer = d.currentLayer
+		d.currentLayer = highestDecodeTarget.Layer
+
+		d.previousActiveDecodeTargetsBitmask = d.activeDecodeTargetsBitmask
+		d.activeDecodeTargetsBitmask = buffer.GetActiveDecodeTargetBitmask(d.currentLayer, ddwdt.DecodeTargets)
+		d.logger.Debugw(
+			"switch to target",
+			"highestDecodeTarget", highestDecodeTarget,
+			"previous", d.previousLayer,
+			"bitmask", *d.activeDecodeTargetsBitmask,
+			"fn", dd.FrameNumber,
+			"efn", extFrameNum,
+			"sn", extPkt.Packet.SequenceNumber,
+			"isKeyFrame", extPkt.IsKeyFrame,
+		)
+
+		result.IsRelevant = true
+	}
+
+	ddExtension := &dede.DependencyDescriptorExtension{
+		Descriptor: dd,
+		Structure:  d.structure,
+	}
+
+	unWrapFn := uint16(d.fnWrapper.UpdateAndGet(extFrameNum, ddwdt.StructureUpdated))
+	var ddClone *dede.DependencyDescriptor
+	if unWrapFn != dd.FrameNumber {
+		clone := *dd
+		ddClone = &clone
+		ddClone.FrameNumber = unWrapFn
+		ddExtension.Descriptor = ddClone
+	}
+
+	if dd.AttachedStructure == nil {
+		if d.activeDecodeTargetsBitmask != nil {
+			if ddClone == nil {
+				// clone and override activebitmask
+				// DD-TODO: if the packet that contains the bitmask is acknowledged by RR, then we don't need it until it changed.
+				clone := *dd
+				ddClone = &clone
+				ddExtension.Descriptor = ddClone
+			}
+			ddClone.ActiveDecodeTargetsBitmask = d.activeDecodeTargetsBitmask
+			// d.logger.Debugw("set active decode targets bitmask", "activeDecodeTargetsBitmask", d.activeDecodeTargetsBitmask)
+		}
+	}
+
+	var ddMarshaled bool
+	func() {
+		defer func() {
+			if r := recover(); r != nil {
+				d.logger.Errorw("panic marshalling dependency descriptor extension", nil,
+					"efn", extFrameNum,
+					"sn", extPkt.Packet.SequenceNumber,
+					"keyframeRequired", ddwdt.ExtKeyFrameNum,
+					"currentKeyframe", d.extKeyFrameNum,
+					"panic", r,
+					"stack", string(debug.Stack()))
+			}
+		}()
+		bytes, err := ddExtension.Marshal()
+		if err != nil {
+			d.logger.Warnw("error marshalling dependency descriptor extension", err)
+		} else {
+			result.DependencyDescriptorExtension = bytes
+			ddMarshaled = true
+		}
+	}()
+
+	if !ddMarshaled {
+		// drop packet if we can't marshal dependency descriptor
+		d.decisions.AddDropped(extFrameNum)
+		return
+	}
+
+	if ddwdt.Integrity {
+		d.decisions.AddForwarded(extFrameNum)
+	}
+	result.RTPMarker = extPkt.Packet.Header.Marker || (dd.LastPacketInFrame && d.currentLayer.Spatial == int32(fd.SpatialId))
+	result.IsSelected = true
+	return
+}
+
+func (d *DependencyDescriptor) Rollback() {
+	d.activeDecodeTargetsBitmask = d.previousActiveDecodeTargetsBitmask
+
+	d.Base.Rollback()
+}
+
+func (d *DependencyDescriptor) updateDependencyStructure(structure *dede.FrameDependencyStructure, decodeTargets []buffer.DependencyDescriptorDecodeTarget, extFrameNum uint64) {
+	d.structure = structure
+	d.extKeyFrameNum = extFrameNum
+	d.keyFrameValid = true
+
+	d.chains = d.chains[:0]
+
+	for chainIdx := 0; chainIdx < structure.NumChains; chainIdx++ {
+		d.chains = append(d.chains, NewFrameChain(d.decisions, chainIdx, d.logger))
+	}
+
+	newTargets := make([]*DecodeTarget, 0, len(decodeTargets))
+	for _, dt := range decodeTargets {
+		var chain *FrameChain
+		// When chain_cnt > 0, each Decode target MUST be protected by exactly one Chain.
+		if structure.NumChains > 0 {
+			chainIdx := structure.DecodeTargetProtectedByChain[dt.Target]
+			if chainIdx >= len(d.chains) {
+				// should not happen
+				d.logger.Errorw("DecodeTargetProtectedByChain chainIdx out of range", nil, "chainIdx", chainIdx, "NumChains", len(d.chains))
+			} else {
+				chain = d.chains[chainIdx]
+			}
+		}
+		newTargets = append(newTargets, NewDecodeTarget(dt, chain))
+	}
+	d.decodeTargetsLock.Lock()
+	d.decodeTargets = newTargets
+	d.decodeTargetsLock.Unlock()
+}
+
+func (d *DependencyDescriptor) updateActiveDecodeTargets(activeDecodeTargetsBitmask uint32) {
+	for _, chain := range d.chains {
+		chain.BeginUpdateActive()
+	}
+
+	d.decodeTargetsLock.RLock()
+	for _, dt := range d.decodeTargets {
+		dt.UpdateActive(activeDecodeTargetsBitmask)
+	}
+	d.decodeTargetsLock.RUnlock()
+
+	for _, chain := range d.chains {
+		chain.EndUpdateActive()
+	}
+}
+
+func (d *DependencyDescriptor) invalidateKeyFrame() {
+	d.keyFrameValid = false
+	d.chains = d.chains[:0]
+	d.decodeTargetsLock.Lock()
+	d.decodeTargets = d.decodeTargets[:0]
+	d.decodeTargetsLock.Unlock()
+}
+
+func (d *DependencyDescriptor) CheckSync() (locked bool, layer int32) {
+	layer = d.GetRequestSpatial()
+	if !d.currentLayer.IsValid() || !d.keyFrameValid {
+		// always declare not locked when trying to resume from nothing
+		return false, layer
+	}
+
+	d.decodeTargetsLock.RLock()
+	defer d.decodeTargetsLock.RUnlock()
+	for _, dt := range d.decodeTargets {
+		if dt.Active() && dt.Layer.Spatial == layer && dt.Valid() {
+			return true, layer
+		}
+	}
+
+	return false, layer
+}
+
+func (d *DependencyDescriptor) restart(generation int) {
+	d.restartGeneration = generation
+	d.invalidateKeyFrame()
+	d.decisions = NewSelectorDecisionCache(decisionCacheMaxElements, decisionCacheNackEntries)
+}

pkg/sfu/videolayerselector/dependencydescriptor_test.go

@@ -0,0 +1,411 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package videolayerselector
+
+import (
+	"slices"
+	"testing"
+
+	"github.com/pion/rtp"
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
+	"github.com/livekit/protocol/logger"
+)
+
+func TestDecodeTarget(t *testing.T) {
+	target := buffer.DependencyDescriptorDecodeTarget{
+		Target: 1,
+		Layer:  buffer.VideoLayer{Spatial: 1, Temporal: 2},
+	}
+
+	t.Run("No Chain", func(t *testing.T) {
+		dt := NewDecodeTarget(target, nil)
+		require.True(t, dt.Valid())
+		// no indication found
+		_, err := dt.OnFrame(1, &dd.FrameDependencyTemplate{
+			DecodeTargetIndications: []dd.DecodeTargetIndication{},
+		})
+		require.Error(t, err)
+
+		ret, err := dt.OnFrame(1, &dd.FrameDependencyTemplate{
+			DecodeTargetIndications: []dd.DecodeTargetIndication{dd.DecodeTargetNotPresent, dd.DecodeTargetRequired},
+		})
+		require.NoError(t, err)
+		require.True(t, ret.TargetValid)
+		require.Equal(t, dd.DecodeTargetRequired, ret.DTI)
+	})
+
+	t.Run("With Chain", func(t *testing.T) {
+		decisions := NewSelectorDecisionCache(256, 80)
+		chain := NewFrameChain(decisions, 1, logger.GetLogger())
+		dt := NewDecodeTarget(target, chain)
+		chain.BeginUpdateActive()
+		dt.UpdateActive(1 << dt.Target)
+		chain.EndUpdateActive()
+		require.True(t, dt.Active())
+		require.False(t, dt.Valid())
+
+		// chain intact
+		frame := &dd.FrameDependencyTemplate{
+			DecodeTargetIndications: []dd.DecodeTargetIndication{dd.DecodeTargetNotPresent, dd.DecodeTargetRequired},
+			ChainDiffs:              []int{0, 0},
+		}
+		chain.OnFrame(1, frame)
+		require.True(t, dt.Valid())
+		ret, err := dt.OnFrame(1, frame)
+		require.NoError(t, err)
+		require.True(t, ret.TargetValid)
+		require.Equal(t, dd.DecodeTargetRequired, ret.DTI)
+
+	})
+}
+
+func TestFrameChain(t *testing.T) {
+	decisions := NewSelectorDecisionCache(256, 3)
+	chain := NewFrameChain(decisions, 0, logger.GetLogger())
+	require.True(t, chain.Broken())
+
+	// chain intact
+	frameNoDiff := &dd.FrameDependencyTemplate{
+		ChainDiffs: []int{0},
+	}
+	// not active
+	require.False(t, chain.OnFrame(1, frameNoDiff))
+
+	chain.BeginUpdateActive()
+	chain.UpdateActive(true)
+	chain.EndUpdateActive()
+
+	require.True(t, chain.OnFrame(1, frameNoDiff))
+	decisions.AddForwarded(1)
+
+	frameDiff1 := &dd.FrameDependencyTemplate{
+		ChainDiffs: []int{1},
+	}
+
+	require.True(t, chain.OnFrame(2, frameDiff1))
+	decisions.AddForwarded(2)
+
+	// frame 5 arrives first , but frame 4 can be recovered by NACK
+	require.True(t, chain.OnFrame(5, frameDiff1))
+	decisions.AddForwarded(5)
+
+	// frame 4 arrives, chain remains intact
+	require.True(t, chain.OnFrame(4, frameDiff1))
+	decisions.AddForwarded(4)
+
+	// frame 3 missed by out of nack range, chain broken
+	decisions.AddForwarded(7)
+	require.True(t, chain.Broken())
+
+	// recovery by non-diff frame
+	require.True(t, chain.OnFrame(1000, frameNoDiff))
+	require.False(t, chain.Broken())
+	decisions.AddForwarded(1000)
+
+	// broken by dropped frame
+	require.True(t, chain.OnFrame(1002, frameDiff1))
+	decisions.AddDropped(1001)
+	require.True(t, chain.Broken())
+
+	// recovery by non-diff frame
+	require.True(t, chain.OnFrame(2000, frameNoDiff))
+	decisions.AddForwarded(2000)
+	decisions.AddDropped(2001)
+	require.False(t, chain.OnFrame(2002, frameDiff1))
+	require.True(t, chain.Broken())
+}
+
+func TestDependencyDescriptor(t *testing.T) {
+	ddSelector := NewDependencyDescriptor(logger.GetLogger())
+	targetLayer := buffer.VideoLayer{Spatial: 1, Temporal: 2}
+	ddSelector.SetTarget(targetLayer)
+	ddSelector.SetRequestSpatial(1)
+
+	// no dd ext, dropped
+	ret := ddSelector.Select(&buffer.ExtPacket{Packet: &rtp.Packet{}}, 0)
+	require.False(t, ret.IsSelected)
+	require.False(t, ret.IsRelevant)
+
+	// non key frame, dropped
+	ret = ddSelector.Select(&buffer.ExtPacket{
+		IsKeyFrame: false,
+		DependencyDescriptor: &buffer.ExtDependencyDescriptor{
+			Descriptor: &dd.DependencyDescriptor{
+				FrameNumber: 1,
+				FrameDependencies: &dd.FrameDependencyTemplate{
+					SpatialId:  int(targetLayer.Spatial),
+					TemporalId: int(targetLayer.Temporal),
+				},
+			},
+		},
+		Packet: &rtp.Packet{},
+	}, 0)
+	require.False(t, ret.IsSelected)
+	require.False(t, ret.IsRelevant)
+
+	frames := createDDFrames(buffer.VideoLayer{Spatial: 2, Temporal: 2}, 3)
+	// key frame, update structure and decode targets
+	ret = ddSelector.Select(frames[0], 0)
+	require.True(t, ret.IsSelected)
+	require.Equal(t, ddSelector.GetCurrent(), ddSelector.GetTarget())
+	sync, _ := ddSelector.CheckSync()
+	require.True(t, sync)
+
+	// forward frame belongs to target layer
+	// drop frame exceeds target layer (not present in target layer or lower layer)
+	// forward frame not present in target layer but present in lower layer
+	var (
+		belongTargetCase bool
+		exceedTargetCase bool
+		lowerTargetCase  bool
+	)
+	idx := 1
+	var frameForwarded, frameDropped []*buffer.ExtPacket
+	for ; idx < len(frames); idx++ {
+		fd := frames[idx].DependencyDescriptor.Descriptor.FrameDependencies
+		ret = ddSelector.Select(frames[idx], 0)
+		switch {
+		case fd.SpatialId == int(targetLayer.Spatial) && fd.TemporalId == int(targetLayer.Temporal):
+			require.True(t, ret.IsSelected)
+			belongTargetCase = true
+			frameForwarded = append(frameForwarded, frames[idx])
+		case fd.SpatialId < int(targetLayer.Spatial) && fd.TemporalId == 0:
+			require.True(t, ret.IsSelected)
+			lowerTargetCase = true
+			frameForwarded = append(frameForwarded, frames[idx])
+		case fd.SpatialId > int(targetLayer.Spatial) || fd.TemporalId > int(targetLayer.Temporal):
+			require.False(t, ret.IsSelected)
+			exceedTargetCase = true
+			frameDropped = append(frameDropped, frames[idx])
+		}
+
+		if belongTargetCase && exceedTargetCase && lowerTargetCase {
+			break
+		}
+	}
+
+	require.True(t, belongTargetCase && exceedTargetCase && lowerTargetCase)
+
+	// select frame already forwarded
+	ret = ddSelector.Select(frameForwarded[0], 0)
+	require.True(t, ret.IsSelected)
+
+	// drop frame already dropped
+	ret = ddSelector.Select(frameDropped[0], 0)
+	require.False(t, ret.IsSelected)
+
+	// drop frame present but not decodable (dependency frame missed)
+	idx++
+	for ; idx < len(frames); idx++ {
+		fd := frames[idx].DependencyDescriptor.Descriptor.FrameDependencies
+		ret = ddSelector.Select(frames[idx], 0)
+		if fd.SpatialId == int(targetLayer.Spatial) && fd.TemporalId == int(targetLayer.Temporal) {
+			break
+		}
+	}
+	notDecodableFrame := frames[idx]
+	notDecodableFrame.DependencyDescriptor.Descriptor.FrameDependencies.FrameDiffs = []int{
+		int(notDecodableFrame.DependencyDescriptor.Descriptor.FrameNumber - frameDropped[0].DependencyDescriptor.Descriptor.FrameNumber),
+	}
+	ret = ddSelector.Select(notDecodableFrame, 0)
+	require.False(t, ret.IsSelected)
+
+	// target layer broken
+	idx++
+	for ; idx < len(frames); idx++ {
+		fd := frames[idx].DependencyDescriptor.Descriptor.FrameDependencies
+		ret = ddSelector.Select(frames[idx], 0)
+		if fd.SpatialId == int(targetLayer.Spatial) && fd.TemporalId == int(targetLayer.Temporal) {
+			break
+		}
+	}
+	brokenFrame := frames[idx]
+	brokenFrame.DependencyDescriptor.Descriptor.FrameDependencies.ChainDiffs[targetLayer.Spatial] =
+		int(notDecodableFrame.DependencyDescriptor.Descriptor.FrameNumber - frameDropped[0].DependencyDescriptor.Descriptor.FrameNumber)
+	ret = ddSelector.Select(brokenFrame, 0)
+	require.False(t, ret.IsSelected)
+
+	// switch to lower layer, forward frame
+	idx++
+	var switchToLower bool
+	for ; idx < len(frames); idx++ {
+		ret = ddSelector.Select(frames[idx], 0)
+		if ret.IsSelected {
+			require.True(t, targetLayer.GreaterThan(ddSelector.GetCurrent()))
+			switchToLower = true
+			break
+		}
+	}
+	require.True(t, switchToLower)
+
+	// not sync with requested layer
+	ddSelector.SetRequestSpatial(targetLayer.Spatial)
+	locked, layer := ddSelector.CheckSync()
+	require.False(t, locked)
+	require.Equal(t, targetLayer.Spatial, layer)
+	// request to current layer, sync
+	ddSelector.SetRequestSpatial(ddSelector.GetCurrent().Spatial)
+	locked, _ = ddSelector.CheckSync()
+	require.True(t, locked)
+
+	// should drop frame that relies on a keyframe is not present in current selection
+	framesPrevious := createDDFrames(buffer.VideoLayer{Spatial: 2, Temporal: 2}, 1000)
+	ret = ddSelector.Select(framesPrevious[1], 0)
+	require.False(t, ret.IsSelected)
+	// keyframe lost, out of sync
+	locked, _ = ddSelector.CheckSync()
+	require.False(t, locked)
+}
+
+func createDDFrames(maxLayer buffer.VideoLayer, startFrameNumber uint16) []*buffer.ExtPacket {
+	var frames []*buffer.ExtPacket
+	var activeBitMask uint32
+	var decodeTargets []buffer.DependencyDescriptorDecodeTarget
+	var decodeTargetsProtectByChain []int
+	for i := 0; i <= int(maxLayer.Spatial); i++ {
+		for j := 0; j <= int(maxLayer.Temporal); j++ {
+			decodeTargets = append(decodeTargets, buffer.DependencyDescriptorDecodeTarget{
+				Target: i*int(maxLayer.Temporal+1) + j,
+				Layer:  buffer.VideoLayer{Spatial: int32(i), Temporal: int32(j)},
+			})
+			decodeTargetsProtectByChain = append(decodeTargetsProtectByChain, i)
+			activeBitMask |= 1 << uint(i*int(maxLayer.Temporal+1)+j)
+		}
+	}
+	slices.SortFunc(decodeTargets, func(a, b buffer.DependencyDescriptorDecodeTarget) int {
+		if a.Layer.GreaterThan(b.Layer) {
+			return -1
+		}
+		if b.Layer.GreaterThan(a.Layer) {
+			return 1
+		}
+		return 0
+	})
+
+	chainDiffs := make([]int, int(maxLayer.Spatial)+1)
+	dtis := make([]dd.DecodeTargetIndication, len(decodeTargets))
+	for _, dt := range decodeTargets {
+		dtis[dt.Target] = dd.DecodeTargetSwitch
+	}
+
+	templates := make([]*dd.FrameDependencyTemplate, len(decodeTargets))
+
+	for _, dt := range decodeTargets {
+		templates[dt.Target] = &dd.FrameDependencyTemplate{
+			SpatialId:               int(dt.Layer.Spatial),
+			TemporalId:              int(dt.Layer.Temporal),
+			ChainDiffs:              chainDiffs,
+			DecodeTargetIndications: dtis,
+		}
+	}
+	keyFrame := &buffer.ExtPacket{
+		IsKeyFrame: true,
+		DependencyDescriptor: &buffer.ExtDependencyDescriptor{
+			Descriptor: &dd.DependencyDescriptor{
+				FrameNumber: startFrameNumber,
+				FrameDependencies: &dd.FrameDependencyTemplate{
+					SpatialId:               0,
+					TemporalId:              0,
+					ChainDiffs:              chainDiffs,
+					DecodeTargetIndications: dtis,
+				},
+				AttachedStructure: &dd.FrameDependencyStructure{
+					NumDecodeTargets:             int((maxLayer.Spatial + 1) * (maxLayer.Temporal + 1)),
+					NumChains:                    int(maxLayer.Spatial) + 1,
+					DecodeTargetProtectedByChain: decodeTargetsProtectByChain,
+					Templates:                    templates,
+				},
+				ActiveDecodeTargetsBitmask: &activeBitMask,
+			},
+			DecodeTargets:              decodeTargets,
+			StructureUpdated:           true,
+			ActiveDecodeTargetsUpdated: true,
+			Integrity:                  true,
+			ExtFrameNum:                uint64(startFrameNumber),
+			ExtKeyFrameNum:             uint64(startFrameNumber),
+		},
+		Packet: &rtp.Packet{
+			Header: rtp.Header{
+				SSRC: 1234,
+			},
+		},
+	}
+
+	frames = append(frames, keyFrame)
+
+	chainPrevFrame := make(map[int]int)
+	for i := 0; i <= int(maxLayer.Spatial); i++ {
+		chainPrevFrame[i] = int(startFrameNumber)
+	}
+	startFrameNumber++
+	for i := 0; i < 10; i++ {
+		for j := len(decodeTargets) - 1; j >= 0; j-- {
+			dt := decodeTargets[j]
+			frameChainDiffs := make([]int, len(chainDiffs))
+			for i := range frameChainDiffs {
+				frameChainDiffs[i] = int(startFrameNumber) - chainPrevFrame[i]
+			}
+
+			frameDtis := make([]dd.DecodeTargetIndication, len(dtis))
+			for k := range frameDtis {
+				if k >= dt.Target {
+					if dt.Layer.Temporal == 0 {
+						frameDtis[k] = dd.DecodeTargetRequired
+					} else {
+						frameDtis[k] = dd.DecodeTargetDiscardable
+					}
+				} else {
+					frameDtis[k] = dd.DecodeTargetNotPresent
+				}
+			}
+
+			frame := &buffer.ExtPacket{
+				DependencyDescriptor: &buffer.ExtDependencyDescriptor{
+					Descriptor: &dd.DependencyDescriptor{
+						FrameNumber: startFrameNumber,
+						FrameDependencies: &dd.FrameDependencyTemplate{
+							SpatialId:               int(dt.Layer.Spatial),
+							TemporalId:              int(dt.Layer.Temporal),
+							ChainDiffs:              frameChainDiffs,
+							DecodeTargetIndications: frameDtis,
+						},
+					},
+					DecodeTargets:  decodeTargets,
+					Integrity:      true,
+					ExtFrameNum:    uint64(startFrameNumber),
+					ExtKeyFrameNum: keyFrame.DependencyDescriptor.ExtFrameNum,
+				},
+				Packet: &rtp.Packet{
+					Header: rtp.Header{
+						SSRC: 1234,
+					},
+				},
+			}
+
+			startFrameNumber++
+
+			if dt.Layer.Temporal == 0 {
+				chainPrevFrame[int(dt.Layer.Spatial)] = int(startFrameNumber)
+			}
+
+			frames = append(frames, frame)
+		}
+	}
+
+	return frames
+}

pkg/sfu/videolayerselector/framechain.go

@@ -0,0 +1,139 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package videolayerselector
+
+import (
+	dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
+	"github.com/livekit/protocol/logger"
+)
+
+type FrameChain struct {
+	logger         logger.Logger
+	decisions      *SelectorDecisionCache
+	broken         bool
+	chainIdx       int
+	active         bool
+	updatingActive bool
+
+	expectFrames []uint64
+}
+
+func NewFrameChain(decisions *SelectorDecisionCache, chainIdx int, logger logger.Logger) *FrameChain {
+	return &FrameChain{
+		logger:    logger,
+		decisions: decisions,
+		broken:    true,
+		chainIdx:  chainIdx,
+		active:    false,
+	}
+}
+
+func (fc *FrameChain) OnFrame(extFrameNum uint64, fd *dd.FrameDependencyTemplate) bool {
+	if !fc.active {
+		return false
+	}
+
+	if len(fd.ChainDiffs) <= fc.chainIdx {
+		fc.logger.Warnw("invalid frame chain diff", nil, "chanIdx", fc.chainIdx, "frame", extFrameNum, "fd", fd)
+		return fc.broken
+	}
+
+	// A decodable frame with frame_chain_fdiff equal to 0 indicates that the Chain is intact.
+	if fd.ChainDiffs[fc.chainIdx] == 0 {
+		if fc.broken {
+			fc.broken = false
+			// fc.logger.Debugw("frame chain intact", "chanIdx", fc.chainIdx, "frame", extFrameNum)
+		}
+		fc.expectFrames = fc.expectFrames[:0]
+		return true
+	}
+
+	if fc.broken {
+		return false
+	}
+
+	prevFrameInChain := extFrameNum - uint64(fd.ChainDiffs[fc.chainIdx])
+	sd, err := fc.decisions.GetDecision(prevFrameInChain)
+	if err != nil {
+		fc.logger.Debugw("could not get decision", "err", err, "chanIdx", fc.chainIdx, "frame", extFrameNum, "prevFrame", prevFrameInChain)
+	}
+
+	var intact bool
+	switch sd {
+	case selectorDecisionForwarded:
+		intact = true
+
+	case selectorDecisionUnknown:
+		// If the previous frame is unknown, means it has not arrived but could be recovered by NACK / out-of-order arrival,
+		// set up a expected callback here to determine if the chain is broken or intact
+		if fc.decisions.ExpectDecision(prevFrameInChain, fc.OnExpectFrameChanged) {
+			intact = true
+			fc.expectFrames = append(fc.expectFrames, prevFrameInChain)
+		}
+	}
+
+	if !intact {
+		fc.broken = true
+		// fc.logger.Debugw("frame chain broken", "chanIdx", fc.chainIdx, "sd", sd, "frame", extFrameNum, "prevFrame", prevFrameInChain)
+	}
+	return intact
+}
+
+func (fc *FrameChain) OnExpectFrameChanged(frameNum uint64, decision selectorDecision) {
+	if fc.broken {
+		return
+	}
+
+	for i, f := range fc.expectFrames {
+		if f == frameNum {
+			if decision != selectorDecisionForwarded {
+				fc.broken = true
+				// fc.logger.Debugw("frame chain broken", "chanIdx", fc.chainIdx, "sd", decision, "frame", frameNum)
+			}
+			fc.expectFrames[i] = fc.expectFrames[len(fc.expectFrames)-1]
+			fc.expectFrames = fc.expectFrames[:len(fc.expectFrames)-1]
+			break
+		}
+	}
+}
+
+func (fc *FrameChain) Broken() bool {
+	return fc.broken
+}
+
+func (fc *FrameChain) BeginUpdateActive() {
+	fc.updatingActive = false
+}
+
+func (fc *FrameChain) UpdateActive(active bool) {
+	fc.updatingActive = fc.updatingActive || active
+}
+
+func (fc *FrameChain) EndUpdateActive() {
+	active := fc.updatingActive
+	fc.updatingActive = false
+
+	if active == fc.active {
+		return
+	}
+
+	// if the chain transit from inactive to active, reset broken to wait a decodable SWITCH frame
+	if !fc.active {
+		fc.broken = true
+		fc.logger.Debugw("frame chain broken by inactive", "chanIdx", fc.chainIdx)
+	}
+
+	fc.active = active
+}

pkg/sfu/videolayerselector/framenumberwrapper.go

@@ -0,0 +1,58 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package videolayerselector
+
+import "github.com/livekit/protocol/logger"
+
+type FrameNumberWrapper struct {
+	offset uint64
+	last   uint64
+	inited bool
+	logger logger.Logger
+}
+
+// UpdateAndGet returns the wrapped frame number from the given frame number, and updates the offset to
+// make sure the returned frame number is always inorder. Should only updateOffset if the new frame is a keyframe
+// because frame dependencies uses on the frame number diff so frames inside a GOP should have the same offset.
+func (f *FrameNumberWrapper) UpdateAndGet(new uint64, updateOffset bool) uint64 {
+	if !f.inited {
+		f.last = new
+		f.inited = true
+		return new
+	}
+
+	if new <= f.last {
+		return new + f.offset
+	}
+
+	if updateOffset {
+		new16 := uint16(new + f.offset)
+		last16 := uint16(f.last + f.offset)
+		// if new frame number wraps around and is considered as earlier by client, increase offset to make it later
+		if diff := new16 - last16; diff > 0x8000 || (diff == 0x8000 && new16 < last16) {
+			// increase offset by 6000, nearly 10 seconds for 30fps video with 3 spatial layers
+			prevOffset := f.offset
+			f.offset += uint64(65535 - diff + 6000)
+
+			f.logger.Debugw("wrap around frame number seen, update offset", "new", new, "last", f.last, "offset", f.offset, "prevOffset", prevOffset, "lastWrapFn", last16, "newWrapFn", new16)
+		}
+	}
+	f.last = new
+	return new + f.offset
+}
+
+func (f *FrameNumberWrapper) LastOrigin() uint64 {
+	return f.last
+}

pkg/sfu/videolayerselector/framenumberwrapper_test.go

@@ -0,0 +1,108 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package videolayerselector
+
+import (
+	"testing"
+
+	"math/rand"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/mediatransportutil/pkg/utils"
+	"github.com/livekit/protocol/logger"
+)
+
+func TestFrameNumberWrapper(t *testing.T) {
+
+	logger.InitFromConfig(&logger.Config{Level: "debug"}, t.Name())
+
+	fnWrap := &FrameNumberWrapper{logger: logger.GetLogger()}
+
+	fnWrapAround := utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false})
+
+	firstF := uint16(1000)
+
+	testFrameOrder := func(frame uint16, isKeyFrame bool, frame2 uint16, isKeyFrame2, expectInorder bool) {
+		frameUnwrap := fnWrapAround.Update(frame).ExtendedVal
+		wrappedFrame := uint16(fnWrap.UpdateAndGet(frameUnwrap, isKeyFrame))
+
+		// make sure wrap around always get in order frame number
+		fnWrapAround.Update(frame + (frame2-frame)/2)
+
+		frame2Unwrap := fnWrapAround.Update(frame2).ExtendedVal
+		wrappedFrame2 := uint16(fnWrap.UpdateAndGet(frame2Unwrap, isKeyFrame2))
+		// keeps order
+		require.Equal(t, expectInorder, inOrder(wrappedFrame2, wrappedFrame), "frame %d, frame2 %d, wrappedFrame %d, wrapped Frame2 %d, frameUnwrap %d, frame2Unwrap %d", frame, frame2, wrappedFrame, wrappedFrame2, frameUnwrap, frame2Unwrap)
+		// frame number diff should be the same if frame2 is not a key frame
+		if !isKeyFrame2 {
+			require.Equal(t, frame2-frame, wrappedFrame2-wrappedFrame)
+		}
+	}
+
+	secondF := getFrame(firstF, true)
+	testFrameOrder(firstF, true, secondF, false, true)
+
+	// non key frame keeps diff and order
+	for i := 0; i < 100; i++ {
+		// frame in order
+		firstF = secondF
+		secondF = getFrame(firstF, true)
+		testFrameOrder(firstF, false, secondF, false, true)
+
+		// frame out of order
+		firstF = secondF
+		secondF = getFrame(firstF, false)
+		// it is possible that an out of order non-keyframe has been converted to in order frame number if the diff is 32768
+		// that is ok because the client can't decode in such case and always need to wait for the key frame.
+		// so it is just a failure of test case and increase the frame number here.
+		if secondF-firstF == 0x8000 {
+			secondF++
+		}
+		testFrameOrder(firstF, false, secondF, false, false)
+
+		// key frame in order
+		firstF = secondF
+		secondF = getFrame(firstF, true)
+		testFrameOrder(firstF, false, secondF, true, true)
+
+		// frame in order
+		firstF = secondF
+		secondF = getFrame(firstF, true)
+		testFrameOrder(firstF, false, secondF, false, true)
+
+		// key frame out of order but should be in order after wrap around
+		firstF = secondF
+		secondF = getFrame(firstF, false)
+		testFrameOrder(firstF, false, secondF, true, true)
+	}
+}
+
+func inOrder(a, b uint16) bool {
+	return a-b < 0x8000 || (a-b == 0x8000 && a > b)
+}
+
+func getFrame(base uint16, inorder bool) uint16 {
+	if inorder {
+		return base + uint16(rand.Intn(0x8000))
+	}
+
+	for {
+		ret := base + uint16(rand.Intn(0x8000)) + 0x8000
+		if !inOrder(ret, base) {
+			return ret
+		}
+	}
+}

pkg/sfu/videolayerselector/null.go

@@ -0,0 +1,29 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package videolayerselector
+
+import (
+	"github.com/livekit/protocol/logger"
+)
+
+type Null struct {
+	*Base
+}
+
+func NewNull(logger logger.Logger) *Null {
+	return &Null{
+		Base: NewBase(logger),
+	}
+}

pkg/sfu/videolayerselector/selectordecisioncache.go

@@ -0,0 +1,197 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package videolayerselector
+
+import (
+	"fmt"
+)
+
+// ----------------------------------------------------------------------
+
+type selectorDecision int
+
+const (
+	selectorDecisionMissing selectorDecision = iota
+	selectorDecisionDropped
+	selectorDecisionForwarded
+	selectorDecisionUnknown
+)
+
+func (s selectorDecision) String() string {
+	switch s {
+	case selectorDecisionMissing:
+		return "MISSING"
+	case selectorDecisionDropped:
+		return "DROPPED"
+	case selectorDecisionForwarded:
+		return "FORWARDED"
+	case selectorDecisionUnknown:
+		return "UNKNOWN"
+	default:
+		return fmt.Sprintf("%d", int(s))
+	}
+}
+
+// ----------------------------------------------------------------------
+
+type SelectorDecisionCache struct {
+	initialized    bool
+	base           uint64
+	last           uint64
+	masks          []uint64
+	numEntries     uint64
+	numNackEntries uint64
+
+	onExpectEntityChanged map[uint64][]func(entity uint64, decision selectorDecision)
+}
+
+func NewSelectorDecisionCache(maxNumElements uint64, numNackEntries uint64) *SelectorDecisionCache {
+	numElements := (maxNumElements*2 + 63) / 64
+	return &SelectorDecisionCache{
+		masks:                 make([]uint64, numElements),
+		numEntries:            numElements * 32, // 2 bits per entry
+		numNackEntries:        numNackEntries,
+		onExpectEntityChanged: make(map[uint64][]func(entity uint64, decision selectorDecision)),
+	}
+}
+
+func (s *SelectorDecisionCache) AddForwarded(entity uint64) {
+	s.addEntity(entity, selectorDecisionForwarded)
+}
+
+func (s *SelectorDecisionCache) AddDropped(entity uint64) {
+	s.addEntity(entity, selectorDecisionDropped)
+}
+
+func (s *SelectorDecisionCache) GetDecision(entity uint64) (selectorDecision, error) {
+	if !s.initialized || entity < s.base {
+		return selectorDecisionMissing, nil
+	}
+
+	if entity > s.last {
+		return selectorDecisionUnknown, nil
+	}
+
+	offset := s.last - entity
+	if offset >= s.numEntries {
+		// asking for something too old
+		return selectorDecisionMissing, fmt.Errorf("too old, oldest: %d, asking: %d", s.last-s.numEntries+1, entity)
+	}
+
+	return s.getEntity(entity), nil
+}
+
+func (s *SelectorDecisionCache) ExpectDecision(entity uint64, f func(entity uint64, decision selectorDecision)) bool {
+	if !s.initialized || entity < s.base {
+		return false
+	}
+
+	if entity < s.last {
+		offset := s.last - entity
+		if offset >= s.numEntries {
+			return false // too old
+		}
+	}
+
+	s.onExpectEntityChanged[entity] = append(s.onExpectEntityChanged[entity], f)
+	return true
+}
+
+func (s *SelectorDecisionCache) addEntity(entity uint64, sd selectorDecision) {
+	if !s.initialized {
+		s.initialized = true
+		s.base = entity
+		s.last = entity
+		s.setEntity(entity, sd)
+		return
+	}
+
+	if entity <= s.base {
+		// before base, too old
+		return
+	}
+
+	if entity <= s.last {
+		s.setEntity(entity, sd)
+		return
+	}
+
+	for e := s.last + 1; e != entity; e++ {
+		s.setEntity(e, selectorDecisionUnknown)
+	}
+
+	// update [last+1-nack, entity-nack) to missing
+	missingStart := s.last
+	if missingStart > s.numNackEntries+s.base {
+		missingStart -= s.numNackEntries
+	} else {
+		missingStart = s.base
+	}
+	missingEnd := entity
+	if missingEnd > s.numNackEntries+s.base {
+		missingEnd -= s.numNackEntries
+	} else {
+		missingEnd = s.base
+	}
+	if missingEnd > missingStart {
+		for e := missingStart; e != missingEnd; e++ {
+			s.setEntityIfUnknown(e, selectorDecisionMissing)
+		}
+	}
+
+	s.setEntity(entity, sd)
+	s.last = entity
+
+	for e, fns := range s.onExpectEntityChanged {
+		if e+s.numEntries < s.last {
+			delete(s.onExpectEntityChanged, e)
+			for _, f := range fns {
+				f(e, selectorDecisionMissing)
+			}
+		}
+	}
+}
+
+func (s *SelectorDecisionCache) setEntityIfUnknown(entity uint64, sd selectorDecision) {
+	if s.getEntity(entity) == selectorDecisionUnknown {
+		s.setEntity(entity, sd)
+	}
+}
+
+func (s *SelectorDecisionCache) setEntity(entity uint64, sd selectorDecision) {
+	index, bitpos := s.getPos(entity)
+	s.masks[index] &= ^(0x3 << bitpos) // clear before bitwise OR
+	s.masks[index] |= (uint64(sd) & 0x3) << bitpos
+
+	if sd != selectorDecisionUnknown {
+		if fns, ok := s.onExpectEntityChanged[entity]; ok {
+			delete(s.onExpectEntityChanged, entity)
+			for _, f := range fns {
+				f(entity, sd)
+			}
+		}
+	}
+}
+
+func (s *SelectorDecisionCache) getEntity(entity uint64) selectorDecision {
+	index, bitpos := s.getPos(entity)
+	return selectorDecision((s.masks[index] >> bitpos) & 0x3)
+}
+
+func (s *SelectorDecisionCache) getPos(entity uint64) (int, int) {
+	// 2 bits per entity, a uint64 mask can hold 32 entities
+	offset := (entity - s.base) % s.numEntries
+	return int(offset >> 5), int(offset&0x1F) * 2
+}

pkg/sfu/videolayerselector/simulcast.go

@@ -0,0 +1,161 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package videolayerselector
+
+import (
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/protocol/logger"
+)
+
+type Simulcast struct {
+	*Base
+}
+
+func NewSimulcast(logger logger.Logger) *Simulcast {
+	return &Simulcast{
+		Base: NewBase(logger),
+	}
+}
+
+func NewSimulcastFromOther(vls VideoLayerSelector) *Simulcast {
+	switch vls := vls.(type) {
+	case *Null:
+		return &Simulcast{
+			Base: vls.Base,
+		}
+
+	case *Simulcast:
+		return &Simulcast{
+			Base: vls.Base,
+		}
+
+	case *DependencyDescriptor:
+		return &Simulcast{
+			Base: vls.Base,
+		}
+
+	case *VP9:
+		return &Simulcast{
+			Base: vls.Base,
+		}
+
+	default:
+		return nil
+	}
+}
+
+func (s *Simulcast) IsOvershootOkay() bool {
+	return true
+}
+
+func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoLayerSelectorResult) {
+	populateSwitches := func(isActive bool, reason string) {
+		result.IsSwitching = true
+
+		if !isActive {
+			result.IsResuming = true
+		}
+
+		if reason != "" {
+			s.logger.Debugw(
+				reason,
+				"previous", s.previousLayer,
+				"current", s.currentLayer,
+				"previousTarget", s.previousTargetLayer,
+				"target", s.targetLayer,
+				"max", s.maxLayer,
+				"layer", layer,
+				"req", s.requestSpatial,
+				"maxSeen", s.maxSeenLayer,
+				"feed", extPkt.Packet.SSRC,
+			)
+		}
+	}
+
+	if s.currentLayer.Spatial != s.targetLayer.Spatial {
+		currentLayer := s.currentLayer
+
+		// Two things to check when not locked to target
+		//   1. Opportunistic layer upgrade - needs a key frame
+		//   2. Need to downgrade - needs a key frame
+		isActive := s.currentLayer.IsValid()
+		found := false
+		reason := ""
+		if extPkt.IsKeyFrame {
+			if s.enableStartAtDesiredQuality && !isActive {
+				// Initial acquisition: latch directly onto the target layer instead of
+				// opportunistically latching onto the first key frame of any lower layer that
+				// happens to arrive first. This avoids a visible low-quality -> high-quality ramp
+				// (e.g. briefly decoding layer 0 before settling on a requested layer 2) for a
+				// subscriber that requested the higher layer.
+				//
+				// The target is chosen by the allocator: during the initial-acquisition grace it
+				// points at the requested layer (so we wait for it); if that layer never shows
+				// up the grace expires and the allocator drops the target to the highest layer
+				// actually seen, so we always end up latching onto a layer that is flowing.
+				if layer == s.targetLayer.Spatial {
+					reason = "acquiring target layer"
+					found = true
+				}
+			} else {
+				// default: opportunistically latch on to / step towards the target layer
+				if layer > s.currentLayer.Spatial && layer <= s.targetLayer.Spatial {
+					reason = "upgrading layer"
+					found = true
+				}
+
+				if layer < s.currentLayer.Spatial && layer >= s.targetLayer.Spatial {
+					reason = "downgrading layer"
+					found = true
+				}
+			}
+
+			if found {
+				currentLayer.Spatial = layer
+				currentLayer.Temporal = extPkt.VideoLayer.Temporal
+			}
+		}
+
+		if found {
+			s.previousLayer = s.currentLayer
+			s.currentLayer = currentLayer
+
+			s.previousTargetLayer = s.targetLayer
+			if s.currentLayer.Spatial >= s.maxLayer.Spatial || s.currentLayer.Spatial == s.maxSeenLayer.Spatial {
+				s.targetLayer.Spatial = s.currentLayer.Spatial
+			}
+
+			populateSwitches(isActive, reason)
+		}
+	}
+
+	// if locked to higher than max layer due to overshoot, check if it can be dialed back
+	if s.currentLayer.Spatial > s.maxLayer.Spatial && layer <= s.maxLayer.Spatial && extPkt.IsKeyFrame {
+		s.previousLayer = s.currentLayer
+		s.currentLayer.Spatial = layer
+
+		s.previousTargetLayer = s.targetLayer
+		if s.currentLayer.Spatial >= s.maxLayer.Spatial || s.currentLayer.Spatial == s.maxSeenLayer.Spatial {
+			s.targetLayer.Spatial = layer
+		}
+
+		populateSwitches(true, "adjusting overshoot")
+	}
+
+	result.RTPMarker = extPkt.Packet.Marker
+	result.IsSelected = layer == s.currentLayer.Spatial
+	result.IsRelevant = false
+	return
+}

pkg/sfu/videolayerselector/simulcast_test.go

@@ -0,0 +1,77 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package videolayerselector
+
+import (
+	"testing"
+
+	"github.com/pion/rtp"
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/protocol/logger"
+)
+
+func keyFrameOnLayer(spatial, temporal int32) *buffer.ExtPacket {
+	return &buffer.ExtPacket{
+		VideoLayer: buffer.VideoLayer{Spatial: spatial, Temporal: temporal},
+		Packet:     &rtp.Packet{},
+		IsKeyFrame: true,
+	}
+}
+
+// On initial acquisition the selector must latch directly onto the target layer and ignore
+// lower-layer key frames that arrive first, so a subscriber requesting the top layer does not
+// briefly decode a lower layer (a visible quality ramp).
+func TestSimulcastSelectAcquiresTargetLayerDirectly(t *testing.T) {
+	s := NewSimulcast(logger.GetLogger())
+	s.SetEnableStartAtDesiredQuality(true)
+	s.SetMax(buffer.VideoLayer{Spatial: 2, Temporal: 2})
+	s.SetMaxSeen(buffer.VideoLayer{Spatial: 2, Temporal: 2})
+	s.SetTarget(buffer.VideoLayer{Spatial: 2, Temporal: 2})
+	s.SetRequestSpatial(2)
+	s.SetCurrent(buffer.InvalidLayer)
+
+	// lower-layer key frames arriving first must NOT be latched
+	require.False(t, s.Select(keyFrameOnLayer(0, 2), 0).IsSelected)
+	require.False(t, s.GetCurrent().IsValid())
+	require.False(t, s.Select(keyFrameOnLayer(1, 2), 1).IsSelected)
+	require.False(t, s.GetCurrent().IsValid())
+
+	// the target layer key frame latches directly
+	require.True(t, s.Select(keyFrameOnLayer(2, 2), 2).IsSelected)
+	require.Equal(t, int32(2), s.GetCurrent().Spatial)
+}
+
+// Acquisition follows whatever target the allocator set: when the target is lowered (e.g. the
+// acquisition grace expired and the allocator fell back to the highest layer seen), the selector
+// latches that lower layer directly. This is the fallback path that prevents a stall when the
+// originally requested layer never shows up.
+func TestSimulcastSelectAcquiresLoweredTarget(t *testing.T) {
+	s := NewSimulcast(logger.GetLogger())
+	s.SetEnableStartAtDesiredQuality(true)
+	s.SetMax(buffer.VideoLayer{Spatial: 2, Temporal: 2})
+	s.SetMaxSeen(buffer.VideoLayer{Spatial: 1, Temporal: 2})
+	// allocator dropped the target to the highest layer actually seen
+	s.SetTarget(buffer.VideoLayer{Spatial: 1, Temporal: 2})
+	s.SetRequestSpatial(1)
+	s.SetCurrent(buffer.InvalidLayer)
+
+	require.False(t, s.Select(keyFrameOnLayer(0, 2), 0).IsSelected)
+	require.False(t, s.GetCurrent().IsValid())
+
+	require.True(t, s.Select(keyFrameOnLayer(1, 2), 1).IsSelected)
+	require.Equal(t, int32(1), s.GetCurrent().Spatial)
+}

pkg/sfu/videolayerselector/temporallayerselector/null.go

@@ -0,0 +1,31 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package temporallayerselector
+
+import (
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+)
+
+type Null struct{}
+
+func NewNull() *Null {
+	return &Null{}
+}
+
+func Select(_extPkt *buffer.ExtPacket, current int32, _target int32) (this int32, next int32) {
+	this = current
+	next = current
+	return
+}

pkg/sfu/videolayerselector/temporallayerselector/temporallayerselector.go

@@ -0,0 +1,21 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package temporallayerselector
+
+import "github.com/livekit/livekit-server/pkg/sfu/buffer"
+
+type TemporalLayerSelector interface {
+	Select(extPkt *buffer.ExtPacket, current int32, target int32) (this int32, next int32)
+}

pkg/sfu/videolayerselector/temporallayerselector/vp8.go

@@ -0,0 +1,57 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package temporallayerselector
+
+import (
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/mediatransportutil/pkg/codec"
+	"github.com/livekit/protocol/logger"
+)
+
+type VP8 struct {
+	logger logger.Logger
+}
+
+func NewVP8(logger logger.Logger) *VP8 {
+	return &VP8{
+		logger: logger,
+	}
+}
+
+func (v *VP8) Select(extPkt *buffer.ExtPacket, current int32, target int32) (this int32, next int32) {
+	this = current
+	next = current
+	if current == target {
+		return
+	}
+
+	vp8, ok := extPkt.Payload.(codec.VP8)
+	if !ok {
+		return
+	}
+
+	tid := extPkt.Temporal
+	if current < target {
+		if tid > current && tid <= target && vp8.S {
+			this = tid
+			next = tid
+		}
+	} else {
+		if extPkt.Packet.Marker {
+			next = target
+		}
+	}
+	return
+}

pkg/sfu/videolayerselector/videolayerselector.go

@@ -0,0 +1,66 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package videolayerselector
+
+import (
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/sfu/videolayerselector/temporallayerselector"
+	"github.com/livekit/protocol/logger"
+)
+
+type VideoLayerSelectorResult struct {
+	IsSelected                    bool
+	IsRelevant                    bool
+	IsSwitching                   bool
+	IsResuming                    bool
+	RTPMarker                     bool
+	DependencyDescriptorExtension []byte
+}
+
+type VideoLayerSelector interface {
+	getBase() *Base
+
+	getLogger() logger.Logger
+
+	IsOvershootOkay() bool
+
+	SetTemporalLayerSelector(tls temporallayerselector.TemporalLayerSelector)
+	SetEnableStartAtDesiredQuality(enable bool)
+
+	SetMax(maxLayer buffer.VideoLayer)
+	SetMaxSpatial(layer int32)
+	SetMaxTemporal(layer int32)
+	GetMax() buffer.VideoLayer
+
+	SetTarget(targetLayer buffer.VideoLayer)
+	GetTarget() buffer.VideoLayer
+
+	SetRequestSpatial(layer int32)
+	GetRequestSpatial() int32
+
+	CheckSync() (locked bool, layer int32)
+
+	SetMaxSeen(maxSeenLayer buffer.VideoLayer)
+	SetMaxSeenSpatial(layer int32)
+	SetMaxSeenTemporal(layer int32)
+	GetMaxSeen() buffer.VideoLayer
+
+	SetCurrent(currentLayer buffer.VideoLayer)
+	GetCurrent() buffer.VideoLayer
+
+	Select(extPkt *buffer.ExtPacket, layer int32) VideoLayerSelectorResult
+	SelectTemporal(extPkt *buffer.ExtPacket) int32
+	Rollback()
+}

pkg/sfu/videolayerselector/vp9.go

@@ -0,0 +1,108 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package videolayerselector
+
+import (
+	"github.com/pion/rtp/codecs"
+
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/protocol/logger"
+)
+
+type VP9 struct {
+	*Base
+}
+
+func NewVP9(logger logger.Logger) *VP9 {
+	return &VP9{
+		Base: NewBase(logger),
+	}
+}
+
+func NewVP9FromOther(vls VideoLayerSelector) *VP9 {
+	return &VP9{Base: vls.getBase()}
+}
+
+func (v *VP9) IsOvershootOkay() bool {
+	return false
+}
+
+func (v *VP9) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerSelectorResult) {
+	vp9, ok := extPkt.Payload.(codecs.VP9Packet)
+	if !ok {
+		return
+	}
+
+	currentLayer := v.currentLayer
+	if v.currentLayer != v.targetLayer {
+		updatedLayer := v.currentLayer
+
+		if !v.currentLayer.IsValid() {
+			if !extPkt.IsKeyFrame {
+				return
+			}
+
+			updatedLayer = extPkt.VideoLayer
+		} else {
+			if v.currentLayer.Temporal != v.targetLayer.Temporal {
+				if v.currentLayer.Temporal < v.targetLayer.Temporal {
+					// temporal scale up
+					if extPkt.VideoLayer.Temporal > v.currentLayer.Temporal && extPkt.VideoLayer.Temporal <= v.targetLayer.Temporal && vp9.U && vp9.B {
+						currentLayer.Temporal = extPkt.VideoLayer.Temporal
+						updatedLayer.Temporal = extPkt.VideoLayer.Temporal
+					}
+				} else {
+					// temporal scale down
+					if vp9.E {
+						updatedLayer.Temporal = v.targetLayer.Temporal
+					}
+				}
+			}
+
+			if v.currentLayer.Spatial != v.targetLayer.Spatial {
+				if v.currentLayer.Spatial < v.targetLayer.Spatial {
+					// spatial scale up
+					if extPkt.VideoLayer.Spatial > v.currentLayer.Spatial && extPkt.VideoLayer.Spatial <= v.targetLayer.Spatial && !vp9.P && vp9.B {
+						currentLayer.Spatial = extPkt.VideoLayer.Spatial
+						updatedLayer.Spatial = extPkt.VideoLayer.Spatial
+					}
+				} else {
+					// spatial scale down
+					if vp9.E {
+						updatedLayer.Spatial = v.targetLayer.Spatial
+					}
+				}
+			}
+		}
+
+		if updatedLayer != v.currentLayer {
+			result.IsSwitching = true
+			if !v.currentLayer.IsValid() && updatedLayer.IsValid() {
+				result.IsResuming = true
+			}
+
+			v.previousLayer = v.currentLayer
+			v.currentLayer = updatedLayer
+		}
+	}
+
+	result.RTPMarker = extPkt.Packet.Marker
+	if vp9.E && extPkt.VideoLayer.Spatial == currentLayer.Spatial && (vp9.P || v.targetLayer.Spatial <= v.currentLayer.Spatial) {
+		result.RTPMarker = true
+	}
+	result.IsSelected = !extPkt.VideoLayer.GreaterThan(currentLayer)
+	result.IsRelevant = true
+	return
+}