snapshot of https://github.com/livekit/livekit.git @ 08c5433449c106943231037d0b892895f6e92d3b (2026-09-15)
This commit is contained in:
upstream-mirror
202
pkg/sfu/interceptor/rtx.go
Normal file
202
pkg/sfu/interceptor/rtx.go
Normal file
@@ -0,0 +1,202 @@
|
||||
// Copyright 2024 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package interceptor
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/pion/interceptor"
|
||||
"github.com/pion/sdp/v3"
|
||||
"github.com/pion/webrtc/v4"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/sfu/buffer"
|
||||
"github.com/livekit/livekit-server/pkg/sfu/utils"
|
||||
"github.com/livekit/protocol/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
SDESRepairRTPStreamIDURI = "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id"
|
||||
|
||||
rtxProbeCount = 10
|
||||
)
|
||||
|
||||
// StreamInfoProber installs a bounded probe that identifies a remote stream from the
|
||||
// mid/rid/rsid header extensions of its packets. Implemented by buffer.Factory.
|
||||
type StreamInfoProber interface {
|
||||
SetStreamInfoProbe(ssrc uint32, probe *buffer.StreamInfoProbe) bool
|
||||
}
|
||||
|
||||
type streamInfo struct {
|
||||
mid string
|
||||
rid string
|
||||
rsid string
|
||||
}
|
||||
|
||||
type RTXInfoExtractorFactory struct {
|
||||
onStreamFound func(*interceptor.StreamInfo)
|
||||
onRTXPairFound func(repair, base uint32, rsid string)
|
||||
prober StreamInfoProber
|
||||
lock sync.Mutex
|
||||
streams map[uint32]streamInfo
|
||||
paired map[uint32]struct{}
|
||||
logger logger.Logger
|
||||
}
|
||||
|
||||
func NewRTXInfoExtractorFactory(
|
||||
onStreamFound func(*interceptor.StreamInfo),
|
||||
onRTXPairFound func(repair, base uint32, rsid string),
|
||||
prober StreamInfoProber,
|
||||
simTracks map[uint32]SimulcastTrackInfo,
|
||||
logger logger.Logger,
|
||||
) *RTXInfoExtractorFactory {
|
||||
f := &RTXInfoExtractorFactory{
|
||||
onStreamFound: onStreamFound,
|
||||
onRTXPairFound: onRTXPairFound,
|
||||
prober: prober,
|
||||
streams: make(map[uint32]streamInfo),
|
||||
paired: make(map[uint32]struct{}),
|
||||
logger: logger,
|
||||
}
|
||||
f.seedSimulcastTracks(simTracks)
|
||||
return f
|
||||
}
|
||||
|
||||
// seedSimulcastTracks pairs migrated streams from the migration info. A migrated client
|
||||
// is mid-stream and stops sending rid/rsid, so the extensions never appear on the wire
|
||||
// and the pairing has to come from what is already known about the tracks.
|
||||
func (f *RTXInfoExtractorFactory) seedSimulcastTracks(simTracks map[uint32]SimulcastTrackInfo) {
|
||||
for ssrc, info := range simTracks {
|
||||
if info.Mid == "" || info.StreamID == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if info.IsRepairStream {
|
||||
f.SetStreamInfo(ssrc, info.Mid, "", info.StreamID)
|
||||
continue
|
||||
}
|
||||
|
||||
f.SetStreamInfo(ssrc, info.Mid, info.StreamID, "")
|
||||
if info.RepairSSRC != 0 {
|
||||
f.SetStreamInfo(info.RepairSSRC, info.Mid, "", info.StreamID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (f *RTXInfoExtractorFactory) NewInterceptor(id string) (interceptor.Interceptor, error) {
|
||||
return &RTXInfoExtractor{
|
||||
factory: f,
|
||||
logger: f.logger,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *RTXInfoExtractorFactory) SetStreamInfo(ssrc uint32, mid, rid, rsid string) {
|
||||
var repairSsrc, baseSsrc uint32
|
||||
var repairSid string
|
||||
f.lock.Lock()
|
||||
|
||||
if mid == "" || (rid == "" && rsid == "") {
|
||||
f.lock.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
// the same stream can be reported by both the packet probe and the migration info
|
||||
if _, ok := f.paired[ssrc]; ok {
|
||||
f.lock.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
if rsid != "" {
|
||||
// repair stream found, find base stream
|
||||
for base, info := range f.streams {
|
||||
if info.mid == mid && info.rid == rsid {
|
||||
repairSsrc = ssrc
|
||||
baseSsrc = base
|
||||
repairSid = rsid
|
||||
delete(f.streams, base)
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// base stream found, find repair stream
|
||||
for repair, info := range f.streams {
|
||||
if info.mid == mid && info.rsid == rid {
|
||||
repairSsrc = repair
|
||||
baseSsrc = ssrc
|
||||
repairSid = rid
|
||||
delete(f.streams, repair)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// no rtx pair found, save it for later
|
||||
if repairSsrc == 0 || baseSsrc == 0 {
|
||||
f.streams[ssrc] = streamInfo{
|
||||
mid: mid,
|
||||
rid: rid,
|
||||
rsid: rsid,
|
||||
}
|
||||
f.lock.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
f.paired[repairSsrc] = struct{}{}
|
||||
f.paired[baseSsrc] = struct{}{}
|
||||
f.lock.Unlock()
|
||||
|
||||
f.onRTXPairFound(repairSsrc, baseSsrc, repairSid)
|
||||
}
|
||||
|
||||
// ------------------------------------------
|
||||
|
||||
type RTXInfoExtractor struct {
|
||||
interceptor.NoOp
|
||||
|
||||
factory *RTXInfoExtractorFactory
|
||||
logger logger.Logger
|
||||
}
|
||||
|
||||
func (u *RTXInfoExtractor) BindRemoteStream(info *interceptor.StreamInfo, reader interceptor.RTPReader) interceptor.RTPReader {
|
||||
u.factory.onStreamFound(info)
|
||||
|
||||
midExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESMidURI})
|
||||
streamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESRTPStreamIDURI})
|
||||
repairStreamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: SDESRepairRTPStreamIDURI})
|
||||
if midExtensionID == 0 || streamIDExtensionID == 0 || repairStreamIDExtensionID == 0 {
|
||||
return reader
|
||||
}
|
||||
|
||||
// Probe on the buffer write path rather than by wrapping this reader. Remote streams
|
||||
// are consumed through SettingEngine.BufferFactory, so nothing here reads the
|
||||
// interceptor chain. pion used to drive the repair stream reader, but since
|
||||
// pion/webrtc#3470 it only does so when the application reads the TrackRemote or
|
||||
// when no BufferFactory is set, neither of which holds.
|
||||
ok := u.factory.prober.SetStreamInfoProbe(info.SSRC, &buffer.StreamInfoProbe{
|
||||
MidExtID: uint8(midExtensionID),
|
||||
RidExtID: uint8(streamIDExtensionID),
|
||||
RsidExtID: uint8(repairStreamIDExtensionID),
|
||||
Tries: rtxProbeCount,
|
||||
OnFound: u.factory.SetStreamInfo,
|
||||
})
|
||||
if !ok {
|
||||
u.logger.Warnw(
|
||||
"could not install stream info probe, rtx pairing will not work", nil,
|
||||
"ssrc", info.SSRC,
|
||||
"mime", info.MimeType,
|
||||
)
|
||||
}
|
||||
|
||||
return reader
|
||||
}
|
||||
183
pkg/sfu/interceptor/unhandlesimulcast.go
Normal file
183
pkg/sfu/interceptor/unhandlesimulcast.go
Normal file
@@ -0,0 +1,183 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package interceptor
|
||||
|
||||
import (
|
||||
"github.com/pion/interceptor"
|
||||
"github.com/pion/rtp"
|
||||
"github.com/pion/sdp/v3"
|
||||
"github.com/pion/webrtc/v4"
|
||||
"go.uber.org/zap/zapcore"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/sfu/utils"
|
||||
"github.com/livekit/protocol/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
simulcastProbeCount = 10
|
||||
)
|
||||
|
||||
type SimulcastTrackInfo struct {
|
||||
Mid string
|
||||
StreamID string
|
||||
RepairSSRC uint32 // set only when `IsRepairStream: false`, i. e. RTX SSRC for the primary stream
|
||||
IsRepairStream bool
|
||||
}
|
||||
|
||||
func (s *SimulcastTrackInfo) MarshalLogObject(e zapcore.ObjectEncoder) error {
|
||||
e.AddString("Mid", s.Mid)
|
||||
e.AddString("StreamID", s.StreamID)
|
||||
e.AddUint32("RepairSSRC", s.RepairSSRC)
|
||||
e.AddBool("IsRepairStream", s.IsRepairStream)
|
||||
return nil
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
type UnhandleSimulcastOption func(u *UnhandleSimulcastInterceptor) error
|
||||
|
||||
func UnhandleSimulcastTracks(logger logger.Logger, tracks map[uint32]SimulcastTrackInfo) UnhandleSimulcastOption {
|
||||
return func(u *UnhandleSimulcastInterceptor) error {
|
||||
u.logger = logger
|
||||
u.simTracks = tracks
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
type UnhandleSimulcastInterceptorFactory struct {
|
||||
opts []UnhandleSimulcastOption
|
||||
}
|
||||
|
||||
func (f *UnhandleSimulcastInterceptorFactory) NewInterceptor(id string) (interceptor.Interceptor, error) {
|
||||
i := &UnhandleSimulcastInterceptor{simTracks: map[uint32]SimulcastTrackInfo{}}
|
||||
for _, o := range f.opts {
|
||||
if err := o(i); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return i, nil
|
||||
}
|
||||
|
||||
func NewUnhandleSimulcastInterceptorFactory(opts ...UnhandleSimulcastOption) (*UnhandleSimulcastInterceptorFactory, error) {
|
||||
return &UnhandleSimulcastInterceptorFactory{opts: opts}, nil
|
||||
}
|
||||
|
||||
type unhandleSimulcastRTPReader struct {
|
||||
SimulcastTrackInfo
|
||||
logger logger.Logger
|
||||
tryTimes int
|
||||
reader interceptor.RTPReader
|
||||
midExtID uint8
|
||||
ridExtID uint8
|
||||
rsidExtID uint8
|
||||
}
|
||||
|
||||
func (u *unhandleSimulcastRTPReader) Read(b []byte, a interceptor.Attributes) (int, interceptor.Attributes, error) {
|
||||
n, a, err := u.reader.Read(b, a)
|
||||
if u.tryTimes < 0 || err != nil {
|
||||
return n, a, err
|
||||
}
|
||||
|
||||
header := rtp.Header{}
|
||||
hsize, err := header.Unmarshal(b[:n])
|
||||
if err != nil {
|
||||
return n, a, nil
|
||||
}
|
||||
var mid, rid, rsid string
|
||||
if payload := header.GetExtension(u.midExtID); payload != nil {
|
||||
mid = string(payload)
|
||||
}
|
||||
|
||||
if payload := header.GetExtension(u.ridExtID); payload != nil {
|
||||
rid = string(payload)
|
||||
}
|
||||
|
||||
if payload := header.GetExtension(u.rsidExtID); payload != nil {
|
||||
rid = string(payload)
|
||||
}
|
||||
|
||||
if mid != "" && (rid != "" || rsid != "") {
|
||||
u.logger.Debugw(
|
||||
"unhandle stream found",
|
||||
"mid", mid,
|
||||
"rid", rid,
|
||||
"rsid", rsid,
|
||||
"ssrc", header.SSRC,
|
||||
"simulcastTrackInfo", u.SimulcastTrackInfo,
|
||||
)
|
||||
u.tryTimes = -1
|
||||
return n, a, nil
|
||||
} else {
|
||||
// ignore padding only packet for probe count
|
||||
if !header.Padding || n-header.MarshalSize()-int(b[n-1]) != 0 {
|
||||
u.tryTimes--
|
||||
}
|
||||
}
|
||||
|
||||
if mid == "" {
|
||||
header.SetExtension(u.midExtID, []byte(u.Mid))
|
||||
}
|
||||
if rid == "" && !u.IsRepairStream {
|
||||
header.SetExtension(u.ridExtID, []byte(u.StreamID))
|
||||
}
|
||||
if rsid == "" && u.IsRepairStream {
|
||||
header.SetExtension(u.rsidExtID, []byte(u.StreamID))
|
||||
}
|
||||
|
||||
hsize2 := header.MarshalSize()
|
||||
|
||||
if hsize2-hsize+n > len(b) { // no enough buf to set extension
|
||||
return n, a, nil
|
||||
}
|
||||
copy(b[hsize2:], b[hsize:n])
|
||||
header.MarshalTo(b)
|
||||
u.logger.Debugw(
|
||||
"unhandle stream injecting",
|
||||
"mid", mid,
|
||||
"rid", rid,
|
||||
"rsid", rsid,
|
||||
"ssrc", header.SSRC,
|
||||
"simulcastTrackInfo", u.SimulcastTrackInfo,
|
||||
)
|
||||
return hsize2 - hsize + n, a, nil
|
||||
}
|
||||
|
||||
type UnhandleSimulcastInterceptor struct {
|
||||
interceptor.NoOp
|
||||
logger logger.Logger
|
||||
simTracks map[uint32]SimulcastTrackInfo
|
||||
}
|
||||
|
||||
func (u *UnhandleSimulcastInterceptor) BindRemoteStream(info *interceptor.StreamInfo, reader interceptor.RTPReader) interceptor.RTPReader {
|
||||
if t, ok := u.simTracks[info.SSRC]; ok {
|
||||
midExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESMidURI})
|
||||
streamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESRTPStreamIDURI})
|
||||
repairStreamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESRepairRTPStreamIDURI})
|
||||
if midExtensionID == 0 || streamIDExtensionID == 0 || repairStreamIDExtensionID == 0 {
|
||||
return reader
|
||||
}
|
||||
|
||||
return &unhandleSimulcastRTPReader{
|
||||
SimulcastTrackInfo: t,
|
||||
logger: u.logger,
|
||||
reader: reader,
|
||||
tryTimes: simulcastProbeCount,
|
||||
midExtID: uint8(midExtensionID),
|
||||
ridExtID: uint8(streamIDExtensionID),
|
||||
rsidExtID: uint8(repairStreamIDExtensionID),
|
||||
}
|
||||
}
|
||||
return reader
|
||||
}
|
||||
Reference in New Issue
Block a user