// Copyright 2023 LiveKit, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package audio import ( "math" "sync" ) const ( silentAudioLevel = 127 negInv20 = -1.0 / 20 ) // -------------------------------------- type AudioLevelConfig struct { // minimum level to be considered active, 0-127, where 0 is loudest ActiveLevel uint8 `yaml:"active_level,omitempty"` // percentile to measure, a participant is considered active if it has exceeded the ActiveLevel more than // MinPercentile% of the time MinPercentile uint8 `yaml:"min_percentile,omitempty"` // interval to update clients, in ms UpdateInterval uint32 `yaml:"update_interval,omitempty"` // smoothing for audioLevel values sent to the client. // audioLevel will be an average of `smooth_intervals`, 0 to disable SmoothIntervals uint32 `yaml:"smooth_intervals,omitempty"` } var ( DefaultAudioLevelConfig = AudioLevelConfig{ ActiveLevel: 35, // -35dBov MinPercentile: 40, UpdateInterval: 400, SmoothIntervals: 2, } ) // -------------------------------------- type AudioLevelParams struct { ClockRate uint32 } // keeps track of audio level for a participant type AudioLevel struct { params AudioLevelParams config AudioLevelConfig // min duration within an observe duration window to be considered active minActiveDuration uint32 smoothFactor float64 activeThreshold float64 lock sync.Mutex smoothedLevel float64 loudestObservedLevel uint8 activeDuration uint32 // ms observedDuration uint32 // ms lastObservedAt int64 highestRTPTimestamp uint32 highestRTPTimestampInitialized bool } func NewAudioLevel(params AudioLevelParams) *AudioLevel { return &AudioLevel{ params: params, smoothFactor: 1, loudestObservedLevel: silentAudioLevel, } } func (l *AudioLevel) SetConfig(config AudioLevelConfig) { l.lock.Lock() defer l.lock.Unlock() l.config = config l.minActiveDuration = uint32(l.config.MinPercentile) * l.config.UpdateInterval / 100 if l.config.SmoothIntervals > 0 { // exponential moving average (EMA), same center of mass with simple moving average (SMA) l.smoothFactor = float64(2) / (float64(l.config.SmoothIntervals + 1)) } else { l.smoothFactor = 1 } l.activeThreshold = ConvertAudioLevel(float64(l.config.ActiveLevel)) } // Observes a new frame func (l *AudioLevel) Observe(level uint8, durationMs uint32, arrivalTime int64) { l.lock.Lock() defer l.lock.Unlock() l.observeLocked(level, durationMs, arrivalTime) } func (l *AudioLevel) observeLocked(level uint8, durationMs uint32, arrivalTime int64) { if l.config.UpdateInterval == 0 { return } l.lastObservedAt = arrivalTime l.observedDuration += durationMs if level <= l.config.ActiveLevel { l.activeDuration += durationMs if l.loudestObservedLevel > level { l.loudestObservedLevel = level } } if l.observedDuration >= l.config.UpdateInterval { smoothedLevel := float64(0.0) // compute and reset if l.activeDuration >= l.minActiveDuration { // adjust loudest observed level by how much of the window was active. // Weight will be 0 if active the entire duration // > 0 if active for longer than observe duration // < 0 if active for less than observe duration activityWeight := 20 * math.Log10(float64(l.activeDuration)/float64(l.config.UpdateInterval)) adjustedLevel := float64(l.loudestObservedLevel) - activityWeight linearLevel := ConvertAudioLevel(adjustedLevel) // exponential smoothing to dampen transients smoothedLevel = l.smoothedLevel + (linearLevel-l.smoothedLevel)*l.smoothFactor } l.resetLocked(smoothedLevel) } } func (l *AudioLevel) ObserveWithRTPTimestamp(level uint8, ts uint32, arrivalTime int64) { l.lock.Lock() defer l.lock.Unlock() if !l.highestRTPTimestampInitialized { l.highestRTPTimestampInitialized = true l.highestRTPTimestamp = ts } if (ts - l.highestRTPTimestamp) < (1 << 31) { durationMs := (ts - l.highestRTPTimestamp) * 1e3 / l.params.ClockRate l.observeLocked(level, durationMs, arrivalTime) l.highestRTPTimestamp = ts } } // returns current smoothed audio level func (l *AudioLevel) GetLevel(now int64) (float64, bool) { l.lock.Lock() defer l.lock.Unlock() if l.config.UpdateInterval == 0 { return 0.0, false } l.resetIfStaleLocked(now) return l.smoothedLevel, l.smoothedLevel >= l.activeThreshold } func (l *AudioLevel) resetIfStaleLocked(arrivalTime int64) { if (arrivalTime-l.lastObservedAt)/1e6 < int64(2*l.config.UpdateInterval) { return } l.resetLocked(0.0) } func (l *AudioLevel) resetLocked(smoothedLevel float64) { l.smoothedLevel = smoothedLevel l.loudestObservedLevel = silentAudioLevel l.activeDuration = 0 l.observedDuration = 0 } // --------------------------------------------------- // convert decibel back to linear func ConvertAudioLevel(level float64) float64 { return math.Pow(10, level*negInv20) }